vendredi 12 mars 2021

Selenium button click in loop fails after first try

Currently I am working on a web crawler that should be able to download text of a dutch newspaper bank. The first link is working correctly but suddenly the second link creates an error of which I do not know how to fix this.

It seems that selenium is unable to click the button in the second link while it succeeds doing so in the first link.

Do you know what causes the second link (telegraaf page) to fail?

Code:

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import pandas as pd
import numpy as np
import re
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions

from selenium.webdriver.common.action_chains import ActionChains

#Set up the path to the chrome driver
driver = webdriver.Chrome()
html = driver.find_element_by_tag_name('html')

all_details = []
for c in range(1,2):
    try:
        driver.get("https://www.delpher.nl/nl/kranten/results?query=kernenergie&facets%5Bpapertitle%5D%5B%5D=Algemeen+Dagblad&facets%5Bpapertitle%5D%5B%5D=De+Volkskrant&facets%5Bpapertitle%5D%5B%5D=De+Telegraaf&facets%5Bpapertitle%5D%5B%5D=Trouw&page={}&sortfield=date&cql%5B%5D=(date+_gte_+%2201-01-1970%22)&cql%5B%5D=(date+_lte_+%2201-01-2018%22)&coll=ddd".format(c))
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        driver.implicitly_wait(10)
        incategory = driver.find_elements_by_class_name("search-result")
        print("https://www.delpher.nl/nl/kranten/results?query=kernenergie&facets%5Bpapertitle%5D%5B%5D=Algemeen+Dagblad&facets%5Bpapertitle%5D%5B%5D=De+Volkskrant&facets%5Bpapertitle%5D%5B%5D=De+Telegraaf&facets%5Bpapertitle%5D%5B%5D=Trouw&page={}&sortfield=date&cql%5B%5D=(date+_gte_+%2201-01-1970%22)&cql%5B%5D=(date+_lte_+%2201-01-2018%22)&coll=ddd".format(c))

        links = []
        for i in range(len(incategory)):
            item = incategory[i]
                    #get the href property
            a = item.find_element_by_class_name("search-result__thumbnail-link").get_property("href")
                    #Append the link to list links
            links.append(a)     

        # Loop through each link 
        for link in links:
            # get one book url
            driver.get(url=link)
                      
            # newspaper 
            newspaper = driver.find_element_by_xpath("//*[@id='content']/div[2]/div/div[2]/header/h1/span[2]")
            # date of the article
            date = driver.find_element_by_xpath("//*[@id='content']/div[2]/div/div[2]/header/div/ul/li[1]")
            # title of the article
            title = driver.find_element_by_xpath("//*[@id='object-viewer__ocr-panel']/div[2]/h3[26]/a")
            # content of article
            delay = 10
            div_element = WebDriverWait(driver, delay).until(expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="canvas"]')))
            hover = ActionChains(driver).move_to_element(div_element)
            hover.perform()
            div_element.click()

            button = WebDriverWait(driver, delay).until(expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="object-viewer__ocr-button"]')))
            hover = ActionChains(driver).move_to_element(button)
            hover.perform()

            button.click()
            
            element = driver.find_element_by_css_selector(".object-viewer__ocr-panel-results")
            driver.execute_script("$(arguments[0]).click();", element)
            
            try:
                content = driver.find_element_by_xpath('//*[@id="object-viewer__ocr-panel"]/div[2]/div[5]')
                content = content.text
            except:
                content = None
            # Define a dictionary 
            r = {
                "1Newspaper":newspaper.text,
                "2Date":date.text,
                "3Title": title,
                "4Content": content,
            }
            # append r to all details
            all_details.append(r)
            
    except:
# If we run to an error
        print("\n")
        print("error")
            
# save the information into a CSV file
df = pd.DataFrame(all_details)
df = df.to_string()

time.sleep(3)
driver.close()



Aucun commentaire:

Enregistrer un commentaire