Currently I am working on a web crawler that should be able to download text of a dutch newspaper bank. The first link is working correctly but suddenly the second link creates an error of which I do not know how to fix this.
It seems that selenium is unable to click the button in the second link while it succeeds doing so in the first link.
Do you know what causes the second link (telegraaf page) to fail?
Code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import pandas as pd
import numpy as np
import re
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.action_chains import ActionChains
#Set up the path to the chrome driver
driver = webdriver.Chrome()
html = driver.find_element_by_tag_name('html')
all_details = []
for c in range(1,2):
try:
driver.get("https://www.delpher.nl/nl/kranten/results?query=kernenergie&facets%5Bpapertitle%5D%5B%5D=Algemeen+Dagblad&facets%5Bpapertitle%5D%5B%5D=De+Volkskrant&facets%5Bpapertitle%5D%5B%5D=De+Telegraaf&facets%5Bpapertitle%5D%5B%5D=Trouw&page={}&sortfield=date&cql%5B%5D=(date+_gte_+%2201-01-1970%22)&cql%5B%5D=(date+_lte_+%2201-01-2018%22)&coll=ddd".format(c))
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
driver.implicitly_wait(10)
incategory = driver.find_elements_by_class_name("search-result")
print("https://www.delpher.nl/nl/kranten/results?query=kernenergie&facets%5Bpapertitle%5D%5B%5D=Algemeen+Dagblad&facets%5Bpapertitle%5D%5B%5D=De+Volkskrant&facets%5Bpapertitle%5D%5B%5D=De+Telegraaf&facets%5Bpapertitle%5D%5B%5D=Trouw&page={}&sortfield=date&cql%5B%5D=(date+_gte_+%2201-01-1970%22)&cql%5B%5D=(date+_lte_+%2201-01-2018%22)&coll=ddd".format(c))
links = []
for i in range(len(incategory)):
item = incategory[i]
#get the href property
a = item.find_element_by_class_name("search-result__thumbnail-link").get_property("href")
#Append the link to list links
links.append(a)
# Loop through each link
for link in links:
# get one book url
driver.get(url=link)
# newspaper
newspaper = driver.find_element_by_xpath("//*[@id='content']/div[2]/div/div[2]/header/h1/span[2]")
# date of the article
date = driver.find_element_by_xpath("//*[@id='content']/div[2]/div/div[2]/header/div/ul/li[1]")
# title of the article
title = driver.find_element_by_xpath("//*[@id='object-viewer__ocr-panel']/div[2]/h3[26]/a")
# content of article
delay = 10
div_element = WebDriverWait(driver, delay).until(expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="canvas"]')))
hover = ActionChains(driver).move_to_element(div_element)
hover.perform()
div_element.click()
button = WebDriverWait(driver, delay).until(expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="object-viewer__ocr-button"]')))
hover = ActionChains(driver).move_to_element(button)
hover.perform()
button.click()
element = driver.find_element_by_css_selector(".object-viewer__ocr-panel-results")
driver.execute_script("$(arguments[0]).click();", element)
try:
content = driver.find_element_by_xpath('//*[@id="object-viewer__ocr-panel"]/div[2]/div[5]')
content = content.text
except:
content = None
# Define a dictionary
r = {
"1Newspaper":newspaper.text,
"2Date":date.text,
"3Title": title,
"4Content": content,
}
# append r to all details
all_details.append(r)
except:
# If we run to an error
print("\n")
print("error")
# save the information into a CSV file
df = pd.DataFrame(all_details)
df = df.to_string()
time.sleep(3)
driver.close()
Aucun commentaire:
Enregistrer un commentaire