lundi 15 avril 2019

Can I web scrape from the mobile twitter site?

So I am trying to webscrape from the mobile twitter site https://mobile.twitter.com/CocaCola because here I can see everyone that has liked a tweet whereas the desktop twitter view I can only see a limited amount of the people that liked a tweet. However, when I was trying to extract information it is None even though it should be returning information.

import re
import requests
import urllib
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
from bs4 import BeautifulSoup
import sys
import unittest, time
url = ["https://mobile.twitter.com/CocaCola"]
for x in url:
   d = webdriver.Chrome()
   actions = ActionChains(d)
   d.get(x)
   res = requests.get(x)
   page = urllib.urlopen(x)
   numb = 0;
   SCROLL_PAUSE_TIME = 0.5
# Get scroll height
   last_height = d.execute_script("return document.body.scrollHeight")
   while True:
    # Scroll down to bottom
      d.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    # Wait to load page
      time.sleep(SCROLL_PAUSE_TIME)
    # Calculate new scroll height and compare with last scroll height
      new_height = d.execute_script("return document.body.scrollHeight")
      soup = BeautifulSoup(page, "html.parser")
      print(soup.findAll('div',{"class":"css-1dbjc4n r-1iusvr4 r-46vdb2 r-5f2r5o r-bcqeeo"}))

I am currently trying to extract the content of each tweet.

Aucun commentaire:

Enregistrer un commentaire