Recently i am learning Scrapy. I would like to scrape zoominfo. I have written user_login function. But i could not redirect to the search page that achieves the data scraping. The url that i would like to redirect is http://ift.tt/1WELmNS
here's my code
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import scrapy
from scrapy.selector import Selector
from scrapy.http import Request, FormRequest
from tutorial.items import TutorialItem
class LoginSpider(scrapy.Spider):
name = 'zoominfo'
login_page = ['http://ift.tt/23gHrqO']
start_urls = [
'http://ift.tt/1WELmNS',
]
headers = {
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Encoding":"gzip, deflate, br",
"Accept-Language":"en-US,en;q=0.5",
"Connectionc":"keep-alive",
"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0",
"Referer":"http://ift.tt/2iTS56k"
}
def init_request(self):
return Request(url=self.login_page, callback=self.login)
def login(self, response):
print "Preparing Login"
return FormRequest.from_response(
response,
headers=self.headers,
formdata={
'username':username,
'password':password},
callback=self.after_login,
dont_filter = Ture,
)
def after_login(self, response):
if username in response.body:
self.log("Success")
self.initialized()
else:
self.log("Bad times")
def parse(self, response):
base_url = 'http://ift.tt/1WELmNS'
text = Selector(response)
item = TutorialItem()
item['title'] = text.xpath('//title/text()').extract()
print {'title':item["title"]}
request = Request(base_url, callback=self.parse)
The main output is below:
2017-01-09 16:52:58 [scrapy.core.engine] INFO: Spider opened
2017-01-09 16:52:59 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2017-01-09 16:52:59 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://ift.tt/2iUiC5B; (referer: None)
{'title': []}
2017-01-09 16:52:59 [scrapy.core.engine] INFO: Closing spider (finished)
Neither the output print "Preparing Login" not print the correct title. Hope someone could give me some hint.Really thanks a lot
Aucun commentaire:
Enregistrer un commentaire