How to pass Selenium html page to htmlXpathSelector

Question!

I need to scrape a page who is using javascript. this is why I'm using Selenium. the problem is that selenium can't fetch the required data.

I want to use htmlXmlSelector to try to fetch the data.

how can I pass the html selenium produced to htmlXmlSelector ?

By : DjangoPy


Answers

Try creating Response manually:

from scrapy.http import TextResponse
from scrapy.selector import HtmlXPathSelector

body = '''
By : warvariuc


Manual response with Selenium:

from scrapy.spider import BaseSpider
from scrapy.http import TextResponse
from scrapy.selector import HtmlXPathSelector
import time
from selenium import selenium

class DemoSpider(BaseSpider):
    name="Demo"
    allowed_domains = ['http://www.example.com']
    start_urls = ["http://www.example.com/demo"]

    def __init__(self):
        BaseSpider.__init__(self)
        self.selenium = selenium("127.0.0.1", 4444, "*chrome", self.start_urls[0])
        self.selenium.start()

    def __del__(self):
       self.selenium.stop()

    def parse (self, response):
        sel = self.selenium
        sel.open(response.url)
        time.sleep(2.0) # wait for javascript execution

        #build the response object from Selenium
        body = sel.get_html_source()
        sel_response = TextResponse(url=response.url, body=body, encoding = 'utf-8')
        hxs = HtmlXPathSelector(sel_response)
        hxs.select("//table").extract()
By : samuel5


This video can help you solving your question :)
By: admin