diff --git a/scrapy_webdriver/download.py b/scrapy_webdriver/download.py index f4a28fd..733baa1 100644 --- a/scrapy_webdriver/download.py +++ b/scrapy_webdriver/download.py @@ -1,5 +1,7 @@ -from scrapy import log, version_info -from scrapy.utils.decorator import inthread +import logging + +from scrapy import version_info +from scrapy.utils.decorators import inthread from scrapy.utils.misc import load_object from .http import WebdriverActionRequest, WebdriverRequest, WebdriverResponse @@ -37,13 +39,13 @@ def download_request(self, request, spider): @inthread def _download_request(self, request, spider): """Download a request URL using webdriver.""" - log.msg('Downloading %s with webdriver' % request.url, level=log.DEBUG) + logging.debug(u'Downloading %s with webdriver' % request.url) request.manager.webdriver.get(request.url) return WebdriverResponse(request.url, request.manager.webdriver) @inthread def _do_action_request(self, request, spider): """Perform an action on a previously webdriver-loaded page.""" - log.msg('Running webdriver actions %s' % request.url, level=log.DEBUG) + logging.debug(u'Running webdriver actions %s' % request.url) request.actions.perform() return WebdriverResponse(request.url, request.manager.webdriver) diff --git a/scrapy_webdriver/tests/test_manager.py b/scrapy_webdriver/tests/test_manager.py index 9f356cf..58945b9 100644 --- a/scrapy_webdriver/tests/test_manager.py +++ b/scrapy_webdriver/tests/test_manager.py @@ -1,5 +1,6 @@ from scrapy.crawler import Crawler from scrapy.settings import Settings +from scrapy.spiders import CrawlSpider from selenium import webdriver from scrapy_webdriver.manager import WebdriverManager @@ -29,19 +30,16 @@ class TestBrowser(object): pass settings = self.settings(WEBDRIVER_BROWSER='Firefox') - crawler = Crawler(Settings(values=settings)) - crawler.configure() + crawler = Crawler(CrawlSpider, settings=Settings(values=settings)) browser = WebdriverManager(crawler) assert issubclass(browser._browser, webdriver.Firefox) settings = self.settings(WEBDRIVER_BROWSER=TestBrowser) - crawler = Crawler(Settings(values=settings)) - crawler.configure() + crawler = Crawler(CrawlSpider, settings=Settings(values=settings)) browser = WebdriverManager(crawler) assert issubclass(browser._browser, TestBrowser) settings = self.settings(WEBDRIVER_BROWSER=TestBrowser()) - crawler = Crawler(Settings(values=settings)) - crawler.configure() + crawler = Crawler(CrawlSpider, settings=Settings(values=settings)) browser = WebdriverManager(crawler) assert isinstance(browser._webdriver, TestBrowser) diff --git a/scrapy_webdriver/tests/test_request_queue.py b/scrapy_webdriver/tests/test_request_queue.py index d285327..1887ec2 100644 --- a/scrapy_webdriver/tests/test_request_queue.py +++ b/scrapy_webdriver/tests/test_request_queue.py @@ -1,14 +1,13 @@ from functools import partial +import logging from time import sleep from mock import call, Mock -from scrapy.crawler import Crawler +from scrapy.crawler import CrawlerProcess from scrapy.http import Request -from scrapy import log, signals +from scrapy import signals from scrapy.settings import Settings from scrapy.spider import BaseSpider -from scrapy.xlib.pydispatch import dispatcher -from twisted.internet import reactor from scrapy_webdriver.http import WebdriverRequest @@ -32,9 +31,6 @@ def settings(self, **options): settings.update(**options) return settings - def _stop_reactor(self): - reactor.stop() - def _wait(self, url, *args, **kwargs): sleep(0.1) @@ -44,15 +40,10 @@ def test_priorization(self): webdriver.get.side_effect = self._wait webdriver.page_source = u'' - dispatcher.connect(self._stop_reactor, signal=signals.spider_closed) - - crawler = Crawler(Settings(values=settings)) - crawler.configure() - spider = self.Spider(name='test', domain='testdomain') - crawler.crawl(spider) - crawler.start() - log.start(loglevel='ERROR') - reactor.run() + process = CrawlerProcess(Settings(values=settings)) + process.crawl(self.Spider, name='test', domain='testdomain') + logging.getLogger('scrapy').setLevel(logging.ERROR) + process.start() assert webdriver.get.mock_calls == [ call('http://testdomain/path?wr=0'),