Skip to content
This repository has been archived by the owner on Jun 10, 2024. It is now read-only.

Commit

Permalink
g cannot copy.copy in python3.4+
Browse files Browse the repository at this point in the history
  • Loading branch information
binux committed Aug 18, 2016
1 parent ac24604 commit 7c57cbb
Showing 1 changed file with 10 additions and 7 deletions.
17 changes: 10 additions & 7 deletions pyspider/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,14 +223,20 @@ def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port,
help='Fetcher class to be used.')
@click.pass_context
def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent,
timeout, fetcher_cls, async=True, get_object=False, g=None):
timeout, fetcher_cls, async=True, get_object=False, no_input=False):
"""
Run Fetcher.
"""
g = g or ctx.obj
g = ctx.obj
Fetcher = load_cls(None, None, fetcher_cls)

fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor,
if no_input:
inqueue = None
outqueue = None
else:
inqueue = g.scheduler2fetcher
outqueue = g.fetcher2processor
fetcher = Fetcher(inqueue=inqueue, outqueue=outqueue,
poolsize=poolsize, proxy=proxy, async=async)
fetcher.phantomjs_proxy = g.phantomjs_proxy
if user_agent:
Expand Down Expand Up @@ -346,10 +352,7 @@ def webui(ctx, host, port, cdn, scheduler_rpc, fetcher_rpc, max_rate, max_burst,
else:
# get fetcher instance for webui
fetcher_config = g.config.get('fetcher', {})
mock_g = copy.copy(g)
mock_g['scheduler2fetcher'] = None
mock_g['fetcher2processor'] = None
webui_fetcher = ctx.invoke(fetcher, async=False, get_object=True, g=mock_g, **fetcher_config)
webui_fetcher = ctx.invoke(fetcher, async=False, get_object=True, no_input=True, **fetcher_config)

app.config['fetch'] = lambda x: webui_fetcher.fetch(x)

Expand Down

0 comments on commit 7c57cbb

Please sign in to comment.