From 9738437bf56d6036b2da6936a40c56fc07ed3843 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 2 Oct 2014 23:00:22 -0400 Subject: [PATCH 01/14] Format dispatcher for 100 chars --- aspen/dispatcher.py | 53 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index e57f89130..c284d87a8 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -107,10 +107,15 @@ def get_wildleaf_fallback(): # check all the possibilities: # node.html, node.html.spt, node.spt, node.html/, %node.html/ %*.html.spt, %*.spt - subnodes = set([ n for n in listnodes(curnode) if not n.startswith('.') ]) # don't serve hidden files + + # don't serve hidden files + subnodes = set([ n for n in listnodes(curnode) if not n.startswith('.') ]) + node_noext, node_ext = splitext(node) - maybe_wild_nodes = [ n for n in sorted(subnodes) if n.startswith("%") ] # only maybe because non-spt files aren't wild + # only maybe because non-spt files aren't wild + maybe_wild_nodes = [ n for n in sorted(subnodes) if n.startswith("%") ] + wild_leaf_ns = [ n for n in maybe_wild_nodes if is_leaf_node(n) and is_spt(n) ] wild_nonleaf_ns = [ n for n in maybe_wild_nodes if not is_leaf_node(n) ] @@ -132,7 +137,8 @@ def get_wildleaf_fallback(): if node == '': # dir request debug(lambda: "...last node is empty") path_so_far = traverse(curnode, node) - # return either an index file or have the path end in '/' which means 404 or autoindex as appropriate + # return either an index file or have the path end in '/' which means 404 or + # autoindex as appropriate found_n = find_index(path_so_far) if found_n is None: found_n = "" @@ -145,13 +151,19 @@ def get_wildleaf_fallback(): elif node in subnodes and is_leaf_node(node): debug(lambda: "...found exact file, must be static") if is_spt(node): - return DispatchResult(DispatchStatus.missing, None, None, "Node %r Not Found" % node) + return DispatchResult( DispatchStatus.missing + , None + , None + , "Node %r Not Found" % node + ) else: found_n = node elif node + ".spt" in subnodes and is_leaf_node(node + ".spt"): debug(lambda: "...found exact spt") found_n = node + ".spt" - elif node_noext + ".spt" in subnodes and is_leaf_node(node_noext + ".spt") and node_ext: # node has an extension + elif node_noext + ".spt" in subnodes and is_leaf_node(node_noext + ".spt") \ + and node_ext: + # node has an extension debug(lambda: "...found indirect spt") # indirect match noext_matched(node) @@ -166,16 +178,28 @@ def get_wildleaf_fallback(): curnode = traverse(curnode, found_n) result = get_wildleaf_fallback() if not result: - return DispatchResult(DispatchStatus.non_leaf, curnode, None, "Tried to access non-leaf node as leaf.") + return DispatchResult( DispatchStatus.non_leaf + , curnode + , None + , "Tried to access non-leaf node as leaf." + ) return result elif node in subnodes: debug(lambda: "exact dirmatch") - return DispatchResult(DispatchStatus.non_leaf, curnode, None, "Tried to access non-leaf node as leaf.") + return DispatchResult( DispatchStatus.non_leaf + , curnode + , None + , "Tried to access non-leaf node as leaf." + ) else: debug(lambda: "fallthrough") result = get_wildleaf_fallback() if not result: - return DispatchResult(DispatchStatus.missing, None, None, "Node %r Not Found" % node) + return DispatchResult( DispatchStatus.missing + , None + , None + , "Node %r Not Found" % node + ) return result if not last_node: # not at last path seg in request @@ -185,7 +209,8 @@ def get_wildleaf_fallback(): debug(lambda: "Exact match " + repr(node)) curnode = traverse(curnode, found_n) elif wild_nonleaf_ns: - # need to match a wildnode, and we're not the last node, so we should match non-leaf first, then leaf + # need to match a wildnode, and we're not the last node, so we should match + # non-leaf first, then leaf found_n = wild_nonleaf_ns[0] wildvals[found_n[1:]] = node debug(lambda: "Wildcard match %r = %r " % (found_n, node)) @@ -194,7 +219,11 @@ def get_wildleaf_fallback(): debug(lambda: "No exact match for " + repr(node)) result = get_wildleaf_fallback() if not result: - return DispatchResult(DispatchStatus.missing, None, None, "Node %r Not Found" % node) + return DispatchResult( DispatchStatus.missing + , None + , None + , "Node %r Not Found" % node + ) return result return DispatchResult(DispatchStatus.okay, curnode, wildvals, "Found.") @@ -269,7 +298,9 @@ def dispatch(website, request, pure_dispatch=False): if pathparts[-1] != '' and matchname in website.indices and \ is_first_index(website.indices, matchbase, matchname): # asked for something that maps to a default index file; redirect to / per issue #175 - debug(lambda: "found default index '%s' maps into %r" % (pathparts[-1], website.indices)) + debug( lambda: "found default index '%s' maps into %r" + % (pathparts[-1], website.indices) + ) uri = request.line.uri location = uri.path.raw[:-len(pathparts[-1])] if uri.querystring.raw: From 55f903c1b0098e8eb5931f3f8be62e171888e6f9 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 2 Oct 2014 23:04:36 -0400 Subject: [PATCH 02/14] Prune unused imports --- tests/test_dispatcher.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index 871e55aa1..2cd78a8a1 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -7,8 +7,7 @@ from pytest import raises import aspen -from aspen import dispatcher, Response -from aspen.http.request import Request +from aspen import Response # Helpers From 810da3a030eec607f415b1b224b9c0c3f1246066 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 2 Oct 2014 23:20:34 -0400 Subject: [PATCH 03/14] Failing tests for dispatch returning a result --- tests/test_dispatcher.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index 2cd78a8a1..13bb2da8f 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -7,7 +7,7 @@ from pytest import raises import aspen -from aspen import Response +from aspen import dispatcher, Response # Helpers @@ -43,6 +43,29 @@ def assert_body(harness, uripath, expected_body): [-----] text/html

Greetings, Program!

""" + +# dispatcher.dispatch +# =================== + +def test_dispatcher_returns_a_result(harness): + request = harness.make_request('Greetings, program!', 'index.html') + result = dispatcher.dispatch(harness.client.website, request) + assert result.status == dispatcher.DispatchStatus.okay + assert result.match == os.path.join(harness.fs.www.root, 'index.html') + assert result.wildcards == {} + assert result.detail == 'Found.' + +def test_dispatcher_returns_a_result_for_autoindex(harness): + request = harness.make_request('Greetings, program!', 'index.html') + os.remove(request.fs) + harness.client.website.list_directories = True + result = dispatcher.dispatch(harness.client.website, request) + assert result.status == dispatcher.DispatchStatus.okay + assert result.match == os.path.join(harness.fs.www.root, '') + assert result.wildcards == {} + assert result.detail == 'Found.' + + # Indices # ======= @@ -462,4 +485,3 @@ def test_dont_serve_hidden_files(harness): def test_dont_serve_spt_file_source(harness): harness.fs.www.mk(('foo.html.spt', "Greetings, program!"),) assert_raises_404(harness, '/foo.html.spt') - From bfa061f0e12163449366ac9a8d19823b29a7fa4a Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 2 Oct 2014 23:34:28 -0400 Subject: [PATCH 04/14] Return a result from dispatcher.dispatch --- aspen/algorithms/website.py | 2 +- aspen/dispatcher.py | 9 ++++----- tests/test_dispatcher.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index 9622e4d0d..1ef737d37 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -61,7 +61,7 @@ def raise_200_for_OPTIONS(request): def dispatch_request_to_filesystem(website, request): - dispatcher.dispatch(website, request) + return {'dispatch_result': dispatcher.dispatch(website, request)} def apply_typecasters_to_path(website, request): diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index c284d87a8..fd9ccbd89 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -55,9 +55,7 @@ class DispatchStatus: okay, missing, non_leaf = range(3) -DispatchResult = namedtuple( 'DispatchResult' - , 'status match wildcards detail'.split() - ) +DispatchResult = namedtuple('DispatchResult', 'status match wildcards detail'.split()) def dispatch_abstract(listnodes, is_leaf, traverse, find_index, noext_matched, @@ -317,7 +315,7 @@ def dispatch(website, request, pure_dispatch=False): if result.status != DispatchStatus.okay: path = request.line.uri.path.raw[1:] request.fs = website.find_ours(path) - return + return DispatchResult(DispatchStatus.okay, request.fs, {}, 'Found.') # robots.txt @@ -342,7 +340,7 @@ def dispatch(website, request, pure_dispatch=False): assert autoindex is not None # sanity check request.headers['X-Aspen-AutoIndexDir'] = result.match request.fs = autoindex - return # return so we skip the no-escape check + return result # return so we skip the no-escape check else: # normal match request.fs = result.match for k, v in result.wildcards.iteritems(): @@ -368,3 +366,4 @@ def dispatch(website, request, pure_dispatch=False): if not request.fs.startswith(startdir): raise Response(404) + return result diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index 13bb2da8f..b2719ec2a 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -5,9 +5,11 @@ import os from pytest import raises +from StringIO import StringIO import aspen from aspen import dispatcher, Response +from aspen.http.request import Request # Helpers @@ -55,6 +57,18 @@ def test_dispatcher_returns_a_result(harness): assert result.wildcards == {} assert result.detail == 'Found.' +def test_dispatcher_returns_a_result_for_favicon(harness): + request = Request.from_wsgi({ b'REQUEST_METHOD': b'GET' + , b'PATH_INFO': b'/favicon.ico' + , b'SERVER_PROTOCOL': b'HTTP/1.1' + , b'wsgi.input': StringIO() + }) + result = dispatcher.dispatch(harness.client.website, request) + assert result.status == dispatcher.DispatchStatus.okay + assert result.match == harness.client.website.find_ours('favicon.ico') + assert result.wildcards == {} + assert result.detail == 'Found.' + def test_dispatcher_returns_a_result_for_autoindex(harness): request = harness.make_request('Greetings, program!', 'index.html') os.remove(request.fs) From e0e4f6cfc73939c0a3d1f5668307ca853935cb5f Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 2 Oct 2014 23:50:52 -0400 Subject: [PATCH 05/14] Stop abusing request.headers Instead of overloading request.headers for implicit content negotiation and autoindexing, store the needed info in a dispatch_result.extra dict, passing dispatch_result around as needed. --- aspen/algorithms/website.py | 6 ++--- aspen/dispatcher.py | 31 +++++++++++++++++--------- aspen/resources/dynamic_resource.py | 11 ++++----- aspen/resources/negotiated_resource.py | 3 ++- aspen/resources/static_resource.py | 2 +- aspen/www/autoindex.html.spt | 2 +- 6 files changed, 33 insertions(+), 22 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index 1ef737d37..a166db004 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -68,13 +68,13 @@ def apply_typecasters_to_path(website, request): typecasting.apply_typecasters(website.typecasters, request.line.uri.path) -def get_resource_for_request(website, request): +def get_resource_for_request(website, request, dispatch_result): return {'resource': resources.get(website, request)} -def get_response_for_resource(request, resource=None): +def get_response_for_resource(request, dispatch_result, resource=None): if resource is not None: - return {'response': resource.respond(request)} + return {'response': resource.respond(request, dispatch_result)} def get_response_for_exception(website, exception): diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index fd9ccbd89..df00aba9c 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -55,7 +55,7 @@ class DispatchStatus: okay, missing, non_leaf = range(3) -DispatchResult = namedtuple('DispatchResult', 'status match wildcards detail'.split()) +DispatchResult = namedtuple('DispatchResult', 'status match wildcards detail extra'.split()) def dispatch_abstract(listnodes, is_leaf, traverse, find_index, noext_matched, @@ -98,7 +98,7 @@ def get_wildleaf_fallback(): ext = lastnode_ext if lastnode_ext in wildleafs else None curnode, wildvals = wildleafs[ext] debug(lambda: "Wildcard leaf match %r and ext %r" % (curnode, ext)) - return DispatchResult(DispatchStatus.okay, curnode, wildvals, "Found.") + return DispatchResult(DispatchStatus.okay, curnode, wildvals, "Found.", {}) return None for depth, node in enumerate(nodepath): @@ -145,7 +145,7 @@ def get_wildleaf_fallback(): curnode = traverse(curnode, found_n) node_name = found_n[1:-4] # strip leading % and trailing .spt wildvals[node_name] = node - return DispatchResult(DispatchStatus.okay, curnode, wildvals, "Found.") + return DispatchResult(DispatchStatus.okay, curnode, wildvals, "Found.", {}) elif node in subnodes and is_leaf_node(node): debug(lambda: "...found exact file, must be static") if is_spt(node): @@ -153,6 +153,7 @@ def get_wildleaf_fallback(): , None , None , "Node %r Not Found" % node + , {} ) else: found_n = node @@ -180,6 +181,7 @@ def get_wildleaf_fallback(): , curnode , None , "Tried to access non-leaf node as leaf." + , {} ) return result elif node in subnodes: @@ -188,6 +190,7 @@ def get_wildleaf_fallback(): , curnode , None , "Tried to access non-leaf node as leaf." + , {} ) else: debug(lambda: "fallthrough") @@ -197,6 +200,7 @@ def get_wildleaf_fallback(): , None , None , "Node %r Not Found" % node + , {} ) return result @@ -221,10 +225,11 @@ def get_wildleaf_fallback(): , None , None , "Node %r Not Found" % node + , {} ) return result - return DispatchResult(DispatchStatus.okay, curnode, wildvals, "Found.") + return DispatchResult(DispatchStatus.okay, curnode, wildvals, "Found.", {}) def match_index(indices, indir): @@ -246,12 +251,12 @@ def is_first_index(indices, basedir, name): return False -def update_neg_type(website, request, filename): +def update_neg_type(website, capture_accept, filename): media_type = mimetypes.guess_type(filename, strict=False)[0] if media_type is None: media_type = website.media_type_default - request.headers['X-Aspen-Accept'] = media_type - debug(lambda: "set x-aspen-accept to %r" % media_type) + capture_accept['accept'] = media_type + debug(lambda: "set result.extra['accept'] to %r" % media_type) def dispatch(website, request, pure_dispatch=False): @@ -259,7 +264,7 @@ def dispatch(website, request, pure_dispatch=False): This is all side-effecty on the request object, setting, at the least, request.fs, and at worst other random contents including but not limited - to: request.line.uri.path, request.headers. + to: request.line.uri.path. """ @@ -269,11 +274,12 @@ def dispatch(website, request, pure_dispatch=False): # Set up the real environment for the dispatcher. # =============================================== + capture_accept = {} listnodes = os.listdir is_leaf = os.path.isfile traverse = os.path.join find_index = lambda x: match_index(website.indices, x) - noext_matched = lambda x: update_neg_type(website, request, x) + noext_matched = lambda x: update_neg_type(website, capture_accept, x) startdir = website.www_root # Dispatch! @@ -290,6 +296,9 @@ def dispatch(website, request, pure_dispatch=False): debug(lambda: "dispatch_abstract returned: " + repr(result)) + if 'accept' in capture_accept: + result.extra['accept'] = capture_accept['accept'] + if result.match: debug(lambda: "result.match is true" ) matchbase, matchname = result.match.rsplit(os.path.sep,1) @@ -315,7 +324,7 @@ def dispatch(website, request, pure_dispatch=False): if result.status != DispatchStatus.okay: path = request.line.uri.path.raw[1:] request.fs = website.find_ours(path) - return DispatchResult(DispatchStatus.okay, request.fs, {}, 'Found.') + return DispatchResult(DispatchStatus.okay, request.fs, {}, 'Found.', {}) # robots.txt @@ -338,8 +347,8 @@ def dispatch(website, request, pure_dispatch=False): raise Response(404) autoindex = website.ours_or_theirs('autoindex.html.spt') assert autoindex is not None # sanity check - request.headers['X-Aspen-AutoIndexDir'] = result.match request.fs = autoindex + result.extra['autoindexdir'] = result.match return result # return so we skip the no-escape check else: # normal match request.fs = result.match diff --git a/aspen/resources/dynamic_resource.py b/aspen/resources/dynamic_resource.py index 5dca15552..f60033141 100644 --- a/aspen/resources/dynamic_resource.py +++ b/aspen/resources/dynamic_resource.py @@ -37,7 +37,7 @@ def __init__(self, *a, **kw): self.pages = self.compile_pages(pages) - def respond(self, request, response=None): + def respond(self, request, dispatch_result, response=None): """Given a Request and maybe a Response, return or raise a Response. """ response = response or Response(charset=self.website.charset_dynamic) @@ -46,7 +46,7 @@ def respond(self, request, response=None): # Populate context. # ================= - context = self.populate_context(request, response) + context = self.populate_context(request, dispatch_result, response) # Exec page two. @@ -77,7 +77,7 @@ def respond(self, request, response=None): return response - def populate_context(self, request, response): + def populate_context(self, request, dispatch_result, response): """Factored out to support testing. """ dynamics = { 'body' : lambda: request.body } @@ -104,8 +104,9 @@ def __getitem__(self, key): # don't let the page override these context.update({ 'request' : request, - 'response': response, - 'resource': self + 'dispatch_result': dispatch_result, + 'resource': self, + 'response': response }) return context diff --git a/aspen/resources/negotiated_resource.py b/aspen/resources/negotiated_resource.py index a20a5e983..05bed35d8 100644 --- a/aspen/resources/negotiated_resource.py +++ b/aspen/resources/negotiated_resource.py @@ -73,9 +73,10 @@ def get_response(self, context): """Given a context dict, return a response object. """ request = context['request'] + dispatch_result = context['dispatch_result'] # find an Accept header - accept = request.headers.get('X-Aspen-Accept', None) + accept = dispatch_result.extra.get('accept', None) if accept is not None: # indirect negotiation failure = Response(404) else: # direct negotiation diff --git a/aspen/resources/static_resource.py b/aspen/resources/static_resource.py index f862bf93a..e454bac2d 100644 --- a/aspen/resources/static_resource.py +++ b/aspen/resources/static_resource.py @@ -19,7 +19,7 @@ def __init__(self, *a, **kw): if self.media_type == 'application/json': self.media_type = self.website.media_type_json - def respond(self, request, response=None): + def respond(self, request, dispatch_result, response=None): """Given a Request and maybe a Response, return or raise a Response. """ response = response or Response() diff --git a/aspen/www/autoindex.html.spt b/aspen/www/autoindex.html.spt index 232316a81..4c8d7c254 100644 --- a/aspen/www/autoindex.html.spt +++ b/aspen/www/autoindex.html.spt @@ -53,7 +53,7 @@ def _get_time(stats): # Support the case where we are in the directory where this file actually # lives! -fspath = request.headers.get('X-Aspen-AutoIndexDir', os.path.dirname(__file__)) +fspath = dispatch_result.get('autoindexdir', os.path.dirname(__file__)) assert os.path.isdir(fspath) # sanity check urlpath = fspath[len(website.www_root):] + os.sep From 1555bdb22af30e5c8466cc030774d36bfbffb823 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Oct 2014 00:06:10 -0400 Subject: [PATCH 06/14] Remove dependence on request.uri in dispatcher --- aspen/algorithms/website.py | 11 +++++++++-- aspen/dispatcher.py | 33 ++++++++++++--------------------- tests/test_dispatcher.py | 11 ++++++++--- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index a166db004..9aa127514 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -61,8 +61,15 @@ def raise_200_for_OPTIONS(request): def dispatch_request_to_filesystem(website, request): - return {'dispatch_result': dispatcher.dispatch(website, request)} - + result = dispatcher.dispatch( website + , request + , pathparts=request.line.uri.path.parts + , uripath=request.line.uri.path.raw + , querystring=request.line.uri.querystring.raw + ) + for k, v in result.wildcards.iteritems(): + request.line.uri.path[k] = v + return {'dispatch_result': result} def apply_typecasters_to_path(website, request): typecasting.apply_typecasters(website.typecasters, request.line.uri.path) diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index df00aba9c..5bf0c27bf 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -259,18 +259,13 @@ def update_neg_type(website, capture_accept, filename): debug(lambda: "set result.extra['accept'] to %r" % media_type) -def dispatch(website, request, pure_dispatch=False): +def dispatch(website, request, pathparts, uripath, querystring, pure_dispatch=False): """Concretize dispatch_abstract. - This is all side-effecty on the request object, setting, at the least, - request.fs, and at worst other random contents including but not limited - to: request.line.uri.path. + This is side-effecty on the request object, setting request.fs. """ - # Handle URI path parts - pathparts = request.line.uri.path.parts - # Set up the real environment for the dispatcher. # =============================================== @@ -282,6 +277,7 @@ def dispatch(website, request, pure_dispatch=False): noext_matched = lambda x: update_neg_type(website, capture_accept, x) startdir = website.www_root + # Dispatch! # ========= @@ -308,10 +304,9 @@ def dispatch(website, request, pure_dispatch=False): debug( lambda: "found default index '%s' maps into %r" % (pathparts[-1], website.indices) ) - uri = request.line.uri - location = uri.path.raw[:-len(pathparts[-1])] - if uri.querystring.raw: - location += '?' + uri.querystring.raw + location = uripath[:-len(pathparts[-1])] + if querystring: + location += '?' + querystring raise Response(302, headers={'Location': location}) if not pure_dispatch: @@ -320,10 +315,9 @@ def dispatch(website, request, pure_dispatch=False): # =========== # Serve Aspen's favicon if there's not one. - if request.line.uri.path.raw == '/favicon.ico': + if uripath == '/favicon.ico': if result.status != DispatchStatus.okay: - path = request.line.uri.path.raw[1:] - request.fs = website.find_ours(path) + request.fs = website.find_ours('favicon.ico') return DispatchResult(DispatchStatus.okay, request.fs, {}, 'Found.', {}) @@ -332,7 +326,7 @@ def dispatch(website, request, pure_dispatch=False): # Don't let robots.txt be handled by anything other than an actual # robots.txt file - if request.line.uri.path.raw == '/robots.txt': + if uripath == '/robots.txt': if result.status != DispatchStatus.missing: if not result.match.endswith('robots.txt'): raise Response(404) @@ -352,14 +346,11 @@ def dispatch(website, request, pure_dispatch=False): return result # return so we skip the no-escape check else: # normal match request.fs = result.match - for k, v in result.wildcards.iteritems(): - request.line.uri.path[k] = v elif result.status == DispatchStatus.non_leaf: # trailing-slash redirect - uri = request.line.uri - location = uri.path.raw + '/' - if uri.querystring.raw: - location += '?' + uri.querystring.raw + location = uripath + '/' + if querystring: + location += '?' + querystring raise Response(302, headers={'Location': location}) elif result.status == DispatchStatus.missing: # 404 diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index b2719ec2a..58a5823ca 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -51,7 +51,7 @@ def assert_body(harness, uripath, expected_body): def test_dispatcher_returns_a_result(harness): request = harness.make_request('Greetings, program!', 'index.html') - result = dispatcher.dispatch(harness.client.website, request) + result = dispatcher.dispatch(harness.client.website, request, [''], '/', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == os.path.join(harness.fs.www.root, 'index.html') assert result.wildcards == {} @@ -63,7 +63,12 @@ def test_dispatcher_returns_a_result_for_favicon(harness): , b'SERVER_PROTOCOL': b'HTTP/1.1' , b'wsgi.input': StringIO() }) - result = dispatcher.dispatch(harness.client.website, request) + result = dispatcher.dispatch( harness.client.website + , request + , ['favicon.ico'] + , '/favicon.ico' + , '' + ) assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('favicon.ico') assert result.wildcards == {} @@ -73,7 +78,7 @@ def test_dispatcher_returns_a_result_for_autoindex(harness): request = harness.make_request('Greetings, program!', 'index.html') os.remove(request.fs) harness.client.website.list_directories = True - result = dispatcher.dispatch(harness.client.website, request) + result = dispatcher.dispatch(harness.client.website, request, [''], '/', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == os.path.join(harness.fs.www.root, '') assert result.wildcards == {} From ba08b26dd345e0037619aae3809110cd696c1bce Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Oct 2014 00:18:07 -0400 Subject: [PATCH 07/14] Move setting of request.fs out of dispatcher --- aspen/algorithms/website.py | 1 + aspen/dispatcher.py | 27 +++++++++++++++------------ tests/test_dispatcher.py | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index 9aa127514..517fd3a4c 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -67,6 +67,7 @@ def dispatch_request_to_filesystem(website, request): , uripath=request.line.uri.path.raw , querystring=request.line.uri.querystring.raw ) + request.fs = result.match for k, v in result.wildcards.iteritems(): request.line.uri.path[k] = v return {'dispatch_result': result} diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index 5bf0c27bf..bc96c100a 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -13,7 +13,6 @@ import os from aspen import Response -from .backcompat import namedtuple def debug_noop(*args, **kwargs): @@ -51,11 +50,17 @@ def debug_ext(): return a, b -class DispatchStatus: +class DispatchStatus(object): okay, missing, non_leaf = range(3) -DispatchResult = namedtuple('DispatchResult', 'status match wildcards detail extra'.split()) +class DispatchResult(object): + def __init__(self, status, match, wildcards, detail, extra): + self.status = status + self.match = match + self.wildcards = wildcards + self.detail = detail + self.extra = extra def dispatch_abstract(listnodes, is_leaf, traverse, find_index, noext_matched, @@ -261,9 +266,6 @@ def update_neg_type(website, capture_accept, filename): def dispatch(website, request, pathparts, uripath, querystring, pure_dispatch=False): """Concretize dispatch_abstract. - - This is side-effecty on the request object, setting request.fs. - """ # Set up the real environment for the dispatcher. @@ -317,8 +319,11 @@ def dispatch(website, request, pathparts, uripath, querystring, pure_dispatch=Fa if uripath == '/favicon.ico': if result.status != DispatchStatus.okay: - request.fs = website.find_ours('favicon.ico') - return DispatchResult(DispatchStatus.okay, request.fs, {}, 'Found.', {}) + result.status = DispatchStatus.okay + result.match = website.find_ours('favicon.ico') + result.wildcards = {} + result.detail = 'Found.' + return result # robots.txt @@ -341,11 +346,9 @@ def dispatch(website, request, pathparts, uripath, querystring, pure_dispatch=Fa raise Response(404) autoindex = website.ours_or_theirs('autoindex.html.spt') assert autoindex is not None # sanity check - request.fs = autoindex + result.match = autoindex result.extra['autoindexdir'] = result.match return result # return so we skip the no-escape check - else: # normal match - request.fs = result.match elif result.status == DispatchStatus.non_leaf: # trailing-slash redirect location = uripath + '/' @@ -363,7 +366,7 @@ def dispatch(website, request, pathparts, uripath, querystring, pure_dispatch=Fa # Protect against escaping the www_root. # ====================================== - if not request.fs.startswith(startdir): + if not result.match.startswith(startdir): raise Response(404) return result diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index 58a5823ca..c6fd02088 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -80,7 +80,7 @@ def test_dispatcher_returns_a_result_for_autoindex(harness): harness.client.website.list_directories = True result = dispatcher.dispatch(harness.client.website, request, [''], '/', '') assert result.status == dispatcher.DispatchStatus.okay - assert result.match == os.path.join(harness.fs.www.root, '') + assert result.match == harness.client.website.find_ours('autoindex.html.spt') assert result.wildcards == {} assert result.detail == 'Found.' From 45cf3940da169740ae19dd9a136957c88b7d42d2 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Oct 2014 00:20:45 -0400 Subject: [PATCH 08/14] Stop passing request to dispatch entirely --- aspen/algorithms/website.py | 1 - aspen/dispatcher.py | 2 +- tests/test_dispatcher.py | 20 ++++---------------- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index 517fd3a4c..b1dde4a13 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -62,7 +62,6 @@ def raise_200_for_OPTIONS(request): def dispatch_request_to_filesystem(website, request): result = dispatcher.dispatch( website - , request , pathparts=request.line.uri.path.parts , uripath=request.line.uri.path.raw , querystring=request.line.uri.querystring.raw diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index bc96c100a..d19da1d40 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -264,7 +264,7 @@ def update_neg_type(website, capture_accept, filename): debug(lambda: "set result.extra['accept'] to %r" % media_type) -def dispatch(website, request, pathparts, uripath, querystring, pure_dispatch=False): +def dispatch(website, pathparts, uripath, querystring, pure_dispatch=False): """Concretize dispatch_abstract. """ diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index c6fd02088..a71017212 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -50,35 +50,23 @@ def assert_body(harness, uripath, expected_body): # =================== def test_dispatcher_returns_a_result(harness): - request = harness.make_request('Greetings, program!', 'index.html') - result = dispatcher.dispatch(harness.client.website, request, [''], '/', '') + harness.fs.www.mk(('index.html', 'Greetings, program!'),) + result = dispatcher.dispatch(harness.client.website, [''], '/', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == os.path.join(harness.fs.www.root, 'index.html') assert result.wildcards == {} assert result.detail == 'Found.' def test_dispatcher_returns_a_result_for_favicon(harness): - request = Request.from_wsgi({ b'REQUEST_METHOD': b'GET' - , b'PATH_INFO': b'/favicon.ico' - , b'SERVER_PROTOCOL': b'HTTP/1.1' - , b'wsgi.input': StringIO() - }) - result = dispatcher.dispatch( harness.client.website - , request - , ['favicon.ico'] - , '/favicon.ico' - , '' - ) + result = dispatcher.dispatch(harness.client.website, ['favicon.ico'], '/favicon.ico', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('favicon.ico') assert result.wildcards == {} assert result.detail == 'Found.' def test_dispatcher_returns_a_result_for_autoindex(harness): - request = harness.make_request('Greetings, program!', 'index.html') - os.remove(request.fs) harness.client.website.list_directories = True - result = dispatcher.dispatch(harness.client.website, request, [''], '/', '') + result = dispatcher.dispatch(harness.client.website, [''], '/', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('autoindex.html.spt') assert result.wildcards == {} From 6e0d7f3017cdc4aa12238b17119cb77652b0850b Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Oct 2014 09:49:24 -0400 Subject: [PATCH 09/14] Get test suite passing again --- aspen/algorithms/website.py | 4 +- aspen/dispatcher.py | 5 +- aspen/www/autoindex.html.spt | 3 +- tests/test_negotiated_resource.py | 78 +++++++++++++++++-------------- 4 files changed, 49 insertions(+), 41 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index b1dde4a13..ccf1b82e4 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -41,6 +41,7 @@ from aspen.http.response import Response from aspen import typecasting from first import first as _first +from aspen.dispatcher import DispatchResult, DispatchStatus def parse_environ_into_request(environ): @@ -119,8 +120,9 @@ def delegate_error_to_simplate(website, request, response, resource=None): # Try to return an error that matches the type of the original resource. request.headers['Accept'] = resource.media_type + ', text/plain; q=0.1' resource = resources.get(website, request) + dispatch_result = DispatchResult(DispatchStatus.okay, fs, {}, 'Found.', {}) try: - response = resource.respond(request, response) + response = resource.respond(request, dispatch_result, response) except Response as response: if response.code != 406: raise diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index d19da1d40..38da43aab 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -344,10 +344,9 @@ def dispatch(website, pathparts, uripath, querystring, pure_dispatch=False): if result.match.endswith('/'): # autoindex if not website.list_directories: raise Response(404) - autoindex = website.ours_or_theirs('autoindex.html.spt') - assert autoindex is not None # sanity check - result.match = autoindex result.extra['autoindexdir'] = result.match + result.match = website.ours_or_theirs('autoindex.html.spt') + assert result.match is not None # sanity check return result # return so we skip the no-escape check elif result.status == DispatchStatus.non_leaf: # trailing-slash redirect diff --git a/aspen/www/autoindex.html.spt b/aspen/www/autoindex.html.spt index 4c8d7c254..2e592d996 100644 --- a/aspen/www/autoindex.html.spt +++ b/aspen/www/autoindex.html.spt @@ -47,13 +47,14 @@ def _get_time(stats): """ return str(datetime.fromtimestamp(stats[stat.ST_MTIME])) +[----------------------------------------] # Get the directory to list. # ========================== # Support the case where we are in the directory where this file actually # lives! -fspath = dispatch_result.get('autoindexdir', os.path.dirname(__file__)) +fspath = dispatch_result.extra.get('autoindexdir', os.path.dirname(__file__)) assert os.path.isdir(fspath) # sanity check urlpath = fspath[len(website.www_root):] + os.sep diff --git a/tests/test_negotiated_resource.py b/tests/test_negotiated_resource.py index 88c9e742e..7f356f314 100644 --- a/tests/test_negotiated_resource.py +++ b/tests/test_negotiated_resource.py @@ -110,11 +110,17 @@ def test_get_renderer_factory_can_raise_syntax_error(get): # get_response -def get_response(website, request, response): - context = { 'request': request +def get_state(harness, *a, **kw): + kw['return_after'] = 'dispatch_request_to_filesystem' + kw['want'] = 'state' + return harness.simple(*a, **kw) + +def get_response(state, response): + context = { 'request': state['request'] + , 'dispatch_result': state['dispatch_result'] , 'response': response } - resource = resources.load(website, request, 0) + resource = resources.load(state['website'], state['request'], 0) return resource.get_response(context) NEGOTIATED_RESOURCE = """\ @@ -128,76 +134,76 @@ def get_response(website, request, response): def test_get_response_gets_response(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) response = Response() - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - actual = get_response(harness.client.website, request, response) + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + actual = get_response(state, response) assert actual is response def test_get_response_is_happy_not_to_negotiate(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - actual = get_response(harness.client.website, request, Response()).body + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + actual = get_response(state, Response()).body assert actual == "Greetings, program!\n" def test_get_response_sets_content_type_when_it_doesnt_negotiate(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - actual = get_response(harness.client.website, request, Response()).headers['Content-Type'] + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + actual = get_response(state, Response()).headers['Content-Type'] assert actual == "text/plain; charset=UTF-8" def test_get_response_doesnt_reset_content_type_when_not_negotiating(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) response = Response() response.headers['Content-Type'] = 'never/mind' - actual = get_response(harness.client.website, request, response).headers['Content-Type'] + actual = get_response(state, response).headers['Content-Type'] assert actual == "never/mind" def test_get_response_negotiates(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - request.headers['Accept'] = 'text/html' - actual = get_response(harness.client.website, request, Response()).body + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + state['request'].headers['Accept'] = 'text/html' + actual = get_response(state, Response()).body assert actual == "

Greetings, program!

\n" def test_handles_busted_accept(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) # Set an invalid Accept header so it will return default (text/plain) - request.headers['Accept'] = 'text/html;' - actual = get_response(harness.client.website, request, Response()).body + state['request'].headers['Accept'] = 'text/html;' + actual = get_response(state, Response()).body assert actual == "Greetings, program!\n" def test_get_response_sets_content_type_when_it_negotiates(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - request.headers['Accept'] = 'text/html' - actual = get_response(harness.client.website, request, Response()).headers['Content-Type'] + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + state['request'].headers['Accept'] = 'text/html' + actual = get_response(state, Response()).headers['Content-Type'] assert actual == "text/html; charset=UTF-8" def test_get_response_doesnt_reset_content_type_when_negotiating(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - request.headers['Accept'] = 'text/html' + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + state['request'].headers['Accept'] = 'text/html' response = Response() response.headers['Content-Type'] = 'never/mind' - actual = get_response(harness.client.website, request, response).headers['Content-Type'] + actual = get_response(state, response).headers['Content-Type'] response = Response() response.headers['Content-Type'] = 'never/mind' - actual = get_response(harness.client.website, request, response).headers['Content-Type'] + actual = get_response(state, response).headers['Content-Type'] assert actual == "never/mind" def test_get_response_raises_406_if_need_be(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - request.headers['Accept'] = 'cheese/head' - actual = raises(Response, get_response, harness.client.website, request, Response()).value.code + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + state['request'].headers['Accept'] = 'cheese/head' + actual = raises(Response, get_response, state, Response()).value.code assert actual == 406 def test_get_response_406_gives_list_of_acceptable_types(harness): harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - request.headers['Accept'] = 'cheese/head' - actual = raises(Response, get_response, harness.client.website, request, Response()).value.body + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + state['request'].headers['Accept'] = 'cheese/head' + actual = raises(Response, get_response, state, Response()).value.body expected = "The following media types are available: text/plain, text/html." assert actual == expected @@ -221,16 +227,16 @@ class GlubberFactory(Factory): def test_can_override_default_renderers_by_mimetype(harness): harness.fs.project.mk(('configure-aspen.py', OVERRIDE_SIMPLATE),) harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE),) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - request.headers['Accept'] = 'text/plain' - actual = get_response(harness.client.website, request, Response()).body + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + state['request'].headers['Accept'] = 'text/plain' + actual = get_response(state, Response()).body assert actual == "glubber" def test_can_override_default_renderer_entirely(harness): harness.fs.project.mk(('configure-aspen.py', OVERRIDE_SIMPLATE)) - request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE) - request.headers['Accept'] = 'text/plain' - actual = get_response(harness.client.website, request, Response()).body + state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE) + state['request'].headers['Accept'] = 'text/plain' + actual = get_response(state, Response()).body assert actual == "glubber" From 52bac47b74d68085c9e9f7fc6f9e5dfdb35b9e73 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Oct 2014 09:53:52 -0400 Subject: [PATCH 10/14] Remove dependency on website.media_type_default --- aspen/algorithms/website.py | 1 + aspen/dispatcher.py | 8 ++++---- tests/test_dispatcher.py | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index ccf1b82e4..55d8ad350 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -63,6 +63,7 @@ def raise_200_for_OPTIONS(request): def dispatch_request_to_filesystem(website, request): result = dispatcher.dispatch( website + , media_type_default=website.media_type_default , pathparts=request.line.uri.path.parts , uripath=request.line.uri.path.raw , querystring=request.line.uri.querystring.raw diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index 38da43aab..75183fd85 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -256,15 +256,15 @@ def is_first_index(indices, basedir, name): return False -def update_neg_type(website, capture_accept, filename): +def update_neg_type(media_type_default, capture_accept, filename): media_type = mimetypes.guess_type(filename, strict=False)[0] if media_type is None: - media_type = website.media_type_default + media_type = media_type_default capture_accept['accept'] = media_type debug(lambda: "set result.extra['accept'] to %r" % media_type) -def dispatch(website, pathparts, uripath, querystring, pure_dispatch=False): +def dispatch(website, media_type_default, pathparts, uripath, querystring, pure_dispatch=False): """Concretize dispatch_abstract. """ @@ -276,7 +276,7 @@ def dispatch(website, pathparts, uripath, querystring, pure_dispatch=False): is_leaf = os.path.isfile traverse = os.path.join find_index = lambda x: match_index(website.indices, x) - noext_matched = lambda x: update_neg_type(website, capture_accept, x) + noext_matched = lambda x: update_neg_type(media_type_default, capture_accept, x) startdir = website.www_root diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index a71017212..ab530c953 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -51,14 +51,14 @@ def assert_body(harness, uripath, expected_body): def test_dispatcher_returns_a_result(harness): harness.fs.www.mk(('index.html', 'Greetings, program!'),) - result = dispatcher.dispatch(harness.client.website, [''], '/', '') + result = dispatcher.dispatch(harness.client.website, '', [''], '/', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == os.path.join(harness.fs.www.root, 'index.html') assert result.wildcards == {} assert result.detail == 'Found.' def test_dispatcher_returns_a_result_for_favicon(harness): - result = dispatcher.dispatch(harness.client.website, ['favicon.ico'], '/favicon.ico', '') + result = dispatcher.dispatch(harness.client.website, '', ['favicon.ico'], '/favicon.ico', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('favicon.ico') assert result.wildcards == {} @@ -66,7 +66,7 @@ def test_dispatcher_returns_a_result_for_favicon(harness): def test_dispatcher_returns_a_result_for_autoindex(harness): harness.client.website.list_directories = True - result = dispatcher.dispatch(harness.client.website, [''], '/', '') + result = dispatcher.dispatch(harness.client.website, '', [''], '/', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('autoindex.html.spt') assert result.wildcards == {} From d2c007cf655dce8fe1c86af233867d8a6f567bcc Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Oct 2014 09:56:41 -0400 Subject: [PATCH 11/14] Remove dependency on website.indices --- aspen/algorithms/website.py | 1 + aspen/dispatcher.py | 11 ++++++----- tests/test_dispatcher.py | 7 ++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index 55d8ad350..76709dff3 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -63,6 +63,7 @@ def raise_200_for_OPTIONS(request): def dispatch_request_to_filesystem(website, request): result = dispatcher.dispatch( website + , indices=website.indices , media_type_default=website.media_type_default , pathparts=request.line.uri.path.parts , uripath=request.line.uri.path.raw diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index 75183fd85..249460987 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -264,7 +264,8 @@ def update_neg_type(media_type_default, capture_accept, filename): debug(lambda: "set result.extra['accept'] to %r" % media_type) -def dispatch(website, media_type_default, pathparts, uripath, querystring, pure_dispatch=False): +def dispatch(website, indices, media_type_default, pathparts, uripath, querystring, + pure_dispatch=False): """Concretize dispatch_abstract. """ @@ -275,7 +276,7 @@ def dispatch(website, media_type_default, pathparts, uripath, querystring, pure_ listnodes = os.listdir is_leaf = os.path.isfile traverse = os.path.join - find_index = lambda x: match_index(website.indices, x) + find_index = lambda x: match_index(indices, x) noext_matched = lambda x: update_neg_type(media_type_default, capture_accept, x) startdir = website.www_root @@ -300,11 +301,11 @@ def dispatch(website, media_type_default, pathparts, uripath, querystring, pure_ if result.match: debug(lambda: "result.match is true" ) matchbase, matchname = result.match.rsplit(os.path.sep,1) - if pathparts[-1] != '' and matchname in website.indices and \ - is_first_index(website.indices, matchbase, matchname): + if pathparts[-1] != '' and matchname in indices and \ + is_first_index(indices, matchbase, matchname): # asked for something that maps to a default index file; redirect to / per issue #175 debug( lambda: "found default index '%s' maps into %r" - % (pathparts[-1], website.indices) + % (pathparts[-1], indices) ) location = uripath[:-len(pathparts[-1])] if querystring: diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index ab530c953..af817acf2 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -51,14 +51,15 @@ def assert_body(harness, uripath, expected_body): def test_dispatcher_returns_a_result(harness): harness.fs.www.mk(('index.html', 'Greetings, program!'),) - result = dispatcher.dispatch(harness.client.website, '', [''], '/', '') + result = dispatcher.dispatch(harness.client.website, ['index.html'], '', [''], '/', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == os.path.join(harness.fs.www.root, 'index.html') assert result.wildcards == {} assert result.detail == 'Found.' def test_dispatcher_returns_a_result_for_favicon(harness): - result = dispatcher.dispatch(harness.client.website, '', ['favicon.ico'], '/favicon.ico', '') + website = harness.client.website + result = dispatcher.dispatch(website, [], '', ['favicon.ico'], '/favicon.ico', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('favicon.ico') assert result.wildcards == {} @@ -66,7 +67,7 @@ def test_dispatcher_returns_a_result_for_favicon(harness): def test_dispatcher_returns_a_result_for_autoindex(harness): harness.client.website.list_directories = True - result = dispatcher.dispatch(harness.client.website, '', [''], '/', '') + result = dispatcher.dispatch(harness.client.website, [], '', [''], '/', '') assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('autoindex.html.spt') assert result.wildcards == {} From 5e6ac6d288ffe080096635713d857d0c4e28a0b7 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Oct 2014 10:01:05 -0400 Subject: [PATCH 12/14] Remove dependency on website.www_root --- aspen/algorithms/website.py | 1 + aspen/dispatcher.py | 3 +-- tests/test_dispatcher.py | 28 ++++++++++++++++++++++++---- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index 76709dff3..d90a37e9b 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -68,6 +68,7 @@ def dispatch_request_to_filesystem(website, request): , pathparts=request.line.uri.path.parts , uripath=request.line.uri.path.raw , querystring=request.line.uri.querystring.raw + , startdir=website.www_root ) request.fs = result.match for k, v in result.wildcards.iteritems(): diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index 249460987..a6e3c6061 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -265,7 +265,7 @@ def update_neg_type(media_type_default, capture_accept, filename): def dispatch(website, indices, media_type_default, pathparts, uripath, querystring, - pure_dispatch=False): + startdir, pure_dispatch=False): """Concretize dispatch_abstract. """ @@ -278,7 +278,6 @@ def dispatch(website, indices, media_type_default, pathparts, uripath, querystri traverse = os.path.join find_index = lambda x: match_index(indices, x) noext_matched = lambda x: update_neg_type(media_type_default, capture_accept, x) - startdir = website.www_root # Dispatch! diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index af817acf2..3e1d9ad17 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -51,15 +51,28 @@ def assert_body(harness, uripath, expected_body): def test_dispatcher_returns_a_result(harness): harness.fs.www.mk(('index.html', 'Greetings, program!'),) - result = dispatcher.dispatch(harness.client.website, ['index.html'], '', [''], '/', '') + result = dispatcher.dispatch( harness.client.website + , ['index.html'] + , '' + , [''] + , '/' + , '' + , harness.fs.www.root + ) assert result.status == dispatcher.DispatchStatus.okay assert result.match == os.path.join(harness.fs.www.root, 'index.html') assert result.wildcards == {} assert result.detail == 'Found.' def test_dispatcher_returns_a_result_for_favicon(harness): - website = harness.client.website - result = dispatcher.dispatch(website, [], '', ['favicon.ico'], '/favicon.ico', '') + result = dispatcher.dispatch( harness.client.website + , [] + , '' + , ['favicon.ico'] + , '/favicon.ico' + , '' + , harness.fs.www.root + ) assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('favicon.ico') assert result.wildcards == {} @@ -67,7 +80,14 @@ def test_dispatcher_returns_a_result_for_favicon(harness): def test_dispatcher_returns_a_result_for_autoindex(harness): harness.client.website.list_directories = True - result = dispatcher.dispatch(harness.client.website, [], '', [''], '/', '') + result = dispatcher.dispatch( harness.client.website + , [] + , '' + , [''] + , '/' + , '' + , harness.fs.www.root + ) assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('autoindex.html.spt') assert result.wildcards == {} From 78c72c5df2918808530b202fb1aec8bc2d67efd9 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Oct 2014 10:17:05 -0400 Subject: [PATCH 13/14] Remove website.list_directories & .ours_or_theirs --- aspen/algorithms/website.py | 23 +++++++++++++++++------ aspen/dispatcher.py | 9 ++------- tests/test_dispatcher.py | 13 ++++++++++--- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index d90a37e9b..3d35a6b4b 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -62,19 +62,30 @@ def raise_200_for_OPTIONS(request): def dispatch_request_to_filesystem(website, request): + + def handle_directory(result): + if not website.list_directories: + raise Response(404) + result.extra['autoindexdir'] = result.match + result.match = website.ours_or_theirs('autoindex.html.spt') + assert result.match is not None # sanity check + return result + result = dispatcher.dispatch( website - , indices=website.indices - , media_type_default=website.media_type_default - , pathparts=request.line.uri.path.parts - , uripath=request.line.uri.path.raw - , querystring=request.line.uri.querystring.raw - , startdir=website.www_root + , indices = website.indices + , media_type_default = website.media_type_default + , pathparts = request.line.uri.path.parts + , uripath = request.line.uri.path.raw + , querystring = request.line.uri.querystring.raw + , startdir = website.www_root + , handle_directory = handle_directory ) request.fs = result.match for k, v in result.wildcards.iteritems(): request.line.uri.path[k] = v return {'dispatch_result': result} + def apply_typecasters_to_path(website, request): typecasting.apply_typecasters(website.typecasters, request.line.uri.path) diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index a6e3c6061..656969578 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -265,7 +265,7 @@ def update_neg_type(media_type_default, capture_accept, filename): def dispatch(website, indices, media_type_default, pathparts, uripath, querystring, - startdir, pure_dispatch=False): + startdir, handle_directory, pure_dispatch=False): """Concretize dispatch_abstract. """ @@ -342,12 +342,7 @@ def dispatch(website, indices, media_type_default, pathparts, uripath, querystri if result.status == DispatchStatus.okay: if result.match.endswith('/'): # autoindex - if not website.list_directories: - raise Response(404) - result.extra['autoindexdir'] = result.match - result.match = website.ours_or_theirs('autoindex.html.spt') - assert result.match is not None # sanity check - return result # return so we skip the no-escape check + return handle_directory(result) # return so we skip the no-escape check elif result.status == DispatchStatus.non_leaf: # trailing-slash redirect location = uripath + '/' diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index 3e1d9ad17..e36b29a14 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -5,11 +5,9 @@ import os from pytest import raises -from StringIO import StringIO import aspen from aspen import dispatcher, Response -from aspen.http.request import Request # Helpers @@ -58,6 +56,7 @@ def test_dispatcher_returns_a_result(harness): , '/' , '' , harness.fs.www.root + , lambda result: result ) assert result.status == dispatcher.DispatchStatus.okay assert result.match == os.path.join(harness.fs.www.root, 'index.html') @@ -72,6 +71,7 @@ def test_dispatcher_returns_a_result_for_favicon(harness): , '/favicon.ico' , '' , harness.fs.www.root + , lambda result: result ) assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('favicon.ico') @@ -80,14 +80,21 @@ def test_dispatcher_returns_a_result_for_favicon(harness): def test_dispatcher_returns_a_result_for_autoindex(harness): harness.client.website.list_directories = True - result = dispatcher.dispatch( harness.client.website + tracer = object() + actual = dispatcher.dispatch( harness.client.website , [] , '' , [''] , '/' , '' , harness.fs.www.root + , lambda result: tracer ) + assert actual is tracer + +def test_dispatcher_in_algorithm_returns_a_better_result_for_autoindex(harness): + harness.client.website.list_directories = True + result = harness.simple(filepath=None, uripath='/', want='dispatch_result') assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('autoindex.html.spt') assert result.wildcards == {} From bebf1586921801bb0c3549efa02f6bc613c88d33 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Oct 2014 12:23:17 -0400 Subject: [PATCH 14/14] Remove website entirely from dispatcher --- aspen/algorithms/website.py | 20 ++++++------ aspen/dispatcher.py | 63 ++++++++++++++++++------------------- tests/test_dispatcher.py | 60 +++++++++++++++++++---------------- 3 files changed, 72 insertions(+), 71 deletions(-) diff --git a/aspen/algorithms/website.py b/aspen/algorithms/website.py index 3d35a6b4b..bc5dc827c 100644 --- a/aspen/algorithms/website.py +++ b/aspen/algorithms/website.py @@ -63,22 +63,20 @@ def raise_200_for_OPTIONS(request): def dispatch_request_to_filesystem(website, request): - def handle_directory(result): - if not website.list_directories: - raise Response(404) - result.extra['autoindexdir'] = result.match - result.match = website.ours_or_theirs('autoindex.html.spt') - assert result.match is not None # sanity check - return result - - result = dispatcher.dispatch( website - , indices = website.indices + if website.list_directories: + directory_default = website.ours_or_theirs('autoindex.html.spt') + assert directory_default is not None # sanity check + else: + directory_default = None + + result = dispatcher.dispatch( indices = website.indices , media_type_default = website.media_type_default , pathparts = request.line.uri.path.parts , uripath = request.line.uri.path.raw , querystring = request.line.uri.querystring.raw , startdir = website.www_root - , handle_directory = handle_directory + , directory_default = directory_default + , favicon_default = website.find_ours('favicon.ico') ) request.fs = result.match for k, v in result.wildcards.iteritems(): diff --git a/aspen/dispatcher.py b/aspen/dispatcher.py index 656969578..2caaf889c 100644 --- a/aspen/dispatcher.py +++ b/aspen/dispatcher.py @@ -61,6 +61,7 @@ def __init__(self, status, match, wildcards, detail, extra): self.wildcards = wildcards self.detail = detail self.extra = extra + self.constrain_path = True def dispatch_abstract(listnodes, is_leaf, traverse, find_index, noext_matched, @@ -264,8 +265,8 @@ def update_neg_type(media_type_default, capture_accept, filename): debug(lambda: "set result.extra['accept'] to %r" % media_type) -def dispatch(website, indices, media_type_default, pathparts, uripath, querystring, - startdir, handle_directory, pure_dispatch=False): +def dispatch(indices, media_type_default, pathparts, uripath, querystring, startdir, + directory_default, favicon_default): """Concretize dispatch_abstract. """ @@ -311,47 +312,43 @@ def dispatch(website, indices, media_type_default, pathparts, uripath, querystri location += '?' + querystring raise Response(302, headers={'Location': location}) - if not pure_dispatch: - - # favicon.ico - # =========== - # Serve Aspen's favicon if there's not one. - - if uripath == '/favicon.ico': - if result.status != DispatchStatus.okay: - result.status = DispatchStatus.okay - result.match = website.find_ours('favicon.ico') - result.wildcards = {} - result.detail = 'Found.' - return result - - - # robots.txt - # ========== - # Don't let robots.txt be handled by anything other than an actual - # robots.txt file - - if uripath == '/robots.txt': - if result.status != DispatchStatus.missing: - if not result.match.endswith('robots.txt'): - raise Response(404) - # Handle returned states. # ======================= + if result.status != DispatchStatus.missing: + if uripath == '/robots.txt' and not result.match.endswith('robots.txt'): # robots.txt + # Don't let robots.txt be handled by anything other than an actual robots.txt file, + # because if you don't have a robots.txt but you do have a wildcard, then you end + # up with logspam. + raise Response(404) + if result.status == DispatchStatus.okay: - if result.match.endswith('/'): # autoindex - return handle_directory(result) # return so we skip the no-escape check + if result.match.endswith('/'): + if directory_default: # autoindex + result.extra['autoindexdir'] = result.match # order matters! + result.match = directory_default + result.wildcards = {} + result.detail = 'Directory default.' + result.constrain_path = False + else: + raise Response(404) - elif result.status == DispatchStatus.non_leaf: # trailing-slash redirect + elif result.status == DispatchStatus.non_leaf: # trailing slash location = uripath + '/' if querystring: location += '?' + querystring raise Response(302, headers={'Location': location}) - elif result.status == DispatchStatus.missing: # 404 - raise Response(404) + elif result.status == DispatchStatus.missing: # 404, but ... + if uripath == '/favicon.ico' and favicon_default: # favicon.ico + result.status = DispatchStatus.okay + result.match = favicon_default + result.wildcards = {} + result.detail = 'Favicon default.' + result.constrain_path = False + else: + raise Response(404) else: raise Response(500, "Unknown result status.") @@ -360,7 +357,7 @@ def dispatch(website, indices, media_type_default, pathparts, uripath, querystri # Protect against escaping the www_root. # ====================================== - if not result.match.startswith(startdir): + if result.constrain_path and not result.match.startswith(startdir): raise Response(404) return result diff --git a/tests/test_dispatcher.py b/tests/test_dispatcher.py index e36b29a14..61f400ee4 100644 --- a/tests/test_dispatcher.py +++ b/tests/test_dispatcher.py @@ -49,14 +49,14 @@ def assert_body(harness, uripath, expected_body): def test_dispatcher_returns_a_result(harness): harness.fs.www.mk(('index.html', 'Greetings, program!'),) - result = dispatcher.dispatch( harness.client.website - , ['index.html'] - , '' - , [''] - , '/' - , '' - , harness.fs.www.root - , lambda result: result + result = dispatcher.dispatch( indices = ['index.html'] + , media_type_default = '' + , pathparts = [''] + , uripath = '/' + , querystring = '' + , startdir = harness.fs.www.root + , directory_default = '' + , favicon_default = '' ) assert result.status == dispatcher.DispatchStatus.okay assert result.match == os.path.join(harness.fs.www.root, 'index.html') @@ -64,33 +64,39 @@ def test_dispatcher_returns_a_result(harness): assert result.detail == 'Found.' def test_dispatcher_returns_a_result_for_favicon(harness): - result = dispatcher.dispatch( harness.client.website - , [] - , '' - , ['favicon.ico'] - , '/favicon.ico' - , '' - , harness.fs.www.root - , lambda result: result + tracer = object() + result = dispatcher.dispatch( indices = [] + , media_type_default = '' + , pathparts = ['favicon.ico'] + , uripath = '/favicon.ico' + , querystring = '' + , startdir = harness.fs.www.root + , directory_default = '' + , favicon_default = tracer ) + assert result.match is tracer + +def test_dispatcher_in_algorithm_returns_a_better_result_for_favicon(harness): + harness.client.website.list_directories = True + result = harness.simple(filepath=None, uripath='/favicon.ico', want='dispatch_result') assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('favicon.ico') assert result.wildcards == {} - assert result.detail == 'Found.' + assert result.detail == 'Favicon default.' def test_dispatcher_returns_a_result_for_autoindex(harness): harness.client.website.list_directories = True tracer = object() - actual = dispatcher.dispatch( harness.client.website - , [] - , '' - , [''] - , '/' - , '' - , harness.fs.www.root - , lambda result: tracer + result = dispatcher.dispatch( indices = [] + , media_type_default = '' + , pathparts = [''] + , uripath = '/' + , querystring = '' + , startdir = harness.fs.www.root + , directory_default = tracer + , favicon_default = '' ) - assert actual is tracer + assert result.match is tracer def test_dispatcher_in_algorithm_returns_a_better_result_for_autoindex(harness): harness.client.website.list_directories = True @@ -98,7 +104,7 @@ def test_dispatcher_in_algorithm_returns_a_better_result_for_autoindex(harness): assert result.status == dispatcher.DispatchStatus.okay assert result.match == harness.client.website.find_ours('autoindex.html.spt') assert result.wildcards == {} - assert result.detail == 'Found.' + assert result.detail == 'Directory default.' # Indices