From fa08731386513e8aff76643414e0277c918495c6 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Sun, 19 Jul 2020 06:29:39 -0400 Subject: [PATCH 01/21] Update setup.py for Python 3 --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 191deb19..2b6e38b9 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,10 @@ import os.path import subprocess -import urllib +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve from setuptools import find_packages, setup, Command from setuptools.command.build_py import build_py @@ -47,7 +50,7 @@ def run(self): print('skipping downloading {}, already exists'.format(destination_path)) else: print('downloading {} to {}'.format(source_url, destination_path)) - urllib.urlretrieve(source_url, destination_path) + urlretrieve(source_url, destination_path) class InitGitSubModules(Command): From 5d8c982c39bd67fe78a0c636a0265d771b22be70 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Sun, 19 Jul 2020 06:31:30 -0400 Subject: [PATCH 02/21] Update testutil.py for Python 3 --- shinysdr/testutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shinysdr/testutil.py b/shinysdr/testutil.py index 96e40b54..9ab50b91 100644 --- a/shinysdr/testutil.py +++ b/shinysdr/testutil.py @@ -416,7 +416,7 @@ def http_get(reactor, url, accept=None): def http_head(reactor, url, accept=None): return (http_request(reactor, url, method='HEAD', accept=accept) - .addCallback(lambda (response, data): response)) + .addCallback(lambda result: result[0])) def http_post_json(reactor, url, value): From f16517b629073059aa8f69443a4c74e2fca39210 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Sun, 19 Jul 2020 06:48:50 -0400 Subject: [PATCH 03/21] More Python 3 support --- shinysdr/i/shared_test_objects.py | 4 +++- shinysdr/i/top.py | 2 +- shinysdr/values.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/shinysdr/i/shared_test_objects.py b/shinysdr/i/shared_test_objects.py index 7b7dfb5f..d9c07959 100644 --- a/shinysdr/i/shared_test_objects.py +++ b/shinysdr/i/shared_test_objects.py @@ -20,6 +20,8 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import six + from shinysdr.values import ExportedState, exported_value @@ -27,6 +29,6 @@ class SharedTestObjects(ExportedState): - @exported_value(type=unicode, changes='never') + @exported_value(type=six.text_type, changes='never') def get_smoke_test(self): return 'SharedTestObjects exists' diff --git a/shinysdr/i/top.py b/shinysdr/i/top.py index b64641cd..a0ceaa34 100644 --- a/shinysdr/i/top.py +++ b/shinysdr/i/top.py @@ -418,7 +418,7 @@ def _trigger_reconnect(self, reason): self._do_connect() def _recursive_lock_hook(self): - for source in self._sources.itervalues(): + for source in six.itervalues(self._sources): source.notify_reconnecting_or_restarting() diff --git a/shinysdr/values.py b/shinysdr/values.py index 003afe3c..e7257318 100644 --- a/shinysdr/values.py +++ b/shinysdr/values.py @@ -910,7 +910,7 @@ def __delitem__(self, key): self._shape_subscription() def __iter__(self): - return self.iterkeys() + return six.iterkeys(self) if six.PY2: def iterkeys(self): From c7a0606349cb57811f5852fd4aaae41321b37cc2 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 02:33:42 -0400 Subject: [PATCH 04/21] Work around Twisted Python 3.8 compatibility issue --- shinysdr/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shinysdr/main.py b/shinysdr/main.py index c5cb9e43..a99dede6 100755 --- a/shinysdr/main.py +++ b/shinysdr/main.py @@ -31,7 +31,7 @@ from twisted.internet import defer from twisted.internet import reactor as singleton_reactor from twisted.internet.task import react -from twisted.logger import Logger, STDLibLogObserver, globalLogBeginner +from twisted.logger import Logger, STDLibLogObserver, globalLogBeginner, textFileLogObserver # Note that gnuradio-dependent modules are loaded later, to avoid the startup time if all we're going to do is give a usage message from shinysdr.i.config import Config, ConfigException, write_default_config, execute_config, print_config_exception @@ -180,7 +180,13 @@ def _check_versions(): def configure_logging(): logging.basicConfig(level=logging.INFO) - globalLogBeginner.beginLoggingTo([STDLibLogObserver(name='shinysdr')]) + observer = STDLibLogObserver(name='shinysdr') + if sys.version_info > (3, 8): + # STDLibLogObserver is incompatible with Python 3.8 + observer = textFileLogObserver(sys.stderr) + globalLogBeginner.beginLoggingTo( + [observer], + redirectStandardIO=False) if __name__ == '__main__': From 12674778df4b068c3371371b9f4dd0d573763d64 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 03:10:26 -0400 Subject: [PATCH 05/21] Call putChild with byte strings for Python 3 compatibility --- shinysdr/i/db.py | 8 ++++---- shinysdr/i/network/export_http.py | 2 +- shinysdr/i/network/session_http.py | 14 +++++++------- shinysdr/i/network/test_audio_http.py | 4 ++-- shinysdr/i/network/webapp.py | 26 +++++++++++++------------- shinysdr/plugins/aprs/__init__.py | 4 ++-- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/shinysdr/i/db.py b/shinysdr/i/db.py index 307fdaeb..2b8b03be 100644 --- a/shinysdr/i/db.py +++ b/shinysdr/i/db.py @@ -140,10 +140,10 @@ class DatabasesResource(resource.Resource): def __init__(self, databases): resource.Resource.__init__(self) - self.putChild('', ElementRenderingResource(_DbsIndexListElement(self))) + self.putChild(b'', ElementRenderingResource(_DbsIndexListElement(self))) self.names = [] for (name, database) in six.iteritems(databases): - self.putChild(name, DatabaseResource(database)) + self.putChild(name.encode(), DatabaseResource(database)) self.names.append(name) self.names.sort() # TODO reconsider case/locale @@ -170,9 +170,9 @@ def __init__(self, database): resource.Resource.__init__(self) def instantiate(rkey): - self.putChild(str(rkey), _RecordResource(database, database.records[rkey])) + self.putChild(str(rkey).encode(), _RecordResource(database, database.records[rkey])) - self.putChild('', _DbIndexResource(database, instantiate)) + self.putChild(b'', _DbIndexResource(database, instantiate)) for rkey in database.records: instantiate(rkey) diff --git a/shinysdr/i/network/export_http.py b/shinysdr/i/network/export_http.py index 12386acd..59fa31de 100644 --- a/shinysdr/i/network/export_http.py +++ b/shinysdr/i/network/export_http.py @@ -71,7 +71,7 @@ def __init__(self, block, wcommon, deleteSelf): if cell.type().is_reference(): self._blockCells[key] = cell else: - self.putChild(key, ValueCellResource(cell, self.__wcommon)) + self.putChild(key.encode(), ValueCellResource(cell, self.__wcommon)) self.__element = _BlockHtmlElement(wcommon) def getChild(self, path, request): diff --git a/shinysdr/i/network/session_http.py b/shinysdr/i/network/session_http.py index b3bf13a4..a0eb91a0 100644 --- a/shinysdr/i/network/session_http.py +++ b/shinysdr/i/network/session_http.py @@ -38,23 +38,23 @@ def __init__(self, session, wcommon, read_only_dbs, writable_db): SlashedResource.__init__(self) # UI entry point - self.putChild('', ElementRenderingResource(_RadioIndexHtmlElement(wcommon))) + self.putChild(b'', ElementRenderingResource(_RadioIndexHtmlElement(wcommon))) # Exported radio control objects - self.putChild(CAP_OBJECT_PATH_ELEMENT, BlockResource(session, wcommon, _not_deletable)) + self.putChild(CAP_OBJECT_PATH_ELEMENT.encode(), BlockResource(session, wcommon, _not_deletable)) # Frequency DB - self.putChild('dbs', shinysdr.i.db.DatabasesResource(read_only_dbs)) - self.putChild('wdb', shinysdr.i.db.DatabaseResource(writable_db)) + self.putChild(b'dbs', shinysdr.i.db.DatabasesResource(read_only_dbs)) + self.putChild(b'wdb', shinysdr.i.db.DatabaseResource(writable_db)) # Debug graph - self.putChild('flow-graph', FlowgraphVizResource(wcommon.reactor, session.flowgraph_for_debug())) + self.putChild(b'flow-graph', FlowgraphVizResource(wcommon.reactor, session.flowgraph_for_debug())) # Ephemeris - self.putChild('ephemeris', EphemerisResource()) + self.putChild(b'ephemeris', EphemerisResource()) # Standard audio-file-over-HTTP audio stream (the ShinySDR web client uses WebSockets instead, but both have the same path modulo protocol) - self.putChild(AUDIO_STREAM_PATH_ELEMENT, AudioStreamResource(session)) + self.putChild(AUDIO_STREAM_PATH_ELEMENT.encode(), AudioStreamResource(session)) class _RadioIndexHtmlElement(EntryPointIndexElement): diff --git a/shinysdr/i/network/test_audio_http.py b/shinysdr/i/network/test_audio_http.py index e0d2ad66..9f398809 100644 --- a/shinysdr/i/network/test_audio_http.py +++ b/shinysdr/i/network/test_audio_http.py @@ -38,8 +38,8 @@ class TestAudioStreamResource(unittest.TestCase): def setUp(self): tree = Resource() - tree.putChild('mono', AudioStreamResource(_FakeSession(1))) - tree.putChild('stereo', AudioStreamResource(_FakeSession(2))) + tree.putChild(b'mono', AudioStreamResource(_FakeSession(1))) + tree.putChild(b'stereo', AudioStreamResource(_FakeSession(2))) self.port = the_reactor.listenTCP(0, SiteWithDefaultHeaders(tree), interface="127.0.0.1") # pylint: disable=no-member def tearDown(self): diff --git a/shinysdr/i/network/webapp.py b/shinysdr/i/network/webapp.py index 5801bfe9..d1853069 100644 --- a/shinysdr/i/network/webapp.py +++ b/shinysdr/i/network/webapp.py @@ -96,7 +96,7 @@ def resource_factory(entry_point): if UNIQUE_PUBLIC_CAP in cap_table: # TODO: consider factoring out "generate URL for cap" - server_root.putChild('', Redirect(_make_cap_url(UNIQUE_PUBLIC_CAP))) + server_root.putChild(b'', Redirect(_make_cap_url(UNIQUE_PUBLIC_CAP))) self.__ws_protocol = txws.WebSocketFactory( FactoryWithArgs.forProtocol(WebSocketDispatcherProtocol, cap_table, subscription_context)) @@ -157,27 +157,27 @@ def _put_root_static(wcommon, container_resource): """Place all the simple resources, that are not necessarily sourced from files but at least are unchanging and public.""" for name in ['', 'client', 'test', 'manual', 'tools']: - container_resource.putChild(name, _make_static_resource(os.path.join(static_resource_path, name if name != '' else 'index.html'))) + container_resource.putChild(name.encode(), _make_static_resource(os.path.join(static_resource_path, name if name != '' else 'index.html'))) # Link deps into /client/. - client = container_resource.children['client'] + client = container_resource.children[b'client'] for name in ['require.js', 'text.js']: - client.putChild(name, _make_static_resource(os.path.join(deps_path, name))) + client.putChild(name.encode(), _make_static_resource(os.path.join(deps_path, name))) for name in ['measviz.js', 'measviz.css']: - client.putChild(name, _make_static_resource(os.path.join(deps_path, 'measviz/src', name))) + client.putChild(name.encode(), _make_static_resource(os.path.join(deps_path, 'measviz/src', name))) # Link deps into /test/. - test = container_resource.children['test'] + test = container_resource.children[b'test'] jasmine = SlashedResource() - test.putChild('jasmine', jasmine) + test.putChild(b'jasmine', jasmine) for name in ['jasmine.css', 'jasmine.js', 'jasmine-html.js']: - jasmine.putChild(name, _make_static_resource(os.path.join( + jasmine.putChild(name.encode(), _make_static_resource(os.path.join( deps_path, 'jasmine/lib/jasmine-core/', name))) # Special resources - container_resource.putChild('favicon.ico', + container_resource.putChild(b'favicon.ico', _make_static_resource(os.path.join(static_resource_path, 'client/icon/icon-32.png'))) - client.putChild('web-app-manifest.json', + client.putChild(b'web-app-manifest.json', WebAppManifestResource(wcommon)) _put_plugin_resources(wcommon, client) @@ -188,10 +188,10 @@ def _put_plugin_resources(wcommon, client_resource): load_list_js = [] mode_table = {} plugin_resources = Resource() - client_resource.putChild('plugins', plugin_resources) + client_resource.putChild(b'plugins', plugin_resources) for resource_def in getPlugins(_IClientResourceDef, shinysdr.plugins): # Add the plugin's resource to static serving - plugin_resources.putChild(resource_def.key, resource_def.resource) + plugin_resources.putChild(resource_def.key.encode(), resource_def.resource) plugin_resource_url = '/client/plugins/' + urllib.parse.quote(resource_def.key, safe='') + '/' # Tell the client to load the plugins # TODO constrain path values to be relative (not on a different origin, to not leak urls) @@ -211,7 +211,7 @@ def _put_plugin_resources(wcommon, client_resource): 'js': load_list_js, 'modes': mode_table, } - client_resource.putChild('client-configuration.json', ClientConfigurationResource(wcommon, plugin_index)) + client_resource.putChild(b'client-configuration.json', ClientConfigurationResource(wcommon, plugin_index)) class ClientConfigurationResource(Resource): diff --git a/shinysdr/plugins/aprs/__init__.py b/shinysdr/plugins/aprs/__init__.py index 59477114..6095cf62 100644 --- a/shinysdr/plugins/aprs/__init__.py +++ b/shinysdr/plugins/aprs/__init__.py @@ -709,8 +709,8 @@ def _parse_telemetry_value(facts, errors, value_str, channel): _plugin_resource = static.File(os.path.join(os.path.split(__file__)[0], 'client')) -_plugin_resource.putChild('symbols', static.File(os.path.join(os.path.split(shinysdr.__file__)[0], 'deps/aprs-symbols/png'))) -_plugin_resource.putChild('symbol-index', static.File(os.path.join(os.path.split(shinysdr.__file__)[0], 'deps/aprs-symbol-index/generated/symbols.dense.json'))) +_plugin_resource.putChild(b'symbols', static.File(os.path.join(os.path.split(shinysdr.__file__)[0], 'deps/aprs-symbols/png'))) +_plugin_resource.putChild(b'symbol-index', static.File(os.path.join(os.path.split(shinysdr.__file__)[0], 'deps/aprs-symbol-index/generated/symbols.dense.json'))) plugin_client_js = ClientResourceDef( key=__name__, From 7b882d4a9f898c27c949a5f30b1f873055c03bfc Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 03:21:50 -0400 Subject: [PATCH 06/21] Fix make_websocket_url to use strings on Python 3 --- shinysdr/i/network/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shinysdr/i/network/base.py b/shinysdr/i/network/base.py index 4212e48c..ab070b45 100644 --- a/shinysdr/i/network/base.py +++ b/shinysdr/i/network/base.py @@ -139,8 +139,8 @@ def make_websocket_url(self, request, path): else: return endpoint_string_to_url( self.__ws_endpoint_string, - hostname=request.getRequestHostname(), - scheme=b'ws', + hostname=six.ensure_str(request.getRequestHostname()), + scheme='ws', path=path) From 9d771737de51917188ea4d67cb90b1a3f9744ef0 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 03:23:35 -0400 Subject: [PATCH 07/21] Filenames and MIME types are strings in Python 3 --- shinysdr/i/network/webapp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shinysdr/i/network/webapp.py b/shinysdr/i/network/webapp.py index d1853069..4bb9aa8a 100644 --- a/shinysdr/i/network/webapp.py +++ b/shinysdr/i/network/webapp.py @@ -53,10 +53,10 @@ def _make_static_resource(pathname): # str() because if we happen to pass unicode as the pathname then directory listings break (discovered with Twisted 16.4.1). r = static.File(str(pathname), - defaultType=b'text/plain', - ignoredExts=[b'.html']) - r.contentTypes[b'.csv'] = b'text/csv' - r.indexNames = [b'index.html'] + defaultType='text/plain', + ignoredExts=['.html']) + r.contentTypes['.csv'] = 'text/csv' + r.indexNames = ['index.html'] return r From 97e2ef02029aa1118c635ef7e2ffda62bd28d0ce Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 03:28:07 -0400 Subject: [PATCH 08/21] Return JSON as utf-8 --- shinysdr/i/db.py | 12 ++++++------ shinysdr/i/ephemeris.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/shinysdr/i/db.py b/shinysdr/i/db.py index 2b8b03be..dced2696 100644 --- a/shinysdr/i/db.py +++ b/shinysdr/i/db.py @@ -186,11 +186,11 @@ def __init__(self, db, instantiate): self.__instantiate = instantiate def render_GET(self, request): - request.setHeader(b'Content-Type', b'application/json') + request.setHeader(b'Content-Type', b'application/json; charset=utf-8') return json.dumps({ u'records': self.__database.records, u'writable': self.__database.writable - }) + }).encode('utf-8') def render_POST(self, request): desc = json.load(request.content) @@ -211,7 +211,7 @@ def render_POST(self, request): request.setResponseCode(http.CREATED) request.setHeader(b'Content-Type', b'text/plain') request.setHeader(b'Location', url) - return url + return url.encode() class _RecordResource(resource.Resource): @@ -223,11 +223,11 @@ def __init__(self, database, record): self.__record = record def render_GET(self, request): - request.setHeader(b'Content-Type', b'application/json') - return json.dumps(self.__record) + request.setHeader(b'Content-Type', b'application/json; charset=utf-8') + return json.dumps(self.__record).encode('utf-8') def render_POST(self, request): - assert request.getHeader(b'Content-Type') == b'application/json' + assert request.getHeader(b'Content-Type') == b'application/json; charset=utf-8' if not self.__database.writable: request.setResponseCode(http.FORBIDDEN) request.setHeader(b'Content-Type', b'text/plain') diff --git a/shinysdr/i/ephemeris.py b/shinysdr/i/ephemeris.py index 1f37ed4a..a12f53e3 100644 --- a/shinysdr/i/ephemeris.py +++ b/shinysdr/i/ephemeris.py @@ -53,8 +53,8 @@ def render_GET(self, request): y = math.cos(sun.az) * math.cos(sun.alt) z = -math.sin(sun.alt) - request.setHeader(b'Content-Type', b'application/json') - return json.dumps([x, y, z]) + request.setHeader(b'Content-Type', b'application/json; charset=utf-8') + return json.dumps([x, y, z]).encode('utf-8') __all__.append('EphemerisResource') From c9dd568bdda237dd9b1b35ea8932a37e300cebf6 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 03:51:16 -0400 Subject: [PATCH 09/21] Parse WebSocket path correctly on Python 3 --- shinysdr/i/network/export_ws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shinysdr/i/network/export_ws.py b/shinysdr/i/network/export_ws.py index f1956a06..63b16ffd 100644 --- a/shinysdr/i/network/export_ws.py +++ b/shinysdr/i/network/export_ws.py @@ -376,7 +376,7 @@ def __dispatch_url(self): self.__log.info('Stream connection to {url}', url=self.transport.location) _scheme, _netloc, path_bytes, _params, query_bytes, _fragment = urlparse(bytes_or_ascii(self.transport.location)) # py2/3: unquote returns str in either version but we want Unicode - path = [six.text_type(urllib.parse.unquote(x)) for x in path_bytes.split(b'/')] + path = [six.ensure_text(urllib.parse.unquote(six.ensure_str(x))) for x in path_bytes.split(b'/')] # parse fixed elements of path # TODO: generally better error reporting, maybe use twisted's Resource dispatch??? From 480d0cfe21a2306d452097cbafa53b3149e5eecc Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 04:14:42 -0400 Subject: [PATCH 10/21] Python blocks must have a reference held on the Python side as long as they are part of a flowgraph --- shinysdr/i/blocks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shinysdr/i/blocks.py b/shinysdr/i/blocks.py index fe6769e9..b0c8f505 100644 --- a/shinysdr/i/blocks.py +++ b/shinysdr/i/blocks.py @@ -302,8 +302,8 @@ def __do_connect(self): # It would make slightly more sense to use unsigned chars, but blocks.float_to_uchar does not support vlen. self.__fft_converter = blocks.float_to_char(vlen=self.__freq_resolution, scale=1.0) - fft_sink = self.__fft_cell.create_sink_internal(numpy.dtype((numpy.int8, output_length))) - scope_sink = self.__scope_cell.create_sink_internal(numpy.dtype(('c8', self.__time_length))) + self.__fft_sink = self.__fft_cell.create_sink_internal(numpy.dtype((numpy.int8, output_length))) + self.__scope_sink = self.__scope_cell.create_sink_internal(numpy.dtype(('c8', self.__time_length))) scope_chunker = blocks.stream_to_vector_decimator( item_size=gr.sizeof_gr_complex, sample_rate=sample_rate, @@ -324,15 +324,15 @@ def __do_connect(self): logarithmizer) if self.__after_fft is not None: self.connect(logarithmizer, self.__after_fft) - self.connect(self.__after_fft, self.__fft_converter, fft_sink) + self.connect(self.__after_fft, self.__fft_converter, self.__fft_sink) self.connect((self.__after_fft, 1), blocks.null_sink(gr.sizeof_float * self.__freq_resolution)) else: - self.connect(logarithmizer, self.__fft_converter, fft_sink) + self.connect(logarithmizer, self.__fft_converter, self.__fft_sink) if self.__enable_scope: self.connect( self.__gate, scope_chunker, - scope_sink) + self.__scope_sink) finally: self.__context.unlock() From 55f2ecef9ba28b530d35f52df852188ad2606f66 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 04:53:14 -0400 Subject: [PATCH 11/21] Monkey-patch txWS to fix it on Python 3 --- shinysdr/i/network/webapp.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/shinysdr/i/network/webapp.py b/shinysdr/i/network/webapp.py index 4bb9aa8a..72f418e3 100644 --- a/shinysdr/i/network/webapp.py +++ b/shinysdr/i/network/webapp.py @@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals +from struct import pack import os import six @@ -50,6 +51,34 @@ from shinysdr.values import SubscriptionContext +# txWS is unfortunately dead upstream, and has buggy Python 3 support. +# On Python 3, upstream txWS can't properly encode frames larger than +# 0x7d bytes. Monkey-patch the function here with a 2+3 compatible +# implementation. +def make_hybi07_frame(buf, opcode=0x1): + """ + Make a HyBi-07 frame. + + This function always creates unmasked frames, and attempts to use the + smallest possible lengths. + """ + + if len(buf) > 0xffff: + length = b"\x7f%s" % pack(">Q", len(buf)) + elif len(buf) > 0x7d: + length = b"\x7e%s" % pack(">H", len(buf)) + else: + length = six.int2byte(len(buf)) + + if isinstance(buf, six.text_type): + buf = buf.encode('utf-8') + + # Always make a normal packet. + header = six.int2byte(0x80 | opcode) + return header + length + buf +txws.make_hybi07_frame = make_hybi07_frame + + def _make_static_resource(pathname): # str() because if we happen to pass unicode as the pathname then directory listings break (discovered with Twisted 16.4.1). r = static.File(str(pathname), From 48bcccc7e034e6ac1a6b87ddb16d88aa7acff67b Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 05:07:51 -0400 Subject: [PATCH 12/21] Fix BlockResource for Python 3 --- shinysdr/i/network/export_http.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/shinysdr/i/network/export_http.py b/shinysdr/i/network/export_http.py index 59fa31de..839dc4e4 100644 --- a/shinysdr/i/network/export_http.py +++ b/shinysdr/i/network/export_http.py @@ -75,15 +75,16 @@ def __init__(self, block, wcommon, deleteSelf): self.__element = _BlockHtmlElement(wcommon) def getChild(self, path, request): + name = path.decode() if self._dynamic: curstate = self._block.state() - if path in curstate: - cell = curstate[path] + if name in curstate: + cell = curstate[name] if cell.type().is_reference(): - return self.__getBlockChild(path, cell.get()) + return self.__getBlockChild(name, cell.get()) else: - if path in self._blockCells: - return self.__getBlockChild(path, self._blockCells[path].get()) + if name in self._blockCells: + return self.__getBlockChild(name, self._blockCells[name].get()) # old-style-class super call return Resource.getChild(self, path, request) @@ -102,7 +103,7 @@ def deleter(): return BlockResource(block, self.__wcommon, deleter) def render_GET(self, request): - accept = request.getHeader('Accept') + accept = request.getHeader(b'Accept') if accept is not None and b'application/json' in accept: # TODO: Implement or obtain correct Accept interpretation request.setHeader(b'Content-Type', b'application/json') return serialize(self.__describe_block()).encode('utf-8') @@ -118,7 +119,7 @@ def render_POST(self, request): assert request.getHeader(b'Content-Type') == b'application/json' reqjson = json.load(request.content) key = block.create_child(reqjson) # note may fail - url = request.prePathURL() + defaultstr('/receivers/') + urllib.parse.quote(key, safe='') + url = six.ensure_text(request.prePathURL()) + '/receivers/' + urllib.parse.quote(key, safe='') request.setResponseCode(201) # Created request.setHeader(b'Location', url) # TODO consider a more useful response From 9089b68c21637047851cc6b40d86bf367b68577d Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 05:21:41 -0400 Subject: [PATCH 13/21] sort can fail on Python 3 if objects are not comparable --- shinysdr/i/network/export_ws.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shinysdr/i/network/export_ws.py b/shinysdr/i/network/export_ws.py index 63b16ffd..409951e5 100644 --- a/shinysdr/i/network/export_ws.py +++ b/shinysdr/i/network/export_ws.py @@ -159,7 +159,7 @@ def __send_references_and_update_refcount(self, objs, is_single): # Decrement refcounts of old (or existing) references. refs = self.__previous_references - refs.sort() # ensure determinism + refs.sort(key=id) # ensure determinism for obj in refs: if obj not in self.__ssi._registered_objs: raise Exception("Shouldn't happen: previous value not registered", obj) @@ -194,7 +194,7 @@ def dec_refcount_and_maybe_notify(self): # capture refs to decrement refs = self.__previous_references - refs.sort() # ensure determinism + refs.sort(key=id) # ensure determinism # drop previous value self.__set_previous_references({}) From 1529d7fe1810cb54827494c31a6e13f2486dbe27 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 05:22:51 -0400 Subject: [PATCH 14/21] Iterators no longer have a next() method in Python 3 --- shinysdr/i/top.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shinysdr/i/top.py b/shinysdr/i/top.py index a0ceaa34..046e19a9 100644 --- a/shinysdr/i/top.py +++ b/shinysdr/i/top.py @@ -155,7 +155,7 @@ def add_receiver(self, mode, key=None, state=None): break if len(self._receivers) > 0: - arbitrary = six.itervalues(self._receivers).next() + arbitrary = next(six.itervalues(self._receivers)) defaults = arbitrary.state_to_json() else: defaults = self.receiver_default_state From 291496f7161cebc7214135c063d9df6dc626ea51 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Tue, 21 Jul 2020 06:33:34 -0400 Subject: [PATCH 15/21] Filenames are not bytes in Py3 --- shinysdr/i/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shinysdr/i/persistence.py b/shinysdr/i/persistence.py index 7c60572e..6546c456 100644 --- a/shinysdr/i/persistence.py +++ b/shinysdr/i/persistence.py @@ -66,7 +66,7 @@ def apply_defaults(): root_object.state_from_json(state_json) # make a backup in case this code version misreads the state and loses things on save (but only if the load succeeded, in case the file but not its backup is bad) # TODO: should automatically use backup if main file is missing or broken - shutil.copyfile(filename, filename + b'~') + shutil.copyfile(filename, filename + '~') else: apply_defaults() From b5b609c4ec6a97e462764afdc2cf6edc985f7202 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Wed, 22 Jul 2020 05:29:24 -0400 Subject: [PATCH 16/21] Handle application/json headers with and without charset specified --- shinysdr/i/db.py | 2 +- shinysdr/i/test_db.py | 4 ++-- shinysdr/testutil.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/shinysdr/i/db.py b/shinysdr/i/db.py index dced2696..21f364f1 100644 --- a/shinysdr/i/db.py +++ b/shinysdr/i/db.py @@ -227,7 +227,7 @@ def render_GET(self, request): return json.dumps(self.__record).encode('utf-8') def render_POST(self, request): - assert request.getHeader(b'Content-Type') == b'application/json; charset=utf-8' + assert request.getHeader(b'Content-Type') in (b'application/json', b'application/json; charset=utf-8') if not self.__database.writable: request.setResponseCode(http.FORBIDDEN) request.setHeader(b'Content-Type', b'text/plain') diff --git a/shinysdr/i/test_db.py b/shinysdr/i/test_db.py index a536ac51..a2b2264e 100644 --- a/shinysdr/i/test_db.py +++ b/shinysdr/i/test_db.py @@ -290,7 +290,7 @@ def test_index_common(self): @defer.inlineCallbacks def test_index_response(self): response, data = yield http_get(reactor, self.__url('/')) - self.assertEqual(response.headers.getRawHeaders('Content-Type'), ['application/json']) + self.assertEqual(response.headers.getRawHeaders('Content-Type'), ['application/json; charset=utf-8']) j = json.loads(data) self.assertEqual(j, self.response_json) @@ -300,7 +300,7 @@ def test_record_common(self): @defer.inlineCallbacks def test_record_response(self): response, data = yield http_get(reactor, self.__url('/1')) - self.assertEqual(response.headers.getRawHeaders('Content-Type'), ['application/json']) + self.assertEqual(response.headers.getRawHeaders('Content-Type'), ['application/json; charset=utf-8']) j = json.loads(data) self.assertEqual(j, self.test_records[1]) diff --git a/shinysdr/testutil.py b/shinysdr/testutil.py index 9ab50b91..9b6ef28b 100644 --- a/shinysdr/testutil.py +++ b/shinysdr/testutil.py @@ -505,7 +505,7 @@ def _assert_http_response_properties(test_case, response, data): if data is not None: # not a HEAD request content_type = response.headers.getRawHeaders(b'Content-Type')[0] - if content_type == 'application/json': + if content_type in ('application/json', 'application/json; charset=utf-8'): json.loads(data) # raises error if it doesn't parse elif content_type.startswith('text/html'): test_case.assertRegex(content_type, r'(?i)text/html;\s*charset=utf-8') From 3a7529bf153706a76e3065791dcb6708fa8145aa Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Wed, 22 Jul 2020 05:35:10 -0400 Subject: [PATCH 17/21] Address lint errors --- shinysdr/i/network/export_http.py | 1 - shinysdr/i/network/webapp.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/shinysdr/i/network/export_http.py b/shinysdr/i/network/export_http.py index 839dc4e4..7660cbcf 100644 --- a/shinysdr/i/network/export_http.py +++ b/shinysdr/i/network/export_http.py @@ -32,7 +32,6 @@ from shinysdr.i.json import serialize from shinysdr.i.network.base import prepath_escaped, template_filepath -from shinysdr.i.pycompat import defaultstr from shinysdr.values import IWritableCollection diff --git a/shinysdr/i/network/webapp.py b/shinysdr/i/network/webapp.py index 72f418e3..334353f3 100644 --- a/shinysdr/i/network/webapp.py +++ b/shinysdr/i/network/webapp.py @@ -76,6 +76,8 @@ def make_hybi07_frame(buf, opcode=0x1): # Always make a normal packet. header = six.int2byte(0x80 | opcode) return header + length + buf + + txws.make_hybi07_frame = make_hybi07_frame From 11b6d2f04ddc6f7159477246891d39b491c8b33c Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Wed, 22 Jul 2020 06:39:26 -0400 Subject: [PATCH 18/21] More bytes/str changes that fix tests on Python 3.8 --- shinysdr/i/config.py | 10 +++++----- shinysdr/i/db.py | 4 ++-- shinysdr/i/depgraph.py | 6 ++++-- shinysdr/i/network/export_http.py | 2 +- shinysdr/i/test_config.py | 3 ++- shinysdr/i/test_db.py | 2 +- shinysdr/i/test_dependencies.py | 2 +- shinysdr/plugins/osmosdr.py | 8 ++++---- shinysdr/plugins/rtl_433.py | 2 +- shinysdr/test_devices.py | 5 +++-- shinysdr/testutil.py | 8 ++++---- 11 files changed, 28 insertions(+), 24 deletions(-) diff --git a/shinysdr/i/config.py b/shinysdr/i/config.py index fe4decb1..597efdbd 100755 --- a/shinysdr/i/config.py +++ b/shinysdr/i/config.py @@ -41,7 +41,7 @@ # Note that gnuradio-dependent modules are loaded lazily, to avoid the startup time if all we're going to do is give a usage message from shinysdr.i.db import DatabaseModel, database_from_csv, databases_from_directory from shinysdr.i.network.base import UNIQUE_PUBLIC_CAP -from shinysdr.i.pycompat import bytes_or_ascii, repr_no_string_tag +from shinysdr.i.pycompat import repr_no_string_tag from shinysdr.i.roots import CapTable, generate_cap @@ -127,8 +127,8 @@ def serve_web(self, self._not_finished() # TODO: See if we're reinventing bits of Twisted service stuff here - http_base_url = _coerce_and_validate_base_url(http_base_url, 'http_base_url', ('http', 'https')) - ws_base_url = _coerce_and_validate_base_url(ws_base_url, 'ws_base_url', ('ws', 'wss'), allow_path=True) + http_base_url = _coerce_and_validate_base_url(http_base_url, 'http_base_url', (b'http', b'https')) + ws_base_url = _coerce_and_validate_base_url(ws_base_url, 'ws_base_url', (b'ws', b'wss'), allow_path=True) if root_cap is not None: root_cap = six.text_type(root_cap) @@ -198,11 +198,11 @@ def _coerce_and_validate_base_url(url_value, label, allowed_schemes, allow_path= if url_value is not None: url_value = str(url_value) - scheme, _netloc, path_bytes, _params, _query_bytes, _fragment = urlparse(bytes_or_ascii(url_value)) + scheme, _netloc, path_bytes, _params, _query_bytes, _fragment = urlparse(six.ensure_binary(url_value)) # Ensure that the protocol is compatible. if scheme.lower() not in allowed_schemes: - raise ConfigException('config.serve_web: {} must be a {} URL but was {}'.format(label, ' or '.join(repr_no_string_tag(s + ':') for s in allowed_schemes), repr_no_string_tag(url_value))) + raise ConfigException('config.serve_web: {} must be a {} URL but was {}'.format(label, ' or '.join(repr_no_string_tag(six.ensure_str(s) + ':') for s in allowed_schemes), repr_no_string_tag(url_value))) # Ensure that there are no path components. There are two reasons for this: # 1. The client makes use of host-relative URLs. diff --git a/shinysdr/i/db.py b/shinysdr/i/db.py index 21f364f1..63510a84 100644 --- a/shinysdr/i/db.py +++ b/shinysdr/i/db.py @@ -207,11 +207,11 @@ def render_POST(self, request): dbdict[rkey] = record self.__database.dirty() # TODO: There is no test that this is done. self.__instantiate(rkey) - url = request.prePathURL() + str(rkey) + url = request.prePathURL() + six.ensure_binary(str(rkey)) request.setResponseCode(http.CREATED) request.setHeader(b'Content-Type', b'text/plain') request.setHeader(b'Location', url) - return url.encode() + return url class _RecordResource(resource.Resource): diff --git a/shinysdr/i/depgraph.py b/shinysdr/i/depgraph.py index 7d727948..ddc5b9a8 100644 --- a/shinysdr/i/depgraph.py +++ b/shinysdr/i/depgraph.py @@ -20,6 +20,8 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import six + from twisted.internet import defer from twisted.internet.task import deferLater from twisted.logger import Logger @@ -115,7 +117,7 @@ def get_fitting(self, other_ff, requester_ff=None): def _schedule_change(self, reason): """Internal for _ActiveFittingHolder -- asynchronously trigger a reevaluation of the graph""" - self.__log.debug('scheduled change ({reason})', reason=unicode(reason)) + self.__log.debug('scheduled change ({reason})', reason=six.ensure_text(reason)) self.__scheduled_change.start() def _mark_for_rebuild(self, fitting_factory): @@ -153,7 +155,7 @@ def add_factory(ff, reason): self.__log.debug('CHANGE: ...completed analysis') - newly_inactive_ffs = (set(self.__active_holders.iterkeys()) + newly_inactive_ffs = (set(self.__active_holders.keys()) .difference(active_ffs) .union(self.__do_not_reuse)) needs_configuration = newly_active_ffs or newly_inactive_ffs diff --git a/shinysdr/i/network/export_http.py b/shinysdr/i/network/export_http.py index 7660cbcf..9ceab6a8 100644 --- a/shinysdr/i/network/export_http.py +++ b/shinysdr/i/network/export_http.py @@ -204,7 +204,7 @@ def render_GET(self, request): 1: 'r', 2: 2 }) - process.pipes[0].write(self.__block.dot_graph()) + process.pipes[0].write(self.__block.dot_graph().encode('utf-8')) process.pipes[0].loseConnection() return NOT_DONE_YET diff --git a/shinysdr/i/test_config.py b/shinysdr/i/test_config.py index 43370595..0c6f7abb 100644 --- a/shinysdr/i/test_config.py +++ b/shinysdr/i/test_config.py @@ -354,7 +354,8 @@ def test_traceback_processing(self): self.assertEqual( file_obj.getvalue() .replace(self.__files.dir, '') - .replace(__file__, ''), + .replace(__file__, '') + .replace('shinysdr.i.config.ConfigException', 'ConfigException'), # Python 3 uses the full name textwrap.dedent("""\ An error occurred while executing the ShinySDR configuration file: File "/config", line 1, in diff --git a/shinysdr/i/test_db.py b/shinysdr/i/test_db.py index a2b2264e..10c61fb5 100644 --- a/shinysdr/i/test_db.py +++ b/shinysdr/i/test_db.py @@ -240,7 +240,7 @@ def test_index_response(self): # TODO: Actually parse/check-that-parses the document self.assertSubstring(textwrap.dedent('''\
  • foo&bar
    • - '''), data) + '''), six.ensure_str(data)) class TestDatabaseResource(unittest.TestCase): diff --git a/shinysdr/i/test_dependencies.py b/shinysdr/i/test_dependencies.py index cb591326..0b52aee9 100644 --- a/shinysdr/i/test_dependencies.py +++ b/shinysdr/i/test_dependencies.py @@ -47,7 +47,7 @@ def test_module_broken_import(self): 'The following libraries/programs are not installed correctly:\n\t (Check: shinysdr.i.test_dependencies_cases.imports failed to import (No module named nonexistent_module_in_dep).)\nPlease (re)install current versions.') else: self.assertEqual(self.t.report(), - 'The following libraries/programs are not installed correctly:\n\t (Check: shinysdr.i.test_dependencies_cases.imports failed to import (No module named \'shinysdr.test.nonexistent_module_in_dep\').)\nPlease (re)install current versions.') + 'The following libraries/programs are not installed correctly:\n\t (Check: shinysdr.i.test_dependencies_cases.imports failed to import (No module named \'shinysdr.nonexistent_module_in_dep\').)\nPlease (re)install current versions.') def test_module_broken_other(self): self.t.check_module('shinysdr.i.test_dependencies_cases.misc', '') diff --git a/shinysdr/plugins/osmosdr.py b/shinysdr/plugins/osmosdr.py index ad487baf..408ef029 100644 --- a/shinysdr/plugins/osmosdr.py +++ b/shinysdr/plugins/osmosdr.py @@ -115,8 +115,8 @@ def __repr__(self): def profile_from_device_string(device_string): # TODO: The input is actually an "args" string, which contains multiple devices space-separated. We should support this, but it is hard because osmosdr does not export the internal args_to_vector function and parsing it ourselves would need to be escaping-aware. - params = {k: v for k, v in osmosdr.device_t(device_string).items()} - for param_key in params.iterkeys(): + params = {k: v for k, v in osmosdr.device_t(six.ensure_str(device_string)).items()} + for param_key in params: if param_key in _default_profiles: # is a device of this type return _default_profiles[param_key] @@ -232,7 +232,7 @@ def OsmoSDRDevice( if profile is None: profile = profile_from_device_string(osmo_device) - source = osmosdr.source(b'numchan=1 ' + osmo_device) + source = osmosdr.source(six.ensure_str('numchan=1 ' + osmo_device)) if source.get_num_channels() < 1: # osmosdr.source doesn't throw an exception, allegedly because gnuradio can't handle it in a hier_block2 initializer. But we want to fail understandably, so recover by detecting it (sample rate = 0, which is otherwise nonsense) raise LookupError('OsmoSDR device not found (device string = %r)' % osmo_device) @@ -392,7 +392,7 @@ def set_agc(self, value): label='Antenna') def get_antenna(self): if self.__source is None: return '' - return six.text_type(self.__source.get_antenna(ch), 'ascii') + return six.ensure_text(self.__source.get_antenna(ch), 'ascii') @setter def set_antenna(self, value): diff --git a/shinysdr/plugins/rtl_433.py b/shinysdr/plugins/rtl_433.py index b0b1ca38..517036c1 100644 --- a/shinysdr/plugins/rtl_433.py +++ b/shinysdr/plugins/rtl_433.py @@ -97,7 +97,7 @@ def __init__(self, mode='433', input_rate=0, context=None): args=[ b'env', b'rtl_433', b'-F', b'json', # output format - b'-r', str(demod_rate) + b'sps:iq:cf32:-', # specify input format and to use stdin + b'-r', b'%dsps:iq:cf32:-' % (demod_rate), # specify input format and to use stdin b'-M', 'newmodel', ], childFDs={ diff --git a/shinysdr/test_devices.py b/shinysdr/test_devices.py index f9a9b0a6..081ec2f0 100644 --- a/shinysdr/test_devices.py +++ b/shinysdr/test_devices.py @@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import six from twisted.trial import unittest from gnuradio import blocks @@ -121,7 +122,7 @@ def test_components_disjoint(self): Device(components={'b': StubComponent()}) ]) self.assertEqual(d, IDevice(d)) - self.assertEqual(sorted(d.get_components_dict().iterkeys()), ['a', 'b']) + self.assertEqual(sorted(six.iterkeys(d.get_components_dict())), ['a', 'b']) def test_components_conflict(self): d = merge_devices([ @@ -129,7 +130,7 @@ def test_components_conflict(self): Device(components={'a': StubComponent()}) ]) self.assertEqual(d, IDevice(d)) - self.assertEqual(sorted(d.get_components_dict().iterkeys()), ['0-a', '1-a']) + self.assertEqual(sorted(six.iterkeys(d.get_components_dict())), ['0-a', '1-a']) def test_vfos(self): d = merge_devices([ diff --git a/shinysdr/testutil.py b/shinysdr/testutil.py index 9b6ef28b..391c801e 100644 --- a/shinysdr/testutil.py +++ b/shinysdr/testutil.py @@ -505,12 +505,12 @@ def _assert_http_response_properties(test_case, response, data): if data is not None: # not a HEAD request content_type = response.headers.getRawHeaders(b'Content-Type')[0] - if content_type in ('application/json', 'application/json; charset=utf-8'): + if content_type in (b'application/json', b'application/json; charset=utf-8'): json.loads(data) # raises error if it doesn't parse - elif content_type.startswith('text/html'): - test_case.assertRegex(content_type, r'(?i)text/html;\s*charset=utf-8') + elif content_type.startswith(b'text/html'): + test_case.assertRegex(content_type, br'(?i)text/html;\s*charset=utf-8') test_case.assertRegex(data, br'(?i)') - elif content_type in ('application/javascript', 'text/javascript', 'audio/wav'): + elif content_type in (b'application/javascript', b'text/javascript', b'audio/wav'): pass else: raise Exception('Don\'t know what content type checking to do', data[0], content_type) From 9835805e14d5f825fb23f1e980e827d86b7a61f3 Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Wed, 22 Jul 2020 07:14:02 -0400 Subject: [PATCH 19/21] Fix lint --- shinysdr/i/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shinysdr/i/test_config.py b/shinysdr/i/test_config.py index 0c6f7abb..1e829c06 100644 --- a/shinysdr/i/test_config.py +++ b/shinysdr/i/test_config.py @@ -355,7 +355,7 @@ def test_traceback_processing(self): file_obj.getvalue() .replace(self.__files.dir, '') .replace(__file__, '') - .replace('shinysdr.i.config.ConfigException', 'ConfigException'), # Python 3 uses the full name + .replace('shinysdr.i.config.ConfigException', 'ConfigException'), # Python 3 uses the full name textwrap.dedent("""\ An error occurred while executing the ShinySDR configuration file: File "/config", line 1, in From d4d6684114872a7771efe3e75f3ff329cb6ca4df Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Wed, 16 Sep 2020 20:59:35 -0400 Subject: [PATCH 20/21] Address review comments --- setup.py | 1 + shinysdr/i/network/export_http.py | 4 ++-- shinysdr/i/network/session_http.py | 4 ++-- shinysdr/i/network/webapp.py | 2 +- shinysdr/plugins/rtl_433.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 2b6e38b9..bd9eec60 100755 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ import os.path import subprocess +# Python 2+3 support try: from urllib.request import urlretrieve except ImportError: diff --git a/shinysdr/i/network/export_http.py b/shinysdr/i/network/export_http.py index 9ceab6a8..a8f558e6 100644 --- a/shinysdr/i/network/export_http.py +++ b/shinysdr/i/network/export_http.py @@ -70,11 +70,11 @@ def __init__(self, block, wcommon, deleteSelf): if cell.type().is_reference(): self._blockCells[key] = cell else: - self.putChild(key.encode(), ValueCellResource(cell, self.__wcommon)) + self.putChild(key.encode('utf-8'), ValueCellResource(cell, self.__wcommon)) self.__element = _BlockHtmlElement(wcommon) def getChild(self, path, request): - name = path.decode() + name = path.decode('utf-8') if self._dynamic: curstate = self._block.state() if name in curstate: diff --git a/shinysdr/i/network/session_http.py b/shinysdr/i/network/session_http.py index a0eb91a0..ef3f87d6 100644 --- a/shinysdr/i/network/session_http.py +++ b/shinysdr/i/network/session_http.py @@ -41,7 +41,7 @@ def __init__(self, session, wcommon, read_only_dbs, writable_db): self.putChild(b'', ElementRenderingResource(_RadioIndexHtmlElement(wcommon))) # Exported radio control objects - self.putChild(CAP_OBJECT_PATH_ELEMENT.encode(), BlockResource(session, wcommon, _not_deletable)) + self.putChild(CAP_OBJECT_PATH_ELEMENT.encode('utf-8'), BlockResource(session, wcommon, _not_deletable)) # Frequency DB self.putChild(b'dbs', shinysdr.i.db.DatabasesResource(read_only_dbs)) @@ -54,7 +54,7 @@ def __init__(self, session, wcommon, read_only_dbs, writable_db): self.putChild(b'ephemeris', EphemerisResource()) # Standard audio-file-over-HTTP audio stream (the ShinySDR web client uses WebSockets instead, but both have the same path modulo protocol) - self.putChild(AUDIO_STREAM_PATH_ELEMENT.encode(), AudioStreamResource(session)) + self.putChild(AUDIO_STREAM_PATH_ELEMENT.encode('utf-8'), AudioStreamResource(session)) class _RadioIndexHtmlElement(EntryPointIndexElement): diff --git a/shinysdr/i/network/webapp.py b/shinysdr/i/network/webapp.py index 334353f3..4d384724 100644 --- a/shinysdr/i/network/webapp.py +++ b/shinysdr/i/network/webapp.py @@ -222,7 +222,7 @@ def _put_plugin_resources(wcommon, client_resource): client_resource.putChild(b'plugins', plugin_resources) for resource_def in getPlugins(_IClientResourceDef, shinysdr.plugins): # Add the plugin's resource to static serving - plugin_resources.putChild(resource_def.key.encode(), resource_def.resource) + plugin_resources.putChild(resource_def.key.encode('utf-8'), resource_def.resource) plugin_resource_url = '/client/plugins/' + urllib.parse.quote(resource_def.key, safe='') + '/' # Tell the client to load the plugins # TODO constrain path values to be relative (not on a different origin, to not leak urls) diff --git a/shinysdr/plugins/rtl_433.py b/shinysdr/plugins/rtl_433.py index 517036c1..8148b4a3 100644 --- a/shinysdr/plugins/rtl_433.py +++ b/shinysdr/plugins/rtl_433.py @@ -97,7 +97,7 @@ def __init__(self, mode='433', input_rate=0, context=None): args=[ b'env', b'rtl_433', b'-F', b'json', # output format - b'-r', b'%dsps:iq:cf32:-' % (demod_rate), # specify input format and to use stdin + b'-r', b'%dsps:iq:cf32:-' % (demod_rate,), # specify input format and to use stdin b'-M', 'newmodel', ], childFDs={ From 0f0992bd1cdcb0ad21f74a8ce425829157cdc57a Mon Sep 17 00:00:00 2001 From: Quentin Smith Date: Thu, 17 Sep 2020 00:23:41 -0400 Subject: [PATCH 21/21] Test removing charset from application/json headers --- shinysdr/i/db.py | 6 +++--- shinysdr/i/ephemeris.py | 2 +- shinysdr/i/test_db.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shinysdr/i/db.py b/shinysdr/i/db.py index 63510a84..3b95c45a 100644 --- a/shinysdr/i/db.py +++ b/shinysdr/i/db.py @@ -186,7 +186,7 @@ def __init__(self, db, instantiate): self.__instantiate = instantiate def render_GET(self, request): - request.setHeader(b'Content-Type', b'application/json; charset=utf-8') + request.setHeader(b'Content-Type', b'application/json') return json.dumps({ u'records': self.__database.records, u'writable': self.__database.writable @@ -223,11 +223,11 @@ def __init__(self, database, record): self.__record = record def render_GET(self, request): - request.setHeader(b'Content-Type', b'application/json; charset=utf-8') + request.setHeader(b'Content-Type', b'application/json') return json.dumps(self.__record).encode('utf-8') def render_POST(self, request): - assert request.getHeader(b'Content-Type') in (b'application/json', b'application/json; charset=utf-8') + assert request.getHeader(b'Content-Type') in (b'application/json',) if not self.__database.writable: request.setResponseCode(http.FORBIDDEN) request.setHeader(b'Content-Type', b'text/plain') diff --git a/shinysdr/i/ephemeris.py b/shinysdr/i/ephemeris.py index a12f53e3..ec73b7cb 100644 --- a/shinysdr/i/ephemeris.py +++ b/shinysdr/i/ephemeris.py @@ -53,7 +53,7 @@ def render_GET(self, request): y = math.cos(sun.az) * math.cos(sun.alt) z = -math.sin(sun.alt) - request.setHeader(b'Content-Type', b'application/json; charset=utf-8') + request.setHeader(b'Content-Type', b'application/json') return json.dumps([x, y, z]).encode('utf-8') diff --git a/shinysdr/i/test_db.py b/shinysdr/i/test_db.py index 10c61fb5..c5d19c86 100644 --- a/shinysdr/i/test_db.py +++ b/shinysdr/i/test_db.py @@ -290,7 +290,7 @@ def test_index_common(self): @defer.inlineCallbacks def test_index_response(self): response, data = yield http_get(reactor, self.__url('/')) - self.assertEqual(response.headers.getRawHeaders('Content-Type'), ['application/json; charset=utf-8']) + self.assertEqual(response.headers.getRawHeaders('Content-Type'), ['application/json']) j = json.loads(data) self.assertEqual(j, self.response_json) @@ -300,7 +300,7 @@ def test_record_common(self): @defer.inlineCallbacks def test_record_response(self): response, data = yield http_get(reactor, self.__url('/1')) - self.assertEqual(response.headers.getRawHeaders('Content-Type'), ['application/json; charset=utf-8']) + self.assertEqual(response.headers.getRawHeaders('Content-Type'), ['application/json']) j = json.loads(data) self.assertEqual(j, self.test_records[1])