From 856055dff1bc3dca7d162be552e3f7fc15ea52ee Mon Sep 17 00:00:00 2001 From: keithsterling Date: Tue, 12 Dec 2017 17:40:23 +0000 Subject: [PATCH] Further improvements to unicode handling across all file operations --- src/programy/clients/client.py | 2 +- src/programy/clients/rest.py | 2 +- src/programy/clients/twitter.py | 4 ++-- src/programy/config/file/json_file.py | 2 +- src/programy/config/file/xml_file.py | 2 +- src/programy/config/file/yaml_file.py | 2 +- src/programy/dialog.py | 4 ++-- src/programy/mappings/base.py | 2 +- src/programy/parser/factory.py | 2 +- src/programy/parser/template/nodes/learnf.py | 2 +- src/programy/processors/processing.py | 2 +- src/programy/security/authorise/usergrouploader.py | 2 +- src/programy/spelling/norvig.py | 2 +- src/programy/utils/geo/geonames.py | 6 +++--- src/programy/utils/geo/google.py | 12 ++++++------ src/programy/utils/license/keys.py | 2 +- src/programy/utils/newsapi/newsapi.py | 4 ++-- src/programy/utils/weather/metoffice.py | 2 +- 18 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/programy/clients/client.py b/src/programy/clients/client.py index 9655f48d0..c4f083e9a 100644 --- a/src/programy/clients/client.py +++ b/src/programy/clients/client.py @@ -105,7 +105,7 @@ def parse_arguments(self, argument_parser): def initiate_logging(self, arguments): if arguments.logging is not None: - with open(arguments.logging, 'r+') as yml_data_file: + with open(arguments.logging, 'r+', encoding="utf-8") as yml_data_file: logging_config = yaml.load(yml_data_file) logging.config.dictConfig(logging_config) if logging.getLogger().isEnabledFor(logging.INFO): diff --git a/src/programy/clients/rest.py b/src/programy/clients/rest.py index 7f0cd03e9..0c9a6733b 100644 --- a/src/programy/clients/rest.py +++ b/src/programy/clients/rest.py @@ -35,7 +35,7 @@ def get_client_configuration(self): def load_api_keys(self): if self.configuration.client_configuration.api_key_file is not None: - with open(self.configuration.client_configuration.api_key_file, "r") as api_key_file: + with open(self.configuration.client_configuration.api_key_file, "r", encoding="utf-8") as api_key_file: for api_key in api_key_file: self.api_keys.append(api_key.strip()) diff --git a/src/programy/clients/twitter.py b/src/programy/clients/twitter.py index 880338425..bdd0094b8 100644 --- a/src/programy/clients/twitter.py +++ b/src/programy/clients/twitter.py @@ -219,7 +219,7 @@ def _get_last_message_ids(self): logging.debug("Reads messages ids from [%s]", self.configuration.client_configuration.storage_location) if os.path.exists(self.configuration.client_configuration.storage_location): try: - with open(self.configuration.client_configuration.storage_location, "r") as idfile: + with open(self.configuration.client_configuration.storage_location, "r", encoding="utf-8") as idfile: last_direct_message_id = int(idfile.readline().strip()) last_status_id = int(idfile.readline().strip()) except Exception as excep: @@ -232,7 +232,7 @@ def _store_last_message_ids(self, last_direct_message_id, last_status_id): if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug("Writing messages ids to [%s]", self.configuration.client_configuration.storage_location) try: - with open(self.configuration.client_configuration.storage_location, "w+") as idfile: + with open(self.configuration.client_configuration.storage_location, "w+", encoding="utf-8") as idfile: idfile.write("%d\n"%last_direct_message_id) idfile.write("%d\n"%last_status_id) except Exception as excep: diff --git a/src/programy/config/file/json_file.py b/src/programy/config/file/json_file.py index 645e0b601..b35a566ee 100644 --- a/src/programy/config/file/json_file.py +++ b/src/programy/config/file/json_file.py @@ -36,7 +36,7 @@ def load_from_text(self, text, client_configuration, bot_root): def load_from_file(self, filename, client_configuration, bot_root): configuration = ProgramyConfiguration(client_configuration) - with open(filename, 'r+') as json_data_file: + with open(filename, 'r+', encoding="utf-8") as json_data_file: self.json_data = json.load(json_data_file) configuration.load_config_data(self, bot_root) return configuration diff --git a/src/programy/config/file/xml_file.py b/src/programy/config/file/xml_file.py index cd823ac10..e1481beb9 100644 --- a/src/programy/config/file/xml_file.py +++ b/src/programy/config/file/xml_file.py @@ -39,7 +39,7 @@ def load_from_text(self, text, client_configuration, bot_root): def load_from_file(self, filename, client_configuration, bot_root): configuration = ProgramyConfiguration(client_configuration) - with open(filename, 'r+') as xml_data_file: + with open(filename, 'r+', encoding="utf-8") as xml_data_file: tree = ET.parse(xml_data_file, parser=LineNumberingParser()) self.xml_data = tree.getroot() configuration.load_config_data(self, bot_root) diff --git a/src/programy/config/file/yaml_file.py b/src/programy/config/file/yaml_file.py index 92aa36d9f..dbe130aa8 100644 --- a/src/programy/config/file/yaml_file.py +++ b/src/programy/config/file/yaml_file.py @@ -37,7 +37,7 @@ def load_from_text(self, text, client_configuration, bot_root): def load_from_file(self, filename, client_configuration, bot_root): configuration = ProgramyConfiguration(client_configuration) - with open(filename, 'r+') as yml_data_file: + with open(filename, 'r+', encoding="utf-8") as yml_data_file: self.yaml_data = yaml.load(yml_data_file) configuration.load_config_data(self, bot_root) return configuration diff --git a/src/programy/dialog.py b/src/programy/dialog.py index 1b5e69da1..d4c0aed38 100644 --- a/src/programy/dialog.py +++ b/src/programy/dialog.py @@ -265,7 +265,7 @@ def save_conversation(self, conversation, clientid): if self._config._dir is not None: if os.path.exists(self._config._dir): filename = self._config._dir + os.sep + clientid + ".convo" - with open(filename, "w+") as convo_file: + with open(filename, "w+", encoding="utf-8") as convo_file: for name, value in conversation._properties.items(): convo_file.write("%s:%s\n"%(name, value)) convo_file.write("\n") @@ -292,7 +292,7 @@ def load_conversation(self, conversation, clientid, restore_last_topic=False): if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug("Loading Conversation") - with open(filename, "r") as convo_file: + with open(filename, "r", encoding="utf-8") as convo_file: for line in convo_file: if ':' in line: splits = line.split(":") diff --git a/src/programy/mappings/base.py b/src/programy/mappings/base.py index 8fb1f6f59..a9307d53a 100644 --- a/src/programy/mappings/base.py +++ b/src/programy/mappings/base.py @@ -44,7 +44,7 @@ def process_line(self, line): def load_from_filename(self, filename): count = 0 - with open(filename, "r") as data_file: + with open(filename, "r", encoding="utf-8") as data_file: for line in data_file: if self.process_line(line): count += 1 diff --git a/src/programy/parser/factory.py b/src/programy/parser/factory.py index 6d1f7fb04..83e6ff3d3 100644 --- a/src/programy/parser/factory.py +++ b/src/programy/parser/factory.py @@ -73,7 +73,7 @@ def load_nodes_config_from_file(self, filename=None): if filename is None or os.path.exists(filename) is False: filename = self.default_config_file() - with open(filename, "r") as node_file: + with open(filename, "r", encoding="utf-8") as node_file: for line in node_file: line = line.strip() self.process_config_line(line) diff --git a/src/programy/parser/template/nodes/learnf.py b/src/programy/parser/template/nodes/learnf.py index ea7a6c434..08ce83188 100644 --- a/src/programy/parser/template/nodes/learnf.py +++ b/src/programy/parser/template/nodes/learnf.py @@ -55,7 +55,7 @@ def write_learnf_to_file(self, bot, clientid, category): logging.debug("Writing learnf to %s", learnf_path) if os.path.isfile(learnf_path) is False: - file = open(learnf_path, "w+") + file = open(learnf_path, "w+", encoding="utf-8") file.write('\n') file.write('\n') file.write('\n') diff --git a/src/programy/processors/processing.py b/src/programy/processors/processing.py index e407d6d82..1a322354b 100644 --- a/src/programy/processors/processing.py +++ b/src/programy/processors/processing.py @@ -30,7 +30,7 @@ def load(self, filename, *args, **kw): if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug("Loading processors from file [%s]", filename) count = 0 - with open(filename, "r") as file: + with open(filename, "r", encoding="utf-8") as file: for line in file: line = line.strip() if line: diff --git a/src/programy/security/authorise/usergrouploader.py b/src/programy/security/authorise/usergrouploader.py index 31d61474d..7d7a88212 100644 --- a/src/programy/security/authorise/usergrouploader.py +++ b/src/programy/security/authorise/usergrouploader.py @@ -30,7 +30,7 @@ def load_users_and_groups_from_text(self, text): return self.load_users_and_groups_from_yaml(yaml_data) def load_users_and_groups_from_file(self, filename): - with open(filename, 'r+') as yml_data_file: + with open(filename, 'r+', encoding="utf-8") as yml_data_file: yaml_data = yaml.load(yml_data_file) return self.load_users_and_groups_from_yaml(yaml_data) diff --git a/src/programy/spelling/norvig.py b/src/programy/spelling/norvig.py index 17b2db639..acc05183e 100644 --- a/src/programy/spelling/norvig.py +++ b/src/programy/spelling/norvig.py @@ -21,7 +21,7 @@ def __init__(self, spelling_config=None): if logging.getLogger().isEnabledFor(logging.INFO): logging.info("Loading spelling corpus [%s]", corpus_filename) - self.words = Counter(self._all_words(open(corpus_filename).read())) + self.words = Counter(self._all_words(open(corpus_filename, encoding="utf-8").read())) self.sum_of_words = sum(self.words.values()) def _all_words(self, text): diff --git a/src/programy/utils/geo/geonames.py b/src/programy/utils/geo/geonames.py index 2198eb0fe..1eb4107ed 100644 --- a/src/programy/utils/geo/geonames.py +++ b/src/programy/utils/geo/geonames.py @@ -62,12 +62,12 @@ def _get_latlong_for_postcode_response(self, postcode): return json.loads(content.decode('utf8')) def load_get_latlong_for_postcode_from_file(self, filename): - with open(filename, "w+") as response_file: + with open(filename, "w+", encoding="utf-8") as response_file: return json.load(response_file) def store_get_latlong_for_postcode_to_file(self, postcode, filename): content = self._get_latlong_for_postcode_response(postcode) - with open(filename, "w+") as response_file: + with open(filename, "w+", encoding="utf-8") as response_file: json.dump(content, response_file, sort_keys=True, indent=2) def get_latlong_for_postcode(self, postcode): @@ -75,7 +75,7 @@ def get_latlong_for_postcode(self, postcode): if GeoNamesApi.get_latlong_for_postcode_response_file is None: data = self._get_latlong_for_postcode_response(postcode) else: - with open(GeoNamesApi.get_latlong_for_postcode_response_file, "r") as datafile: + with open(GeoNamesApi.get_latlong_for_postcode_response_file, "r", encoding="utf-8") as datafile: data = json.load(datafile) if 'postalCodes' not in data: diff --git a/src/programy/utils/geo/google.py b/src/programy/utils/geo/google.py index 5db2aa903..be14041b2 100644 --- a/src/programy/utils/geo/google.py +++ b/src/programy/utils/geo/google.py @@ -248,7 +248,7 @@ def get_latlong_for_location(self, location): else: if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug("get_latlong_for_location - using mock file") - with open(self.response_file_for_get_latlong_for_location, "r") as response_file: + with open(self.response_file_for_get_latlong_for_location, "r", encoding="utf-8") as response_file: response = json.load(response_file) geodata = GoogelMapsResult() @@ -257,7 +257,7 @@ def get_latlong_for_location(self, location): def store_get_latlong_for_location_to_file(self, location, filename): response = self._get_latlong_for_location_response(location) - with open(filename, "w+") as data_file: + with open(filename, "w+", encoding="utf-8") as data_file: json.dump(response, data_file, sort_keys=True, indent=2) ################## @@ -281,7 +281,7 @@ def get_distance_between_addresses(self, origin, destination, country="UK", mode else: if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug("get_distance_between_addresses - using mock file") - with open(self.response_file_for_get_distance_between_addresses, "r") as response_file: + with open(self.response_file_for_get_distance_between_addresses, "r", encoding="utf-8") as response_file: response = json.load(response_file) if response['status'] == 'OK': @@ -298,7 +298,7 @@ def get_distance_between_addresses(self, origin, destination, country="UK", mode def store_get_distance_between_addresses_as_file(self, origin, destination, filename, country="UK", mode="driving", units="imperial"): response = self._get_distance_between_addresses(origin, destination, country, mode, units) - with open(filename, "w+") as data_file: + with open(filename, "w+", encoding="utf-8") as data_file: json.dump(response, data_file, sort_keys=True, indent=2) ################## @@ -322,7 +322,7 @@ def get_directions_between_addresses(self, origin, destination, country="UK", mo else: if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug("get_directions_between_addresses - using mock file") - with open(self.response_file_for_get_directions_between_addresses, "r") as response_file: + with open(self.response_file_for_get_directions_between_addresses, "r", encoding="utf-8") as response_file: response = json.load(response_file) if response['status'] == 'OK': @@ -339,5 +339,5 @@ def get_directions_between_addresses(self, origin, destination, country="UK", mo def store_get_directions_between_addresses_as_file(self, origin, destination, filename, country="UK", mode="driving", units="imperial"): response = self._get_directions_between_addresses_response(origin, destination, country, mode, units) - with open(filename, "w+") as data_file: + with open(filename, "w+", encoding="utf-8") as data_file: json.dump(response, data_file, sort_keys=True, indent=2) diff --git a/src/programy/utils/license/keys.py b/src/programy/utils/license/keys.py index 202d90f30..7bb06e13d 100644 --- a/src/programy/utils/license/keys.py +++ b/src/programy/utils/license/keys.py @@ -46,7 +46,7 @@ def load_license_key_file(self, license_key_filename): try: if logging.getLogger().isEnabledFor(logging.INFO): logging.info("Loading license key file: [%s]", license_key_filename) - with open(license_key_filename, "r") as license_file: + with open(license_key_filename, "r", encoding="utf-8") as license_file: for line in license_file: self._process_license_key_line(line) except Exception: diff --git a/src/programy/utils/newsapi/newsapi.py b/src/programy/utils/newsapi/newsapi.py index 73cf74059..dba86582e 100644 --- a/src/programy/utils/newsapi/newsapi.py +++ b/src/programy/utils/newsapi/newsapi.py @@ -273,12 +273,12 @@ def to_json(articles): @staticmethod def json_to_file(filename, json_data): - with open(filename, 'w+') as json_file: + with open(filename, 'w+', encoding="utf-8") as json_file: json.dump(json_data, json_file) @staticmethod def json_from_file(filename): - with open(filename, 'r+') as json_file: + with open(filename, 'r+', encoding="utf-8") as json_file: return json.load(json_file) @staticmethod diff --git a/src/programy/utils/weather/metoffice.py b/src/programy/utils/weather/metoffice.py index c15962a58..91cb6b1ec 100644 --- a/src/programy/utils/weather/metoffice.py +++ b/src/programy/utils/weather/metoffice.py @@ -592,7 +592,7 @@ def current_observation(self, lat, lon): return self.nearest_location_observation(lat, lon) def write_datapoints_to_file(self, datapoints, filename): - with open(filename, "w+") as data_file: + with open(filename, "w+", encoding="utf-8") as data_file: json.dump(datapoints, data_file, sort_keys=True, indent=2) def load_datapoints_from_file(self, filename):