Further improvements to unicode handling across all file operations

src/programy/clients/client.py

@@ -105,7 +105,7 @@ def parse_arguments(self, argument_parser):
 
     def initiate_logging(self, arguments):
         if arguments.logging is not None:
-            with open(arguments.logging, 'r+') as yml_data_file:
+            with open(arguments.logging, 'r+', encoding="utf-8") as yml_data_file:
                 logging_config = yaml.load(yml_data_file)
                 logging.config.dictConfig(logging_config)
                 if logging.getLogger().isEnabledFor(logging.INFO):

src/programy/clients/rest.py

@@ -35,7 +35,7 @@ def get_client_configuration(self):
 
     def load_api_keys(self):
         if self.configuration.client_configuration.api_key_file is not None:
-            with open(self.configuration.client_configuration.api_key_file, "r") as api_key_file:
+            with open(self.configuration.client_configuration.api_key_file, "r", encoding="utf-8") as api_key_file:
                 for api_key in api_key_file:
                     self.api_keys.append(api_key.strip())
 

src/programy/clients/twitter.py

@@ -219,7 +219,7 @@ def _get_last_message_ids(self):
                 logging.debug("Reads messages ids from [%s]", self.configuration.client_configuration.storage_location)
             if os.path.exists(self.configuration.client_configuration.storage_location):
                 try:
-                    with open(self.configuration.client_configuration.storage_location, "r") as idfile:
+                    with open(self.configuration.client_configuration.storage_location, "r", encoding="utf-8") as idfile:
                         last_direct_message_id = int(idfile.readline().strip())
                         last_status_id = int(idfile.readline().strip())
                 except Exception as excep:
@@ -232,7 +232,7 @@ def _store_last_message_ids(self, last_direct_message_id, last_status_id):
             if logging.getLogger().isEnabledFor(logging.DEBUG):
                 logging.debug("Writing messages ids to [%s]", self.configuration.client_configuration.storage_location)
             try:
-                with open(self.configuration.client_configuration.storage_location, "w+") as idfile:
+                with open(self.configuration.client_configuration.storage_location, "w+", encoding="utf-8") as idfile:
                     idfile.write("%d\n"%last_direct_message_id)
                     idfile.write("%d\n"%last_status_id)
             except Exception as excep:

src/programy/config/file/json_file.py

@@ -36,7 +36,7 @@ def load_from_text(self, text, client_configuration, bot_root):
 
     def load_from_file(self, filename, client_configuration, bot_root):
         configuration = ProgramyConfiguration(client_configuration)
-        with open(filename, 'r+') as json_data_file:
+        with open(filename, 'r+', encoding="utf-8") as json_data_file:
             self.json_data = json.load(json_data_file)
             configuration.load_config_data(self, bot_root)
         return configuration

src/programy/config/file/xml_file.py

@@ -39,7 +39,7 @@ def load_from_text(self, text, client_configuration, bot_root):
 
     def load_from_file(self, filename, client_configuration, bot_root):
         configuration = ProgramyConfiguration(client_configuration)
-        with open(filename, 'r+') as xml_data_file:
+        with open(filename, 'r+', encoding="utf-8") as xml_data_file:
             tree = ET.parse(xml_data_file, parser=LineNumberingParser())
             self.xml_data = tree.getroot()
             configuration.load_config_data(self, bot_root)

src/programy/config/file/yaml_file.py

@@ -37,7 +37,7 @@ def load_from_text(self, text, client_configuration, bot_root):
 
     def load_from_file(self, filename, client_configuration, bot_root):
         configuration = ProgramyConfiguration(client_configuration)
-        with open(filename, 'r+') as yml_data_file:
+        with open(filename, 'r+', encoding="utf-8") as yml_data_file:
             self.yaml_data = yaml.load(yml_data_file)
             configuration.load_config_data(self, bot_root)
         return configuration

src/programy/dialog.py

@@ -265,7 +265,7 @@ def save_conversation(self, conversation, clientid):
             if self._config._dir is not None:
                 if os.path.exists(self._config._dir):
                     filename = self._config._dir + os.sep + clientid + ".convo"
-                    with open(filename, "w+") as convo_file:
+                    with open(filename, "w+", encoding="utf-8") as convo_file:
                         for name, value in conversation._properties.items():
                             convo_file.write("%s:%s\n"%(name, value))
                         convo_file.write("\n")
@@ -292,7 +292,7 @@ def load_conversation(self, conversation, clientid, restore_last_topic=False):
                             if logging.getLogger().isEnabledFor(logging.DEBUG):
                                 logging.debug("Loading Conversation")
 
-                            with open(filename, "r") as convo_file:
+                            with open(filename, "r", encoding="utf-8") as convo_file:
                                 for line in convo_file:
                                     if ':' in line:
                                         splits = line.split(":")

src/programy/mappings/base.py

@@ -44,7 +44,7 @@ def process_line(self, line):
 
     def load_from_filename(self, filename):
         count = 0
-        with open(filename, "r") as data_file:
+        with open(filename, "r", encoding="utf-8") as data_file:
             for line in data_file:
                 if self.process_line(line):
                     count += 1

src/programy/parser/factory.py

@@ -73,7 +73,7 @@ def load_nodes_config_from_file(self, filename=None):
             if filename is None or os.path.exists(filename) is False:
                 filename = self.default_config_file()
 
-            with open(filename, "r") as node_file:
+            with open(filename, "r", encoding="utf-8") as node_file:
                 for line in node_file:
                     line = line.strip()
                     self.process_config_line(line)

src/programy/parser/template/nodes/learnf.py

@@ -55,7 +55,7 @@ def write_learnf_to_file(self, bot, clientid, category):
             logging.debug("Writing learnf to %s", learnf_path)
 
         if os.path.isfile(learnf_path) is False:
-            file = open(learnf_path, "w+")
+            file = open(learnf_path, "w+", encoding="utf-8")
             file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
             file.write('<aiml>\n')
             file.write('</aiml>\n')

src/programy/processors/processing.py

@@ -30,7 +30,7 @@ def load(self, filename, *args, **kw):
         if logging.getLogger().isEnabledFor(logging.DEBUG):
             logging.debug("Loading processors from file [%s]", filename)
         count = 0
-        with open(filename, "r") as file:
+        with open(filename, "r", encoding="utf-8") as file:
             for line in file:
                 line = line.strip()
                 if line:

src/programy/security/authorise/usergrouploader.py

@@ -30,7 +30,7 @@ def load_users_and_groups_from_text(self, text):
         return self.load_users_and_groups_from_yaml(yaml_data)
 
     def load_users_and_groups_from_file(self, filename):
-        with open(filename, 'r+') as yml_data_file:
+        with open(filename, 'r+', encoding="utf-8") as yml_data_file:
             yaml_data = yaml.load(yml_data_file)
         return self.load_users_and_groups_from_yaml(yaml_data)
 

src/programy/spelling/norvig.py

@@ -21,7 +21,7 @@ def __init__(self, spelling_config=None):
 
         if logging.getLogger().isEnabledFor(logging.INFO):
             logging.info("Loading spelling corpus [%s]", corpus_filename)
-        self.words = Counter(self._all_words(open(corpus_filename).read()))
+        self.words = Counter(self._all_words(open(corpus_filename, encoding="utf-8").read()))
         self.sum_of_words = sum(self.words.values())
 
     def _all_words(self, text):

src/programy/utils/geo/geonames.py

@@ -62,20 +62,20 @@ def _get_latlong_for_postcode_response(self, postcode):
         return json.loads(content.decode('utf8'))
 
     def load_get_latlong_for_postcode_from_file(self, filename):
-        with open(filename, "w+") as response_file:
+        with open(filename, "w+", encoding="utf-8") as response_file:
             return json.load(response_file)
 
     def store_get_latlong_for_postcode_to_file(self, postcode, filename):
         content = self._get_latlong_for_postcode_response(postcode)
-        with open(filename, "w+") as response_file:
+        with open(filename, "w+", encoding="utf-8") as response_file:
             json.dump(content, response_file, sort_keys=True, indent=2)
 
     def get_latlong_for_postcode(self, postcode):
 
         if GeoNamesApi.get_latlong_for_postcode_response_file is None:
             data = self._get_latlong_for_postcode_response(postcode)
         else:
-            with open(GeoNamesApi.get_latlong_for_postcode_response_file, "r") as datafile:
+            with open(GeoNamesApi.get_latlong_for_postcode_response_file, "r", encoding="utf-8") as datafile:
                 data = json.load(datafile)
 
         if 'postalCodes' not in data:

src/programy/utils/geo/google.py

@@ -248,7 +248,7 @@ def get_latlong_for_location(self, location):
         else:
             if logging.getLogger().isEnabledFor(logging.DEBUG):
                 logging.debug("get_latlong_for_location - using mock file")
-            with open(self.response_file_for_get_latlong_for_location, "r") as response_file:
+            with open(self.response_file_for_get_latlong_for_location, "r", encoding="utf-8") as response_file:
                 response = json.load(response_file)
 
         geodata = GoogelMapsResult()
@@ -257,7 +257,7 @@ def get_latlong_for_location(self, location):
 
     def store_get_latlong_for_location_to_file(self, location, filename):
         response = self._get_latlong_for_location_response(location)
-        with open(filename, "w+") as data_file:
+        with open(filename, "w+", encoding="utf-8") as data_file:
             json.dump(response, data_file, sort_keys=True, indent=2)
 
     ##################
@@ -281,7 +281,7 @@ def get_distance_between_addresses(self, origin, destination, country="UK", mode
         else:
             if logging.getLogger().isEnabledFor(logging.DEBUG):
                 logging.debug("get_distance_between_addresses - using mock file")
-            with open(self.response_file_for_get_distance_between_addresses, "r") as response_file:
+            with open(self.response_file_for_get_distance_between_addresses, "r", encoding="utf-8") as response_file:
                 response = json.load(response_file)
 
         if response['status'] == 'OK':
@@ -298,7 +298,7 @@ def get_distance_between_addresses(self, origin, destination, country="UK", mode
     def store_get_distance_between_addresses_as_file(self, origin, destination, filename, country="UK",
                                                      mode="driving", units="imperial"):
         response = self._get_distance_between_addresses(origin, destination, country, mode, units)
-        with open(filename, "w+") as data_file:
+        with open(filename, "w+", encoding="utf-8") as data_file:
             json.dump(response, data_file, sort_keys=True, indent=2)
 
     ##################
@@ -322,7 +322,7 @@ def get_directions_between_addresses(self, origin, destination, country="UK", mo
         else:
             if logging.getLogger().isEnabledFor(logging.DEBUG):
                 logging.debug("get_directions_between_addresses - using mock file")
-            with open(self.response_file_for_get_directions_between_addresses, "r") as response_file:
+            with open(self.response_file_for_get_directions_between_addresses, "r", encoding="utf-8") as response_file:
                 response = json.load(response_file)
 
         if response['status'] == 'OK':
@@ -339,5 +339,5 @@ def get_directions_between_addresses(self, origin, destination, country="UK", mo
     def store_get_directions_between_addresses_as_file(self, origin, destination, filename, country="UK",
                                                        mode="driving", units="imperial"):
         response = self._get_directions_between_addresses_response(origin, destination, country, mode, units)
-        with open(filename, "w+") as data_file:
+        with open(filename, "w+", encoding="utf-8") as data_file:
             json.dump(response, data_file, sort_keys=True, indent=2)

src/programy/utils/license/keys.py

@@ -46,7 +46,7 @@ def load_license_key_file(self, license_key_filename):
         try:
             if logging.getLogger().isEnabledFor(logging.INFO):
                 logging.info("Loading license key file: [%s]", license_key_filename)
-            with open(license_key_filename, "r") as license_file:
+            with open(license_key_filename, "r", encoding="utf-8") as license_file:
                 for line in license_file:
                     self._process_license_key_line(line)
         except Exception:

src/programy/utils/newsapi/newsapi.py

@@ -273,12 +273,12 @@ def to_json(articles):
 
     @staticmethod
     def json_to_file(filename, json_data):
-        with open(filename, 'w+') as json_file:
+        with open(filename, 'w+', encoding="utf-8") as json_file:
             json.dump(json_data, json_file)
 
     @staticmethod
     def json_from_file(filename):
-        with open(filename, 'r+') as json_file:
+        with open(filename, 'r+', encoding="utf-8") as json_file:
             return json.load(json_file)
 
     @staticmethod

src/programy/utils/weather/metoffice.py

@@ -592,7 +592,7 @@ def current_observation(self, lat, lon):
         return self.nearest_location_observation(lat, lon)
 
     def write_datapoints_to_file(self, datapoints, filename):
-        with open(filename, "w+") as data_file:
+        with open(filename, "w+", encoding="utf-8") as data_file:
             json.dump(datapoints, data_file, sort_keys=True, indent=2)
 
     def load_datapoints_from_file(self, filename):