add ethos dataset

Author: pandermatt

.gitignore

@@ -131,3 +131,4 @@ dmypy.json
 application.yml
 data/result
 application.yml
+data/cache

config.py

@@ -30,6 +30,9 @@ def result_file(self, file_name):
     def input_file(self, file_name):
         return join(self.data_dir('input'), file_name)
 
+    def cache_dir(self):
+        return self.data_dir('cache')
+
     def get_env(self, var):
         return self.__get_var(var)
 

data_model/abstract_data.py

@@ -5,18 +5,34 @@
 import pandas as pd
 
 from config import config
+from util.downloader import Downloader
 
 
 class AbstractData(ABC):
     def __init__(self):
-        input_file = config.input_file(self.FILE_NAME)
-        self.data = pd.read_csv(input_file)
+        file = self._load_file()
+        self.data = self._preprocess_data(file)
+
+    def _load_file(self):
+        if self.FILE_URL:
+            input_file = Downloader().download(self.FILE_NAME, self.FILE_URL)
+        else:
+            input_file = config.input_file(self.FILE_NAME)
+        return input_file
+
+    def _preprocess_data(self, file):
+        return pd.read_csv(file)
 
     @property
     @abstractmethod
     def FILE_NAME(self):
         pass
 
+    @property
+    @abstractmethod
+    def FILE_URL(self):
+        pass
+
     @abstractmethod
     def get_data(self):
         pass

data_model/ethos_data.py

@@ -0,0 +1,18 @@
+import pandas as pd
+
+from config import config
+from data_model.abstract_data import AbstractData
+
+
+class EthosData(AbstractData):
+    FILE_NAME = 'Ethos_Dataset_Binary.csv'
+    FILE_URL = 'https://raw.githubusercontent.com/intelligence-csd-auth-gr/Ethos-Hate-Speech-Dataset/master/ethos/ethos_data/Ethos_Dataset_Binary.csv'
+
+    def _preprocess_data(self, file):
+        return pd.read_csv(file, sep=';')
+
+    def get_data(self):
+        return self.data['comment']
+
+    def get_label(self):
+        return self.data['isHate']

main.py

@@ -2,6 +2,7 @@
 
 from config import config
 from data_analysis import word_cloud_generator
+from data_model.ethos_data import EthosData
 from data_model.south_park_data import SouthParkData
 from data_model.twitter_data import TwitterData
 
@@ -19,3 +20,13 @@
     all_text = ' '.join(data[labels == 1])
     print(word_cloud_generator.generate(all_text, 'twitter_word_cloud_label_1.png'))
 
+    data = EthosData().get_data()
+    labels = EthosData().get_label()
+
+    all_text = ' '.join(data[labels == 0])
+    word_cloud_generator.generate(all_text, 'ethos_word_cloud_label_0.png')
+
+    all_text = ' '.join(data[labels == 1])
+    word_cloud_generator.generate(all_text, 'ethos_word_cloud_label_1.png')
+
+

util/cache_handler.py

@@ -0,0 +1,19 @@
+import os
+
+from config import config
+from util.logger import log
+
+
+class Downloader:
+    @staticmethod
+    def download(file_name, url, zip=False):
+        cache_dr = config.cache_dir()
+        local_path = os.path.join(cache_dr, file_name)
+        if not os.path.exists(local_path):
+            import dload
+            if zip:
+                dload.save_unzip(url, cache_dr)
+            else:
+                dload.save(url, local_path)
+            log.info(f"Downloading finished: {url}")
+        return local_path