scripts: add extractor and filtering tools

Author: stefan-it

extractor.py

@@ -0,0 +1,41 @@
+import sys
+import time
+import tarfile
+
+from ast import literal_eval
+
+filename = sys.argv[1]
+
+out_filename = filename.replace(".tar.gz", ".txt")
+
+out_f_p = open(out_filename, "wt")
+
+start_time = time.time()
+
+with tarfile.open(filename, "r:gz") as f_p:
+    members = f_p.getmembers()
+
+    print("Extracting:", members)
+
+    assert len(members) == 1
+
+    extracted_file = f_p.extractfile(members[0])
+
+    file_content = extracted_file.read().decode('utf-8')
+
+    file_content = file_content.replace("}{'url'", "}\n{'url'")
+
+    for line in file_content.split("\n"):
+        try:
+            data = literal_eval(line)
+
+            language_score = data["language_score"]
+
+            if language_score > 0.98:
+                out_f_p.write(line + "\n")
+        except:
+            continue
+
+print("Extraction took:", round(time.time() - start_time, 2), "seconds.")
+
+out_f_p.close()

filtering.py

@@ -0,0 +1,48 @@
+import sys
+import time
+
+from ast import literal_eval
+
+filename = sys.argv[1]
+
+out_filename = filename.replace(".txt", "_filtered.txt")
+
+out_f_p = open(out_filename, "wt")
+
+start_time = time.time()
+
+with open(filename, "rt") as f_p:
+    for line in f_p:
+        try:
+            data = literal_eval(line)
+
+            date_download = data["date_download"]
+            digest = data["digest"]
+
+            raw_content = data["raw_content"]
+
+            # Some filtering rules
+            if "<" in raw_content:
+                continue
+
+            if ">" in raw_content:
+                continue
+
+            if "http:" in raw_content:
+                continue
+
+            if "https:" in raw_content:
+                continue
+
+            if "�" in raw_content:
+                continue
+
+            out_f_p.write(raw_content + "\n")
+
+        except Exception as e:
+            print(e)
+            continue
+
+print("Filtering for", filename, "took:", round(time.time() - start_time, 2), "seconds.")
+
+out_f_p.close()