Commit for venv stuff

FlashpointProject · Mar 20, 2021 · 39763f1 · 39763f1
1 parent b51f59c
commit 39763f1
Show file tree

Hide file tree

Showing 6 changed files with 109 additions and 73 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,22 @@
-urls.txt
-errors.txt
+# Don't include compiled python stuff
 __pycache__
 *.pyc
+
+# Don't include build and dist data and files
+fpcurator.spec
 build
+build-all.py
 build.bat
 dist
+
+# Don't include python venv data
+Include
+Lib
+Scripts
+pyvenv.cfg
+
+# Don't include generated files from fpcurator
 data.json
+errors.txt
 search
 output
diff --git a/fpcurator.py b/fpcurator.py
@@ -29,6 +29,21 @@
 
 from tooltip import Tooltip
 
+MONTHS = {
+    "Jan": "01",
+    "Feb": "02",
+    "Mar": "03",
+    "Apr": "04",
+    "May": "05",
+    "Jun": "06",
+    "Jul": "07", 
+    "Aug": "08",
+    "Sep": "09",
+    "Oct": "10",
+    "Nov": "11",
+    "Dec": "12"
+}
+
 HELP_HTML = """<!doctype html>
 <html><body>
     <h1><center>Help</center></h1>
@@ -618,14 +633,19 @@ def __init__(self):
         # Create checkboxes
         cframe = tk.Frame(self, bg="white")
         cframe.pack(padx=5)
+        c2frame = tk.Frame(self, bg="white")
+        c2frame.pack(padx=5, pady=5)
 
-        self.original = tk.BooleanVar()
-        self.original.set(True)
         self.keep_vars = tk.BooleanVar()
         self.clear = tk.BooleanVar()
         self.clear.set(True)
         self.show_done = tk.BooleanVar()
         self.show_done.set(True)
+
+        self.original = tk.BooleanVar()
+        self.original.set(True)
+        self.replace_https = tk.BooleanVar()
+        self.replace_https.set(True)
 
         original = tk.Checkbutton(cframe, bg="white", text='Delete "web.archive.org"', var=self.original)
         original.pack(side="left")

diff --git a/fpcurator.spec b/fpcurator.spec
diff --git a/sites/AddictingGames.py b/sites/AddictingGames.py
@@ -3,22 +3,9 @@
 from __main__ import fpclib
 from __main__ import re
 
-regex = 'addictinggames.com'
+from __main__ import MONTHS
 
-MONTHS = {
-    "Jan": "01",
-    "Feb": "02",
-    "Mar": "03",
-    "Apr": "04",
-    "May": "05",
-    "Jun": "06",
-    "Jul": "07", 
-    "Aug": "08",
-    "Sep": "09",
-    "Oct": "10",
-    "Nov": "11",
-    "Dec": "12"
-}
+regex = 'addictinggames.com'
 
 TAGS = {
     "Shooting": "Action; Shooter",
@@ -37,6 +24,11 @@
     "Clicker": "Arcade; Clicker"
 }
 
+HTML_EMBED = """<body>
+    <iframe width="100%" height="100%" src="%s"></iframe>
+</body>
+"""
+
 DATA_PARSER = re.compile("type: '(.*?)',\s*source: '(.*?)'")
 MARKUP_MOVIE = re.compile('<param *name="movie" *value="(.*?)"')
 
@@ -76,7 +68,8 @@ def parse(self, soup):
             # This is an HTML5 game
             self.platform = "HTML5"
             self.app = fpclib.BASILISK
-            self.if_url = url
+            self.if_url = fpclib.normalize(if_url, keep_vars=True)
+            self.if_file = fpclib.normalize(if_url)
             self.cmd = fpclib.normalize(self.src)
         elif data[1] == "markup":
             # Markup games are special
@@ -99,10 +92,12 @@ def soupify(self):
 
     def get_files(self):
         if self.platform == "HTML5":
-            # Create html file for game
-            fpclib.write(self.cmd[7:], fpclib.read_url(self.if_url))
+            # Download iframe file
+            fpclib.download_all((self.if_url,))
             # Replace all references to https with http
-            fpclib.replace(self.cmd[7:], "https:", "http:")
+            fpclib.replace(self.if_file[7:], "https:", "http:")
+            # Create html file for game
+            fpclib.write(self.cmd[7:], HTML_EMBED % self.if_file)
         else:
             # Flash games are downloaded normally
             super().get_files()

diff --git a/sites/Kongregate.py b/sites/Kongregate.py
@@ -2,22 +2,9 @@
 from __main__ import fpclib
 from __main__ import bs4, re, urllib, uuid
 
-regex = 'kongregate.com'
+from __main__ import MONTHS
 
-MONTHS = {
-    "Jan": "01",
-    "Feb": "02",
-    "Mar": "03",
-    "Apr": "04",
-    "May": "05",
-    "Jun": "06",
-    "Jul": "07", 
-    "Aug": "08",
-    "Sep": "09",
-    "Oct": "10",
-    "Nov": "11",
-    "Dec": "12"
-}
+regex = 'kongregate.com'
 
 IF_URL = re.compile('[\'"]iframe_url[\'"]:[\'"](.*?)[\'"]')
 SWF_URL = re.compile('[\'"]swfurl[\'"]:[\'"](.*?)[\'"]')
@@ -100,16 +87,17 @@ def parse(self, soup):
             self.app = fpclib.BASILISK
             self.cmd = fpclib.normalize(self.src)
             self.if_url = fpclib.normalize(if_url, keep_vars=True)
+            self.if_file = fpclib.normalize(if_url)
             self.size = SIZE.search(if_script)
 
     def get_files(self):
         if self.platform == "HTML5":
             # Download iframe that ought to be embedded
             fpclib.download_all((self.if_url,))
             # Replace all references to https with http
-            fpclib.replace(self.if_url[7:], "https:", "http:")
+            fpclib.replace(self.if_file[7:], "https:", "http:")
             # Create file to embed swf
-            fpclib.write(self.cmd[7:], HTML_EMBED % (self.title, self.size[1], self.size[2], fpclib.normalize(self.if_url, keep_vars=True)))
+            fpclib.write(self.cmd[7:], HTML_EMBED % (self.title, self.size[1], self.size[2], self.if_file))
         else:
             # Flash games are downloaded normally
             super().get_files()

diff --git a/sites/Miniclip.py b/sites/Miniclip.py
@@ -0,0 +1,54 @@
+# Miniclip definition. Only supports HTML5.
+from __main__ import fpclib
+from __main__ import re, json
+
+regex = 'miniclip.com'
+
+HTML_EMBED = """<body>
+    <iframe width="100%" height="100%" src="%s"></iframe>
+</body>
+"""
+
+GAME_URL = re.compile('game_url: "(.*?)"')
+
+class Miniclip(fpclib.Curation):
+    def parse(self, soup):
+        # Base everything off of application data, which is hopefully more stable than the webpage format
+        data = json.loads(soup.select_one("#jsonLdSchema").string)
+
+        self.title = data["name"]
+        self.logo = data["image"]
+        self.pub = "Miniclip"
+        # This theoretically could be used to generate more accurate tags, but whatever
+        self.tags = data["genre"]
+        self.date = data["datePublished"][:10]
+
+        # Get Description
+        try: self.desc = soup.select_one(".game-description").text.strip()
+        except: pass
+
+        # Only HTML5 is supported
+        self.platform = "HTML5"
+        self.app = fpclib.BASILISK
+
+        # Get file for Launch Command
+        url = GAME_URL.search(soup.select(".game-embed-wrapper > script")[1].string)[1]
+        if url[0] == "/" and url[1] != "/":
+            url = "http://www.miniclip.com" + url
+        self.if_url = fpclib.normalize(url)
+
+        self.cmd = fpclib.normalize(self.src)
+
+    def get_files(self):
+        # Download iframe that ought to be embedded
+        fpclib.download_all((self.if_url,))
+        # Replace all references to https with http
+        fpclib.replace(self.if_url[7:], "https:", "http:")
+        # Create file to embed swf
+        fpclib.write(self.cmd[7:], HTML_EMBED % self.if_url))
+
+    def save_image(self, url, file_name):
+        # Surround save image with a try catch loop as some logos cannot be gotten.
+        try:
+            fpclib.download_image(url, name=file_name)
+        except: pass