Expose url, stronger cog package validity check

.github/workflows/run-indexer.yml

@@ -1,7 +1,7 @@
 name: Run indexer
 
 on:
-  schedule: # When it's ready!
+  schedule:
   - cron: "*/15 * * * *"
   workflow_dispatch:
     branches: [ master ]

indexer.py

@@ -3,9 +3,10 @@
 import sys
 from pathlib import Path
 
+RX_PROTOCOL = 1 # This should be incremented when breaking changes to the format are implemented
 GEN_PATH = Path("index")
-GEN_FILE = GEN_PATH / Path("1.json") # Pretty, for QA checking
-GEN_MIN_FILE = GEN_PATH / Path("1-min.json") # Minified, for user download
+GEN_FILE = GEN_PATH / Path(f"{RX_PROTOCOL}.json") # Pretty, for QA checking
+GEN_MIN_FILE = GEN_PATH / Path(f"{RX_PROTOCOL}-min.json") # Minified, for user download
 
 
 class CustomEncoder(json.JSONEncoder):
@@ -16,24 +17,29 @@ def default(self, obj):
             return json.JSONEncoder.default(self, obj)
 
 class Repo:
-    def __init__(self, url, category):
-        """Anything exposed here will be serialized later"""
-        self.category = category # approved / unapproved
+    def __init__(self, url: str, category: str):
+        """Anything exposed here will be serialized later
+
+        Attributes starting with rx_ deviate from the info.json spec
+        and as such they have this prefix to avoid future conflicts"""
+        self.rx_category = category # approved / unapproved
         self._owner_repo = ""
-        self._url = url
         self._error = ""
         self._path = None
-        self.cogs = [] # ?
+        self.rx_cogs = []
         self.author = []
         self.description = ""
         self.short = ""
+        self._url = url
+        self.name = ""
+        self.rx_branch = ""
         try:
-            self.name, self.branch = self.get_name_and_branch(url)
+            self.parse_name_branch_url(url)
         except:
             self._error = ("Something went wrong while parsing the url. "
                             "Is it a valid address?")
 
-    def get_name_and_branch(self, url):
+    def parse_name_branch_url(self, url):
         branch = ""
         name = url.split("/")[4]
         # Owner/RepoName will be useful when it's time to exclude cogs
@@ -42,7 +48,14 @@ def get_name_and_branch(self, url):
             name, branch = name.split("@")
         if name.endswith("/"):
             name = name[:-1]
-        return name, branch
+
+        url = url.replace("/@", "@")
+        if url.endswith("/"):
+            url = url[:-1]
+
+        self.name = name
+        self.rx_branch = branch
+        self._url = url
 
     def folder_check_and_get_info(self):
         if self._error:
@@ -52,51 +65,56 @@ def folder_check_and_get_info(self):
         if not path.is_dir():
             self._error = "Repo path does not exist. Cloning failed?"
             return
-        
+
         self._path = path
-        
+
         path = Path(self.name) / Path("info.json")
         if not path.is_file():
             self._error = "No repo info.json found."
             return
-        
+
         try:
             with open(str(path)) as f:
                 info = json.load(f)
         except:
             self._error = "Error reading repo info.json. Possibly invalid."
             return
-        
+
         self.author = info.get("author", [])
         self.description = info.get("description", "")
         self.short = info.get("short", "")
-    
+
     def populate_cogs(self):
         if self._error:
             return
         sub_dirs = [p for p in self._path.iterdir() if p.is_dir() and not p.name.startswith(".")]
-        
+
         for d in sub_dirs:
             path = d / Path("info.json")
             if path.is_file(): # Dirs with no info.json inside are simply ignored
-                self.cogs.append(Cog(d.name, d))
-        
-        if not self.cogs:
+                self.rx_cogs.append(Cog(d.name, d))
+
+        if not self.rx_cogs:
             self._error = "Repo contains no valid cogs"
-    
+
     def process_cogs(self):
         if self._error:
             return
-        
-        for cog in self.cogs:
+
+        for cog in self.rx_cogs:
             cog.get_info()
+            cog.check_cog_validity()
 
     def __json__(self):
-        return {k:v for (k, v) in self.__dict__.items() if not k.startswith("_") and not callable(k)}
+        _dict = {k:v for (k, v) in self.__dict__.items() if not k.startswith("_") and not callable(k)}
+        return {self._url: _dict}
 
 class Cog:
-    def __init__(self, name, path):
-        """Anything exposed here will be serialized later"""
+    def __init__(self, name: str, path: Path):
+        """Anything exposed here will be serialized later
+
+        Attributes starting with rx_ deviate from the info.json spec
+        and as such they have this prefix to avoid future conflicts"""
         self._name = name
         self._path = path
         self.min_bot_version = ""
@@ -110,7 +128,17 @@ def __init__(self, name, path):
         self.type = "" # Still a thing?
         self._error = ""
 
+    def check_cog_validity(self):
+        if self._error:
+            return
+
+        initpath = self._path / Path("__init__.py")
+        if not initpath.exists():
+            self._error = "Info.json is present but no __init__.py was found. Invalid cog package."
+
     def get_info(self):
+        if self._error:
+            return
         info_path = self._path / Path("info.json")
 
         try:
@@ -119,7 +147,7 @@ def get_info(self):
         except:
             self._error = "Error reading cog info.json. Possibly invalid."
             return
-        
+
         self.min_bot_version = data.get("min_bot_version", "")
         self.max_bot_version = data.get("max_bot_version", "")
         self.min_python_version = data.get("min_python_version", "")
@@ -129,11 +157,11 @@ def get_info(self):
         self.requirements = data.get("requirements", [])
         self.tags = data.get("tags", [])
         self.type = data.get("type", "")
-    
+
     def __json__(self):
         _dict = {k:v for (k, v) in self.__dict__.items() if not k.startswith("_") and not callable(k)}
         return {self._name: _dict}
-    
+
 def main():
     yamlfile = sys.argv[1]
 
@@ -146,18 +174,18 @@ def main():
         if data[k]: # Can be None if empty
             for url in data[k]:
                 repos.append(Repo(url, k))
-    
+
     for r in repos:
         r.folder_check_and_get_info()
         r.populate_cogs()
         r.process_cogs()
-    
+
     # Remove errored repos and cogs. TODO: Write an error log for QA
     repos = [r for r in repos if not r._error]
-    
+
     for r in repos:
-        r.cogs = [c for c in r.cogs if not c._error]
-    
+        r.rx_cogs = [c for c in r.rx_cogs if not c._error]
+
     if data["flagged-cogs"]:
         for url, flagged_cogs in data["flagged-cogs"].items():
             for r in repos:
@@ -166,12 +194,12 @@ def main():
                 if r._owner_repo not in url:
                     continue
                 to_remove = []
-                for c in r.cogs:
+                for c in r.rx_cogs:
                     if c._name in flagged_cogs:
                         to_remove.append(c)
                 if to_remove:
-                    r.cogs = [c for c in r.cogs if c not in to_remove]
-    
+                    r.rx_cogs = [c for c in r.rx_cogs if c not in to_remove]
+
     if repos:
         if not GEN_PATH.exists():
             GEN_PATH.mkdir()

repositories-example.yaml

@@ -11,7 +11,7 @@ approved:
   - https://github.com/User2/RepoName@branchname
   - https://github.com/User3/RepoName
 
-# List of approved repos.
+# List of unapproved repos.
 # Will be parsed and categorized as "unapproved" by the indexer
 unapproved:
   - https://github.com/User4/RepoName@branchname

repositories.yaml

@@ -40,7 +40,7 @@ approved:
   - https://github.com/grayconcaves/FanCogs
   - https://github.com/flapjax/FlapJack-Cogs
 
-# List of approved repos.
+# List of unapproved repos.
 # Will be parsed and categorized as "unapproved" by the indexer
 unapproved:
   - https://github.com/baiumbg/baiumbg-Cogs