diff --git a/.github/workflows/run-indexer.yml b/.github/workflows/run-indexer.yml index f229ea83..8b986c58 100644 --- a/.github/workflows/run-indexer.yml +++ b/.github/workflows/run-indexer.yml @@ -1,7 +1,7 @@ name: Run indexer on: - schedule: # When it's ready! + schedule: - cron: "*/15 * * * *" workflow_dispatch: branches: [ master ] diff --git a/indexer.py b/indexer.py index 519618b5..513d1f13 100644 --- a/indexer.py +++ b/indexer.py @@ -3,9 +3,10 @@ import sys from pathlib import Path +RX_PROTOCOL = 1 # This should be incremented when breaking changes to the format are implemented GEN_PATH = Path("index") -GEN_FILE = GEN_PATH / Path("1.json") # Pretty, for QA checking -GEN_MIN_FILE = GEN_PATH / Path("1-min.json") # Minified, for user download +GEN_FILE = GEN_PATH / Path(f"{RX_PROTOCOL}.json") # Pretty, for QA checking +GEN_MIN_FILE = GEN_PATH / Path(f"{RX_PROTOCOL}-min.json") # Minified, for user download class CustomEncoder(json.JSONEncoder): @@ -16,24 +17,29 @@ def default(self, obj): return json.JSONEncoder.default(self, obj) class Repo: - def __init__(self, url, category): - """Anything exposed here will be serialized later""" - self.category = category # approved / unapproved + def __init__(self, url: str, category: str): + """Anything exposed here will be serialized later + + Attributes starting with rx_ deviate from the info.json spec + and as such they have this prefix to avoid future conflicts""" + self.rx_category = category # approved / unapproved self._owner_repo = "" - self._url = url self._error = "" self._path = None - self.cogs = [] # ? + self.rx_cogs = [] self.author = [] self.description = "" self.short = "" + self._url = url + self.name = "" + self.rx_branch = "" try: - self.name, self.branch = self.get_name_and_branch(url) + self.parse_name_branch_url(url) except: self._error = ("Something went wrong while parsing the url. " "Is it a valid address?") - def get_name_and_branch(self, url): + def parse_name_branch_url(self, url): branch = "" name = url.split("/")[4] # Owner/RepoName will be useful when it's time to exclude cogs @@ -42,7 +48,14 @@ def get_name_and_branch(self, url): name, branch = name.split("@") if name.endswith("/"): name = name[:-1] - return name, branch + + url = url.replace("/@", "@") + if url.endswith("/"): + url = url[:-1] + + self.name = name + self.rx_branch = branch + self._url = url def folder_check_and_get_info(self): if self._error: @@ -52,51 +65,56 @@ def folder_check_and_get_info(self): if not path.is_dir(): self._error = "Repo path does not exist. Cloning failed?" return - + self._path = path - + path = Path(self.name) / Path("info.json") if not path.is_file(): self._error = "No repo info.json found." return - + try: with open(str(path)) as f: info = json.load(f) except: self._error = "Error reading repo info.json. Possibly invalid." return - + self.author = info.get("author", []) self.description = info.get("description", "") self.short = info.get("short", "") - + def populate_cogs(self): if self._error: return sub_dirs = [p for p in self._path.iterdir() if p.is_dir() and not p.name.startswith(".")] - + for d in sub_dirs: path = d / Path("info.json") if path.is_file(): # Dirs with no info.json inside are simply ignored - self.cogs.append(Cog(d.name, d)) - - if not self.cogs: + self.rx_cogs.append(Cog(d.name, d)) + + if not self.rx_cogs: self._error = "Repo contains no valid cogs" - + def process_cogs(self): if self._error: return - - for cog in self.cogs: + + for cog in self.rx_cogs: cog.get_info() + cog.check_cog_validity() def __json__(self): - return {k:v for (k, v) in self.__dict__.items() if not k.startswith("_") and not callable(k)} + _dict = {k:v for (k, v) in self.__dict__.items() if not k.startswith("_") and not callable(k)} + return {self._url: _dict} class Cog: - def __init__(self, name, path): - """Anything exposed here will be serialized later""" + def __init__(self, name: str, path: Path): + """Anything exposed here will be serialized later + + Attributes starting with rx_ deviate from the info.json spec + and as such they have this prefix to avoid future conflicts""" self._name = name self._path = path self.min_bot_version = "" @@ -110,7 +128,17 @@ def __init__(self, name, path): self.type = "" # Still a thing? self._error = "" + def check_cog_validity(self): + if self._error: + return + + initpath = self._path / Path("__init__.py") + if not initpath.exists(): + self._error = "Info.json is present but no __init__.py was found. Invalid cog package." + def get_info(self): + if self._error: + return info_path = self._path / Path("info.json") try: @@ -119,7 +147,7 @@ def get_info(self): except: self._error = "Error reading cog info.json. Possibly invalid." return - + self.min_bot_version = data.get("min_bot_version", "") self.max_bot_version = data.get("max_bot_version", "") self.min_python_version = data.get("min_python_version", "") @@ -129,11 +157,11 @@ def get_info(self): self.requirements = data.get("requirements", []) self.tags = data.get("tags", []) self.type = data.get("type", "") - + def __json__(self): _dict = {k:v for (k, v) in self.__dict__.items() if not k.startswith("_") and not callable(k)} return {self._name: _dict} - + def main(): yamlfile = sys.argv[1] @@ -146,18 +174,18 @@ def main(): if data[k]: # Can be None if empty for url in data[k]: repos.append(Repo(url, k)) - + for r in repos: r.folder_check_and_get_info() r.populate_cogs() r.process_cogs() - + # Remove errored repos and cogs. TODO: Write an error log for QA repos = [r for r in repos if not r._error] - + for r in repos: - r.cogs = [c for c in r.cogs if not c._error] - + r.rx_cogs = [c for c in r.rx_cogs if not c._error] + if data["flagged-cogs"]: for url, flagged_cogs in data["flagged-cogs"].items(): for r in repos: @@ -166,12 +194,12 @@ def main(): if r._owner_repo not in url: continue to_remove = [] - for c in r.cogs: + for c in r.rx_cogs: if c._name in flagged_cogs: to_remove.append(c) if to_remove: - r.cogs = [c for c in r.cogs if c not in to_remove] - + r.rx_cogs = [c for c in r.rx_cogs if c not in to_remove] + if repos: if not GEN_PATH.exists(): GEN_PATH.mkdir() diff --git a/repositories-example.yaml b/repositories-example.yaml index d90b955d..2966cf86 100644 --- a/repositories-example.yaml +++ b/repositories-example.yaml @@ -11,7 +11,7 @@ approved: - https://github.com/User2/RepoName@branchname - https://github.com/User3/RepoName -# List of approved repos. +# List of unapproved repos. # Will be parsed and categorized as "unapproved" by the indexer unapproved: - https://github.com/User4/RepoName@branchname diff --git a/repositories.yaml b/repositories.yaml index a3841250..2057ae90 100644 --- a/repositories.yaml +++ b/repositories.yaml @@ -40,7 +40,7 @@ approved: - https://github.com/grayconcaves/FanCogs - https://github.com/flapjax/FlapJack-Cogs -# List of approved repos. +# List of unapproved repos. # Will be parsed and categorized as "unapproved" by the indexer unapproved: - https://github.com/baiumbg/baiumbg-Cogs