More solid cloning / exit code 0 on no changes (#1)

.github/workflows/run-indexer.yml

@@ -20,6 +20,7 @@ jobs:
         python-version: 3.8
     - name: Create bash file for cloning
       run: |
+        mkdir cache
         python -m pip install pyyaml
         python parser.py repositories.yaml clonerepos.sh
         chmod +x clonerepos.sh
@@ -28,10 +29,12 @@ jobs:
     - name: Run indexer
       run: python indexer.py repositories.yaml
     - name: Commit files
+      continue-on-error: true # No changes = Failure
       run: |
         git config --local user.email "[email protected]"
         git config --local user.name "Indexer"
-        git add index/*.json
+        cd index
+        git add .
         git commit -m "Index update"
     - name: Push changes
       uses: ad-m/github-push-action@master

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+clonerepos.sh
+cache/

README.md

@@ -4,7 +4,5 @@ Scheduled to run every 15 minutes and provide an up-to-date [index list](https:/
 See [repositories-example.yaml](repositories-example.yaml) to learn how to compile it
 
 ### TODO:
-- Exit code 0 if no changes are committed
-- Further checks for cog package validity
 - Error log generation in index folder
 - Interface this thing with Downloader :-)

indexer.py

@@ -2,6 +2,9 @@
 import yaml
 import sys
 from pathlib import Path
+from hashlib import sha1
+
+CACHE = Path("cache")
 
 RX_PROTOCOL = 1 # This should be incremented when breaking changes to the format are implemented
 GEN_PATH = Path("index")
@@ -61,20 +64,20 @@ def folder_check_and_get_info(self):
         if self._error:
             return
 
-        path = Path(self.name)
-        if not path.is_dir():
+        base_path = CACHE / Path(sha1_digest(self._url))
+        if not base_path.is_dir():
             self._error = "Repo path does not exist. Cloning failed?"
             return
 
-        self._path = path
+        self._path = base_path
 
-        path = Path(self.name) / Path("info.json")
-        if not path.is_file():
+        infofile = base_path / Path("info.json")
+        if not infofile.is_file():
             self._error = "No repo info.json found."
             return
 
         try:
-            with open(str(path)) as f:
+            with open(str(infofile)) as f:
                 info = json.load(f)
         except:
             self._error = "Error reading repo info.json. Possibly invalid."
@@ -170,6 +173,9 @@ def get_info(self):
     def __json__(self):
         return {k:v for (k, v) in self.__dict__.items() if not k.startswith("_") and not callable(k)}
 
+def sha1_digest(url):
+    return sha1(url.encode('utf-8')).hexdigest()
+
 def main():
     yamlfile = sys.argv[1]
 

parser.py

@@ -1,5 +1,22 @@
 import yaml
 import sys
+from hashlib import sha1
+from pathlib import Path
+
+CACHE = Path("cache")
+
+def sha1_digest(url):
+    return sha1(url.encode('utf-8')).hexdigest()
+
+def get_clean_url(url):
+    branch = ""
+    if "/@" in url:
+        url.replace("/@", "@")
+    if "@" in url:
+        url, branch = url.split("@")
+    if url.endswith("/"):
+        url = url[:-1]
+    return url, branch
 
 if __name__ == "__main__":
     infile = sys.argv[1]
@@ -18,12 +35,15 @@
         repos.extend(data["unapproved"])
 
     for r in repos:
-        if "@" in r:
-            repo, branch = r.split("@")
-            branch = branch.replace("/", "") # In case of leading /
-            sh += f"git clone {repo} --branch {branch} --single-branch\n"
+        url, branch = get_clean_url(r)
+        if branch:
+            sha = sha1_digest(f"{url}@{branch}")
+            dest = CACHE / Path(sha)
+            sh += f"git clone {url} --branch {branch} --single-branch {dest}\n"
         else:
-            sh += f"git clone {r}\n"
+            sha = sha1_digest(url)
+            dest = CACHE / Path(sha)
+            sh += f"git clone {url} {dest}\n"
 
     with open(outfile, "w") as f:
         f.write(sh)