Add a Makefile to the repo to allow easier local usage (#87)

.github/workflows/run-indexer.yml

@@ -22,14 +22,11 @@ jobs:
         python-version: 3.8
     - name: Create bash file for cloning
       run: |
-        mkdir cache
-        python -m pip install pyyaml
-        python parser.py repositories.yaml clonerepos.sh
-        chmod +x clonerepos.sh git-retry.sh
+        make clonerepos.sh
     - name: Clone repos
-      run: ./clonerepos.sh
+      run: make clone
     - name: Run indexer
-      run: python indexer.py repositories.yaml
+      run: make index
     - name: Commit files
       continue-on-error: true # No changes = Failure
       run: |

Makefile

@@ -0,0 +1,29 @@
+# This Makefile supports incremental builds, please don't break that if you make any changes.
+
+# user-facing targets
+
+all: clone index
+
+clone: cleancache cache
+
+index: cache
+	.venv/bin/python indexer.py repositories.yaml
+
+clean: cleancache
+	-rm -rf clonerepos.sh .venv
+
+cleancache:
+	-rm -rf cache
+
+# other targets
+
+clonerepos.sh: .venv repositories.yaml
+	.venv/bin/python parser.py repositories.yaml clonerepos.sh
+
+cache: clonerepos.sh
+	-rm -rf cache
+	./clonerepos.sh
+
+.venv:
+	python3 -m venv .venv
+	.venv/bin/pip install pyyaml

parser.py

@@ -1,10 +1,15 @@
-import yaml
+import os
 import sys
 from hashlib import sha1
 from pathlib import Path
 
+import yaml
+
 CACHE = Path("cache")
 
+def executable_opener(path, flags):
+    return os.open(path, flags, 0o755)
+
 def sha1_digest(url):
     return sha1(url.encode('utf-8')).hexdigest()
 
@@ -23,7 +28,7 @@ def get_clean_url(url):
     with open(infile) as f:
         data = yaml.safe_load(f.read())
 
-    sh = ""
+    sh = "mkdir -p cache\n"
 
     repos = []
 
@@ -43,5 +48,5 @@ def get_clean_url(url):
             dest = CACHE / Path(sha)
             sh += f"./git-retry.sh clone --depth=1 {url} {dest}\n"
 
-    with open(outfile, "w") as f:
+    with open(outfile, "w", opener=executable_opener) as f:
         f.write(sh)