diff --git a/.github/workflows/run-indexer.yml b/.github/workflows/run-indexer.yml index 2be805be..ddb364c7 100644 --- a/.github/workflows/run-indexer.yml +++ b/.github/workflows/run-indexer.yml @@ -22,14 +22,11 @@ jobs: python-version: 3.8 - name: Create bash file for cloning run: | - mkdir cache - python -m pip install pyyaml - python parser.py repositories.yaml clonerepos.sh - chmod +x clonerepos.sh git-retry.sh + make clonerepos.sh - name: Clone repos - run: ./clonerepos.sh + run: make clone - name: Run indexer - run: python indexer.py repositories.yaml + run: make index - name: Commit files continue-on-error: true # No changes = Failure run: | diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..ff6b5a0a --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +# This Makefile supports incremental builds, please don't break that if you make any changes. + +# user-facing targets + +all: clone index + +clone: cleancache cache + +index: cache + .venv/bin/python indexer.py repositories.yaml + +clean: cleancache + -rm -rf clonerepos.sh .venv + +cleancache: + -rm -rf cache + +# other targets + +clonerepos.sh: .venv repositories.yaml + .venv/bin/python parser.py repositories.yaml clonerepos.sh + +cache: clonerepos.sh + -rm -rf cache + ./clonerepos.sh + +.venv: + python3 -m venv .venv + .venv/bin/pip install pyyaml diff --git a/parser.py b/parser.py index 8538b7c8..01c1f43e 100644 --- a/parser.py +++ b/parser.py @@ -1,10 +1,15 @@ -import yaml +import os import sys from hashlib import sha1 from pathlib import Path +import yaml + CACHE = Path("cache") +def executable_opener(path, flags): + return os.open(path, flags, 0o755) + def sha1_digest(url): return sha1(url.encode('utf-8')).hexdigest() @@ -23,7 +28,7 @@ def get_clean_url(url): with open(infile) as f: data = yaml.safe_load(f.read()) - sh = "" + sh = "mkdir -p cache\n" repos = [] @@ -43,5 +48,5 @@ def get_clean_url(url): dest = CACHE / Path(sha) sh += f"./git-retry.sh clone --depth=1 {url} {dest}\n" - with open(outfile, "w") as f: + with open(outfile, "w", opener=executable_opener) as f: f.write(sh)