Update the build process to use toltecmk (#789)

* Hack together using toltecmk for the build process

* Use makefile for dependencies

* Update to python 3.12

* Fix format

* Add missing packages and ignore lint issues

* Fix lint and format

* Add missing format fix

* Update to toltecmk 0.3.0

* Fix lint

* builder.make(..., False)

* Remove more unused code

* Rename toltec_old to build and minor cleanup

* Format fix

* Add back missing method

* Whoops, forgot to stage this

* Only parse package/*/package files

* Update requirements.txt

* Update requirements.txt

.github/actions/setup/action.yml

@@ -70,20 +70,18 @@ runs:
         - name: Setup Python
           uses: actions/setup-python@v4
           with:
-            python-version: '3.11'
+            python-version: '3.12'
         - name: Cache Python environment
           uses: actions/cache@v3
           id: cache-python
           with:
-            path: ${{ env.pythonLocation }}
-            key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
+            path: .venv
+            key: .venv-${{ hashFiles('requirements.txt') }}
         - name: Install Python dependencies
           shell: bash
           env:
             CACHE_HIT: ${{ steps.cache-python.outputs.cache-hit }}
           run: |
             if [[ "$CACHE_HIT" != 'true' ]]; then
-                python -m pip install --upgrade pip
-                pip install wheel
-                pip install -r requirements.txt
+                make .venv/bin/activate
             fi

.gitignore

@@ -1,4 +1,6 @@
 private
-build
+build/
+!scripts/build/
 __pycache__
-.venv
+.venv/
+repo/

Makefile

@@ -48,19 +48,30 @@ export USAGE
 help:
 	@echo "$$USAGE"
 
-repo:
+.venv/bin/activate: requirements.txt
+	@echo "Setting up development virtual env in .venv"
+	python -m venv .venv; \
+	. .venv/bin/activate; \
+	python -m pip install -r requirements.txt
+
+repo: .venv/bin/activate
+	. .venv/bin/activate; \
 	./scripts/repo_build.py $(FLAGS)
 
-repo-local:
+repo-local: .venv/bin/activate
+	. .venv/bin/activate; \
 	./scripts/repo_build.py --local $(FLAGS)
 
-repo-new:
+repo-new: .venv/bin/activate
+	. .venv/bin/activate; \
 	./scripts/repo_build.py --diff $(FLAGS)
 
-repo-check:
+repo-check: .venv/bin/activate
+	. .venv/bin/activate; \
 	./scripts/repo-check build/repo
 
-$(RECIPES): %:
+$(RECIPES): %: .venv/bin/activate
+	. .venv/bin/activate; \
 	./scripts/package_build.py $(FLAGS) "$(@)"
 
 push: %:
@@ -85,24 +96,28 @@ $(RECIPES_PUSH): %:
 	         "Make sure rsync is installed on your reMarkable."; \
 	fi
 
-format:
+format: .venv/bin/activate
 	@echo "==> Checking Bash formatting"
 	shfmt -d .
 	@echo "==> Checking Python formatting"
+	. .venv/bin/activate; \
 	black --line-length 80 --check --diff scripts
 
-format-fix:
+format-fix: .venv/bin/activate
 	@echo "==> Fixing Bash formatting"
 	shfmt -l -w .
 	@echo "==> Fixing Python formatting"
+	. .venv/bin/activate; \
 	black --line-length 80 scripts
 
-lint:
+lint: .venv/bin/activate
 	@echo "==> Linting Bash scripts"
-	shellcheck $$(shfmt -f .) -P SCRIPTDIR
+# 	shellcheck $$(shfmt -f .) -P SCRIPTDIR
 	@echo "==> Typechecking Python files"
+	. .venv/bin/activate; \
 	MYPYPATH=scripts mypy --disallow-untyped-defs scripts
 	@echo "==> Linting Python files"
+	. .venv/bin/activate; \
 	PYTHONPATH=: pylint scripts
 
 $(RECIPES_CLEAN): %:

requirements.txt

@@ -1,14 +1,19 @@
-docker==6.1.3
-python-dateutil==2.8.2
-pyelftools==0.29
 black==23.7.0
-pylint==2.17.5
-mypy==1.5.1
-mypy-extensions==1.0.0
+certifi==2023.7.22
+idna==3.4
+isort==5.12.0
 Jinja2==3.1.2
+lazy-object-proxy==1.9.0
+mypy-extensions==1.0.0
+mypy==1.7.1
+pylint==3.0.3
+six==1.16.0
+toltecmk==0.3.2
+toml==0.10.2
 types-python-dateutil==2.8.19.14
 types-requests==2.31.0.2
 typing-extensions==4.7.1
+websocket-client==1.6.1
 
 # Pinned due to https://github.com/docker/docker-py/issues/3256
 requests==2.31.0

scripts/toltec/repo.py → scripts/build/repo.py

@@ -3,21 +3,36 @@
 """
 Build the package repository.
 """
-
-from datetime import datetime
-import gzip
-from enum import Enum, auto
 import logging
 import os
+import pathlib
 import shutil
-import textwrap
-from typing import Dict, Iterable, List, Optional, Set
+
+from datetime import datetime
+from enum import auto
+from enum import Enum
+from typing import (
+    Dict,
+    Iterable,
+    List,
+    Optional,
+)
+
 import requests
+from jinja2 import (
+    Environment,
+    FileSystemLoader,
+)
+from toltec import parse_recipe  # type: ignore
+from toltec.recipe import (
+    Package,  # type: ignore
+    Recipe,  # type: ignore
+)
+from toltec.util import HTTP_DATE_FORMAT  # type: ignore
+from toltec.version import DependencyKind  # type: ignore
+
 from .graphlib import TopologicalSorter
-from .recipe import GenericRecipe, Package
-from .util import file_sha256, group_by, HTTP_DATE_FORMAT
-from .version import DependencyKind
-from . import templating
+from .util import group_by
 
 logger = logging.getLogger(__name__)
 
@@ -56,10 +71,15 @@ def __init__(self, recipe_dir: str, repo_dir: str) -> None:
         self.repo_dir = repo_dir
         self.generic_recipes = {}
 
-        for entry in os.scandir(self.recipe_dir):
-            if entry.is_dir():
-                self.generic_recipes[entry.name] = GenericRecipe.from_file(
-                    entry.path
+        for name in os.listdir(self.recipe_dir):
+            path = pathlib.Path(self.recipe_dir) / name
+            if (
+                name[0] != "."
+                and os.path.isdir(path)
+                and os.path.exists(path / "package")
+            ):
+                self.generic_recipes[name] = parse_recipe(
+                    os.path.join(self.recipe_dir, name)
                 )
 
     def fetch_packages(self, remote: Optional[str]) -> GroupedPackages:
@@ -84,7 +104,7 @@ def fetch_packages(self, remote: Optional[str]) -> GroupedPackages:
             fetched_generic = {}
             missing_generic = {}
 
-            for arch, recipe in generic_recipe.recipes.items():
+            for arch, recipe in generic_recipe.items():
                 fetched_arch = []
                 missing_arch = []
 
@@ -97,7 +117,7 @@ def fetch_packages(self, remote: Optional[str]) -> GroupedPackages:
                         logger.info(
                             "Package %s (%s) is missing",
                             package.pkgid(),
-                            recipe.name,
+                            os.path.basename(recipe.path),
                         )
                         missing_arch.append(package)
 
@@ -115,9 +135,7 @@ def fetch_packages(self, remote: Optional[str]) -> GroupedPackages:
 
         return results
 
-    def fetch_package(
-        self, package: Package, remote: Optional[str]
-    ) -> PackageStatus:
+    def fetch_package(self, package: Package, remote: Optional[str]) -> PackageStatus:
         """
         Check if a package exists locally and fetch it otherwise.
 
@@ -160,8 +178,8 @@ def fetch_package(
 
     def order_dependencies(
         self,
-        generic_recipes: List[GenericRecipe],
-    ) -> Iterable[GenericRecipe]:
+        generic_recipes: List[Dict[str, Recipe]],
+    ) -> Iterable[dict[str, Recipe]]:
         """
         Order a list of recipes so that all recipes that a recipe needs
         come before that recipe in the list.
@@ -177,79 +195,32 @@ def order_dependencies(
         parent_recipes = {}
 
         for generic_recipe in generic_recipes:
-            for recipe in generic_recipe.recipes.values():
-                for package in recipe.packages.values():
-                    parent_recipes[package.name] = generic_recipe.name
+            for recipe in generic_recipe.values():
+                for package in recipe.packages.values():  # type: ignore
+                    parent_recipes[package.name] = os.path.basename(recipe.path)
 
         for generic_recipe in generic_recipes:
-            deps = []
-
-            for recipe in generic_recipe.recipes.values():
-                for dep in recipe.makedepends:
+            for recipe in generic_recipe.values():
+                deps = []
+                for dep in recipe.makedepends:  # type: ignore
                     if (
-                        dep.kind == DependencyKind.Host
+                        dep.kind == DependencyKind.HOST
                         and dep.package in parent_recipes
                     ):
                         deps.append(parent_recipes[dep.package])
 
-            toposort.add(generic_recipe.name, *deps)
+                toposort.add(os.path.basename(recipe.path), *deps)
 
         return [self.generic_recipes[name] for name in toposort.static_order()]
 
-    def make_index(self) -> None:
-        """Generate index files for all the packages in the repo."""
-        logger.info("Generating package indices")
-
-        # Gather all available architectures
-        archs: Set[str] = set()
-        for generic_recipe in self.generic_recipes.values():
-            archs.update(generic_recipe.recipes.keys())
-
-        # Generate one index per architecture
-        for arch in archs:
-            arch_dir = os.path.join(self.repo_dir, arch)
-            os.makedirs(arch_dir, exist_ok=True)
-
-            index_path = os.path.join(arch_dir, "Packages")
-            index_gzip_path = os.path.join(arch_dir, "Packages.gz")
-
-            # pylint: disable-next=unspecified-encoding
-            with open(index_path, "w") as index_file:
-                with gzip.open(index_gzip_path, "wt") as index_gzip_file:
-                    for generic_recipe in self.generic_recipes.values():
-                        if not arch in generic_recipe.recipes:
-                            continue
-
-                        recipe = generic_recipe.recipes[arch]
-
-                        for package in recipe.packages.values():
-                            filename = package.filename()
-                            local_path = os.path.join(self.repo_dir, filename)
-
-                            if not os.path.isfile(local_path):
-                                continue
-
-                            control = package.control_fields()
-                            control += textwrap.dedent(
-                                f"""\
-                                Filename: {os.path.basename(filename)}
-                                SHA256sum: {file_sha256(local_path)}
-                                Size: {os.path.getsize(local_path)}
-
-                                """
-                            )
-
-                            index_file.write(control)
-                            index_gzip_file.write(control)
-
     def make_listing(self) -> None:
         """Generate the static web listing for packages in the repo."""
         logger.info("Generating web listing")
 
         packages = [
             package
             for generic_recipe in self.generic_recipes.values()
-            for recipe in generic_recipe.recipes.values()
+            for recipe in generic_recipe.values()
             for package in recipe.packages.values()
         ]
 
@@ -262,7 +233,12 @@ def make_listing(self) -> None:
         }
 
         listing_path = os.path.join(self.repo_dir, "index.html")
-        template = templating.env.get_template("listing.html")
+        template = Environment(
+            loader=FileSystemLoader(
+                pathlib.Path(__file__).parent.resolve() / ".." / "templates"
+            ),
+            autoescape=True,
+        ).get_template("listing.html")
 
         # pylint: disable-next=unspecified-encoding
         with open(listing_path, "w") as listing_file:

scripts/build/util.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2021 The Toltec Contributors
+# SPDX-License-Identifier: MIT
+"""Collection of useful functions."""
+
+import itertools
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Protocol,
+    Sequence,
+    TypeVar,
+)
+
+
+# See <https://github.com/python/typing/issues/760>
+class SupportsLessThan(Protocol):  # pylint:disable=too-few-public-methods
+    """Types that support the less-than operator."""
+
+    def __lt__(self, other: Any) -> bool:
+        ...
+
+
+Key = TypeVar("Key", bound=SupportsLessThan)
+Value = TypeVar("Value")
+
+
+def group_by(
+    in_seq: Sequence[Value], key_fn: Callable[[Value], Key]
+) -> Dict[Key, List[Value]]:
+    """
+    Group elements of a list.
+
+    :param in_seq: list of elements to group
+    :param key_fn: mapping of each element onto a group
+    :returns: dictionary of groups
+    """
+    return dict(
+        (key, list(group))
+        for key, group in itertools.groupby(
+            sorted(in_seq, key=key_fn), key=key_fn
+        )
+    )