Add migration system

vinca/distro.py

@@ -65,6 +65,9 @@ def get_released_repo(self, pkg_name):
         release_tag = get_release_tag(repo, pkg_name)
         return repo.url, release_tag
 
+    def get_released_repo_name(self, package_name):
+        return self._distro.release_packages[package_name].repository_name
+
     def check_package(self, pkg_name):
         if pkg_name in self._distro.release_packages:
             return True

vinca/migrate.py

@@ -1,203 +1,159 @@
-import yaml
-import sys
-import os
 import argparse
-import re
 import networkx as nx
-import subprocess
-import shutil
-import ruamel.yaml
-from .utils import get_repodata
-from vinca import config
-from vinca.distro import Distro
-from distutils.dir_util import copy_tree
-
-distro_version = None
-ros_prefix = None
-
-# arches = ["linux-64", "linux-aarch64", "win-64", "osx-64", "osx-arm64"]
-# arch_to_fname = {
-#     "linux-64": "linux",
-#     "linux-aarch64": "linux_aarch_64",
-#     "win-64": "win",
-#     "osx-64": "osx",
-#     "osx-arm64": "osx_arm64"
-# }
-
-
-def to_ros_name(distro, pkg_name):
-    shortname = pkg_name[len(ros_prefix) + 1 :]
-    if distro.check_package(shortname):
-        return shortname
-    elif distro.check_package(shortname.replace("-", "_")):
-        return shortname.replace("-", "_")
-    else:
-        raise RuntimeError(f"Couldnt convert {pkg_name} to ROS pkg name")
-
-
-def create_migration_instructions(arch, packages_to_migrate, trigger_branch):
-    url = "https://conda.anaconda.org/robostack/"
-
-    yaml = ruamel.yaml.YAML()
-    with open("vinca.yaml", "r") as fi:
-        vinca_conf = yaml.load(fi)
-
-    global distro_version, ros_prefix
-    distro_version = vinca_conf["ros_distro"]
-    ros_prefix = f"ros-{distro_version}"
-
-    repodata = get_repodata(url, arch)
+from ruamel.yaml import YAML
+from .distro import Distro
+from .utils import get_repodata, get_pinnings
 
-    packages = repodata["packages"]
-    to_migrate = set()
-    ros_pkgs = set()
-    for pkey in packages:
-        if not pkey.startswith(ros_prefix):
-            continue
-
-        pname = pkey.rsplit("-", 2)[0]
-        ros_pkgs.add(pname)
-
-        p = packages[pkey]
-
-        for d in p.get("depends", []):
-            if d.split()[0] in packages_to_migrate:
-                # print(f"need to migrate {pkey}")
-                to_migrate.add(pname)
-
-    latest = {}
-    for pkg in ros_pkgs:
-        current = current_version = None
-        for pkey in packages:
-            if packages[pkey]["name"] == pkg:
-                tmp = packages[pkey]["version"].split(".")
-                version = []
-                for el in tmp:
-                    if el.isdecimal():
-                        version.append(int(el))
-                    else:
-                        x = re.search(r"[^0-9]", version).start()
-                        version.append(int(el[:x]))
-
-                version = tuple(version)
-
-                if not current or version > current_version:
-                    current_version = version
-                    current = pkey
-        latest[pkg] = current
 
-    # now we can build the graph ...
+def main():
+    parser = argparse.ArgumentParser(
+        description="Dependency migration tool for ROS packages"
+    )
+    parser.add_argument(
+        type=str,
+        dest="pinnings",
+        help="Path to the local pinnings file",
+        metavar="PINNINGS",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        dest="all",
+        help="Show all dependencies",
+        required=False,
+    )
+    parser.add_argument(
+        "-p",
+        "--platform",
+        type=str,
+        dest="platform",
+        choices=["win-64", "linux-64", "linux-aarch64", "osx-64", "osx-arm64"],
+        default="linux-64",
+        help="Platform to target (default: linux-64)",
+        required=False,
+    )
+    parser.add_argument(
+        "-v",
+        "--vinca",
+        type=str,
+        dest="vinca",
+        default="vinca.yaml",
+        help="Path to the vinca configuration file",
+        required=False,
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        dest="dry_run",
+        help="Show changes without modifying files",
+        required=False,
+    )
+    parser.add_argument(
+        "--pinnings-only",
+        action="store_true",
+        dest="pinnings_only",
+        help="Only update the pinnings file",
+        required=False,
+    )
+    parser.add_argument(
+        "--repodata",
+        type=str,
+        dest="repodata",
+        default="https://conda.anaconda.org/robostack-staging/",
+        help="URL to the repodata file",
+        required=False,
+    )
+    parser.add_argument(
+        "--upstream",
+        type=str,
+        dest="upstream",
+        default="https://raw.githubusercontent.com/conda-forge/conda-forge-pinning-feedstock/refs/heads/main/recipe/conda_build_config.yaml",
+        help="URL to the upstream pinnings file",
+        required=False,
+    )
+    args = parser.parse_args()
 
-    G = nx.DiGraph()
-    for pkg, pkgkey in latest.items():
-        full_pkg = packages[pkgkey]
-        for dep in full_pkg.get("depends", []):
-            req = dep.split(" ")[0]
-            G.add_node(pkg)
-            if req.startswith(ros_prefix):
-                G.add_edge(pkg, req)
+    with open(args.vinca, "r") as f:
+        vinca = YAML().load(f)
 
-    gsorted = nx.topological_sort(G)
-    gsorted = list(reversed([g for g in gsorted]))
+    ros_distro = vinca["ros_distro"]
 
-    to_migrate = sorted(to_migrate, key=lambda x: gsorted.index(x))
+    repodata = get_repodata(args.repodata, args.platform)
+    packages = repodata["packages"]
 
-    print("Sorted to migrate: ", to_migrate)
+    deps = set()
+    for pkg in packages:
+        for dep in packages[pkg].get("depends", []):
+            deps.add(dep.split()[0])
 
-    distro = Distro(distro_version)
-    # import IPython; IPython.embed()
+    local = get_pinnings(args.pinnings)
+    upstream = get_pinnings(args.upstream)
 
-    ros_names = []
-    for pkg in to_migrate:
-        ros_names.append(to_ros_name(distro, pkg))
-    print("Final names: ", ros_names)
+    common = sorted(deps.intersection(local.keys()))
+    max_len = max(len(name) for name in common)
+    print("\033[1m{0:{2}} {1}\033[0m".format("Package", "Versions", max_len))
 
-    vinca_conf["packages_select_by_deps"] = ros_names
-    vinca_conf["skip_all_deps"] = True
-    vinca_conf["is_migration"] = True
-    vinca_conf["skip_existing"] = []
+    changed = []
 
-    with open("vinca.yaml", "w") as fo:
-        yaml.dump(vinca_conf, fo)
+    for name in common:
+        current = local[name]
+        latest = upstream[name]
 
-    if os.path.exists("recipes"):
-        shutil.rmtree("recipes")
+        if current == latest:
+            if args.all:
+                print("{0:{2}} {1}".format(name, current, max_len))
+            continue
 
-    mutex_path = os.path.join(
-        config.parsed_args.dir, "additional_recipes/ros-distro-mutex"
-    )
-    if os.path.exists(mutex_path):
-        goal_folder = os.path.join(
-            config.parsed_args.dir, "recipes", "ros-distro-mutex"
-        )
-        os.makedirs(goal_folder, exist_ok=True)
-        copy_tree(mutex_path, goal_folder)
-
-    subprocess.check_call(
-        ["vinca", "-d", config.parsed_args.dir, "--multiple", "--platform", arch]
-    )
+        print("{0:{3}} {1} -> {2}".format(name, current, latest, max_len))
 
-    # TODO remove hard coded build branch here!
-    recipe_dir = os.path.join(config.parsed_args.dir, "recipes")
-    subprocess.check_call(
-        [
-            "vinca-azure",
-            "--platform",
-            arch,
-            "--trigger-branch",
-            "buildbranch_linux",
-            "-d",
-            recipe_dir,
-            "--additional-recipes",
-        ]
-    )
+        local[name] = latest
+        changed.append(name)
 
+    if not changed:
+        print("No packages to migrate")
+        return
 
-def parse_command_line(argv):
-    parser = argparse.ArgumentParser(
-        description="Conda recipe Azure pipeline generator for ROS packages"
-    )
+    if not args.dry_run:
+        with open(args.pinnings, "w") as f:
+            yaml = YAML()
+            yaml.indent(mapping=2, sequence=4, offset=2)
+            yaml.compact_seq_seq = False
+            # TODO: check output formatting
+            yaml.dump(local, f)
 
-    default_dir = "./recipes"
-    parser.add_argument(
-        "-d",
-        "--dir",
-        dest="dir",
-        default=default_dir,
-        help="The recipes directory to process (default: {}).".format(default_dir),
-    )
+    if args.pinnings_only:
+        return
 
-    parser.add_argument(
-        "-t", "--trigger-branch", dest="trigger_branch", help="Trigger branch for Azure"
-    )
+    graph = nx.DiGraph()
+    for pkg in packages:
+        pkg_name = packages[pkg].get("name")
+        if not pkg_name.startswith("ros-" + ros_distro):
+            continue
+        graph.add_node(pkg_name)
+        for dep in packages[pkg].get("depends", []):
+            req = dep.split()[0]
+            graph.add_edge(pkg_name, req)
 
-    parser.add_argument(
-        "-p",
-        "--platform",
-        dest="platform",
-        default="linux-64",
-        help="Platform to emit build pipeline for",
-    )
+    graph = graph.reverse()
+    distro = Distro(ros_distro)
 
-    parser.add_argument(
-        "-a",
-        "--additional-recipes",
-        action="store_true",
-        help="search for additional_recipes folder?",
-    )
+    rebuild = set()
+    for pkg in changed:
+        if pkg not in graph:
+            continue
+        for node in nx.dfs_predecessors(graph, pkg):
+            ros_name = node.removeprefix("ros-" + ros_distro + "-").replace("-", "_")
+            repo_name = distro.get_released_repo_name(ros_name)
+            rebuild.add(repo_name)
 
-    arguments = parser.parse_args(argv[1:])
-    config.parsed_args = arguments
-    return arguments
+    new_build_number = vinca["build_number"] + 1
+    yaml_data = {package: {"build_number": new_build_number} for package in list(rebuild)}
 
+    if not args.dry_run:
+        with open("pkg_additional_info.yaml", "w") as f:
+            yaml.dump(yaml_data, f)
 
-def main():
-    args = parse_command_line(sys.argv)
-
-    mfile = os.path.join(args.dir + "/migration.yaml")
-    with open(mfile, "r") as fi:
-        migration = yaml.safe_load(fi)
-        print(migration)
-        create_migration_instructions(
-            args.platform, migration.get("packages", []), args.trigger_branch
-        )
+    print("\n\033[1mPackages to rebuild:\033[0m")
+    for pkg in rebuild:
+        print(pkg)

vinca/utils.py

@@ -4,6 +4,7 @@
 import time
 import json
 import requests
+import ruamel.yaml
 
 
 class folded_unicode(str):
@@ -64,3 +65,15 @@ def get_repodata(url_or_path, platform=None):
     with open(fn, "w") as fcache:
         fcache.write(content.decode("utf-8"))
     return json.loads(content)
+
+
+def get_pinnings(url_or_path):
+    yaml = ruamel.yaml.YAML()
+    yaml.allow_duplicate_keys = True
+
+    if "://" not in url_or_path:
+        with open(url_or_path) as fi:
+            return yaml.load(fi)
+
+    response = requests.get(url_or_path)
+    return yaml.load(response.content)