Merge pull request #4 from nichmor/feat/add-more-recipes

feat: add more recipes and fail nice
prefix-dev · May 16, 2024 · 6bee516 · 6bee516
2 parents cc7e94a + d55a8cf
commit 6bee516
Show file tree

Hide file tree

Showing 2 changed files with 115 additions and 83 deletions.
diff --git a/config.yaml b/config.yaml
@@ -4,6 +4,18 @@ repositories:
     recipes:
       - path: recipe/recipe.yaml
 
+  - url: https://github.com/prefix-dev/rattler-build.git
+    branch: main
+    recipes:
+      - path: examples/xtensor/recipe.yaml
+      - path: examples/ros-humble-turtlebot4-msgs/recipe.yaml # is failing at build
+      - path: examples/rich/recipe.yaml
+      - path: examples/mamba/recipe.yaml # is failing at build
+      - path: examples/linking/recipe.yaml # is failing
+      - path: examples/curl/recipe.yaml
+      - path: examples/cmpcdt/recipe.yaml
+      - path: examples/cargo-edit/recipe.yaml
+
 
 local:
   - path: recipes/recipe.yaml
diff --git a/generate_chart_data.py b/generate_chart_data.py
@@ -7,128 +7,148 @@
 import hashlib
 import shutil
 import tempfile
-import yaml
 from build import build_conda_recipe, rebuild_conda_package
 import conf
 from git import checkout_branch_or_commit, clone_repo
 from util import calculate_hash, find_conda_build, get_recipe_name, move_file
 
 
 if __name__ == "__main__":
-
     config = conf.load_config()
 
-    build_dir = Path('build_outputs')
-    build_dir.mkdir(exist_ok=True)
-
-    build_results = {}
-
-    for repo in config['repositories']:
-        repo_url = repo['url']
-        ref = repo.get('branch') or repo.get('commit')
-        clone_dir = Path('cloned').joinpath(repo_url.split('/')[-1].replace('.git', ''))
-
-        if clone_dir.exists():
-            shutil.rmtree(clone_dir)
-
-        print(f'Cloning repository: {repo_url}')
-
-
-        clone_repo(repo_url, clone_dir)
-
-        if ref:
-            print(f'Checking out {ref}')
-            checkout_branch_or_commit(clone_dir, ref)
-
-
-        for recipe in repo['recipes']:
-            recipe_path = clone_dir / recipe['path']
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        build_dir = Path(tmp_dir) / "build_outputs"
+        build_dir.mkdir(exist_ok=True)
+
+        cloned_prefix_dir = Path(tmp_dir) / "cloned"
+
+        build_results = {}
+
+        for repo in config.get("repositories", []):
+            repo_url = repo["url"]
+            ref = repo.get("branch") or repo.get("commit")
+            clone_dir = cloned_prefix_dir.joinpath(
+                repo_url.split("/")[-1].replace(".git", "")
+            )
+
+            if clone_dir.exists():
+                shutil.rmtree(clone_dir)
+
+            print(f"Cloning repository: {repo_url}")
+
+            clone_repo(repo_url, clone_dir)
+
+            if ref:
+                print(f"Checking out {ref}")
+                checkout_branch_or_commit(clone_dir, ref)
+
+            for recipe in repo["recipes"]:
+                recipe_path = clone_dir / recipe["path"]
+                recipe_name = recipe_path.name
+
+                print(f"Building recipe: {recipe_name}")
+
+                # First build
+                first_build_dir = build_dir / f"{recipe_name}_first"
+                first_build_dir.mkdir(parents=True, exist_ok=True)
+
+                try:
+                    build_conda_recipe(recipe_path, first_build_dir)
+                except subprocess.CalledProcessError:
+                    # something went wrong with building it
+                    # for now we record it as not rebuildable
+                    # and skip to next recipe
+                    build_results[str(recipe_path)] = False
+                    continue
+
+                # let's record first hash
+                conda_file = find_conda_build(first_build_dir)
+                print(conda_file)
+                first_build_hash = calculate_hash(conda_file)
+
+                # let's move the build
+                rebuild_directory = build_dir / f"{recipe_name}_rebuild"
+                rebuild_file_loc = move_file(conda_file, rebuild_directory)
+
+                # let's rebuild it
+                try:
+                    rebuild_conda_package(rebuild_file_loc, first_build_dir)
+                except subprocess.CalledProcessError:
+                    # something went wrong with building it
+                    # for now we record it as not rebuildable
+                    # and skip to next recipe
+                    build_results[str(recipe_path)] = False
+                    continue
+
+                # let's record rebuild hash
+                re_conda_file = find_conda_build(first_build_dir)
+                print(re_conda_file)
+                re_build_hash = calculate_hash(re_conda_file)
+
+                if first_build_hash == re_build_hash:
+                    build_results[str(recipe_path)] = True
+                    print("they are the same!")
+                else:
+                    print("they are not the same!")
+                    build_results[str(recipe_path)] = False
+
+        for local in config.get("local", []):
+            recipe_path = Path(local["path"])
             recipe_name = recipe_path.name
+            package_name = get_recipe_name(recipe_path)
+            print(f"Building recipe: {recipe_name}")
 
-            print(f'Building recipe: {recipe_name}')
-
             # First build
-            first_build_dir = build_dir / f'{recipe_name}_first'
+            first_build_dir = build_dir / f"{recipe_name}_first"
             first_build_dir.mkdir(parents=True, exist_ok=True)
-            build_conda_recipe(recipe_path, first_build_dir)
-
+            try:
+                build_conda_recipe(recipe_path, first_build_dir)
+            except subprocess.CalledProcessError:
+                # something went wrong with building it
+                # for now we record it as not rebuildable
+                # and skip to next recipe
+                build_results[str(recipe_path)] = False
+                continue
 
             # let's record first hash
             conda_file = find_conda_build(first_build_dir)
             print(conda_file)
             first_build_hash = calculate_hash(conda_file)
 
             # let's move the build
-            rebuild_directory = build_dir / f'{recipe_name}_rebuild'
+            rebuild_directory = build_dir / f"{recipe_name}_rebuild"
             rebuild_file_loc = move_file(conda_file, rebuild_directory)
 
-
             # let's rebuild it
-            rebuild_conda_package(rebuild_file_loc, first_build_dir)
+            try:
+                rebuild_conda_package(rebuild_file_loc, first_build_dir)
+            except subprocess.CalledProcessError:
+                # something went wrong with building it
+                # for now we record it as not rebuildable
+                # and skip to next recipe
+                build_results[str(recipe_path)] = False
+                continue
 
             # let's record rebuild hash
             re_conda_file = find_conda_build(first_build_dir)
             print(re_conda_file)
             re_build_hash = calculate_hash(re_conda_file)
 
-
             if first_build_hash == re_build_hash:
-                build_results[str(recipe_path)] = True
                 print("they are the same!")
+                build_results[str(recipe_path)] = True
+
             else:
                 print("they are not the same!")
                 build_results[str(recipe_path)] = False
 
-
-    for local in config['local']:
-
-        recipe_path = Path(local['path'])
-        recipe_name = recipe_path.name
-        package_name = get_recipe_name(recipe_path)
-        print(f'Building recipe: {recipe_name}')
-
-        # First build
-        first_build_dir = build_dir / f'{recipe_name}_first'
-        first_build_dir.mkdir(parents=True, exist_ok=True)
-        build_conda_recipe(recipe_path, first_build_dir)
-
-
-        # let's record first hash
-        conda_file = find_conda_build(first_build_dir)
-        print(conda_file)
-        first_build_hash = calculate_hash(conda_file)
-
-        # let's move the build
-        rebuild_directory = build_dir / f'{recipe_name}_rebuild'
-        rebuild_file_loc = move_file(conda_file, rebuild_directory)
-
-
-        # let's rebuild it
-        rebuild_conda_package(rebuild_file_loc, first_build_dir)
-
-        # let's record rebuild hash
-        re_conda_file = find_conda_build(first_build_dir)
-        print(re_conda_file)
-        re_build_hash = calculate_hash(re_conda_file)
-
-
-        if first_build_hash == re_build_hash:
-            print("they are the same!")
-            build_results[str(recipe_path)] = True
-
-        else:
-            print("they are not the same!")
-            build_results[str(recipe_path)] = False
-
-
     total_packages = len(build_results)
     reproducible = sum(value for value in build_results.values() if value)
     not_reproducible = sum(value for value in build_results.values() if not value)
 
-
     today_date = datetime.datetime.now().strftime("%Y-%m-%d")
-    with open(f'data/chart_data_{today_date}.txt', 'w') as f:
-        f.write(f'{total_packages} {reproducible} {not_reproducible}\n')
+    with open(f"data/chart_data_{today_date}.txt", "w") as f:
+        f.write(f"{total_packages} {reproducible} {not_reproducible}\n")
 
-    with open(f'data/packages_info_{today_date}.json', 'w') as pkg_info:
+    with open(f"data/packages_info_{today_date}.json", "w") as pkg_info:
         json.dump(build_results, pkg_info)