diff --git a/config.yaml b/config.yaml index 06a80ae0..1e3b490d 100644 --- a/config.yaml +++ b/config.yaml @@ -4,6 +4,18 @@ repositories: recipes: - path: recipe/recipe.yaml + - url: https://github.com/prefix-dev/rattler-build.git + branch: main + recipes: + - path: examples/xtensor/recipe.yaml + - path: examples/ros-humble-turtlebot4-msgs/recipe.yaml # is failing at build + - path: examples/rich/recipe.yaml + - path: examples/mamba/recipe.yaml # is failing at build + - path: examples/linking/recipe.yaml # is failing + - path: examples/curl/recipe.yaml + - path: examples/cmpcdt/recipe.yaml + - path: examples/cargo-edit/recipe.yaml + local: - path: recipes/recipe.yaml \ No newline at end of file diff --git a/generate_chart_data.py b/generate_chart_data.py index 43ad40f0..04028fc8 100644 --- a/generate_chart_data.py +++ b/generate_chart_data.py @@ -7,7 +7,6 @@ import hashlib import shutil import tempfile -import yaml from build import build_conda_recipe, rebuild_conda_package import conf from git import checkout_branch_or_commit, clone_repo @@ -15,43 +14,101 @@ if __name__ == "__main__": - config = conf.load_config() - build_dir = Path('build_outputs') - build_dir.mkdir(exist_ok=True) - - build_results = {} - - for repo in config['repositories']: - repo_url = repo['url'] - ref = repo.get('branch') or repo.get('commit') - clone_dir = Path('cloned').joinpath(repo_url.split('/')[-1].replace('.git', '')) - - if clone_dir.exists(): - shutil.rmtree(clone_dir) - - print(f'Cloning repository: {repo_url}') - - - clone_repo(repo_url, clone_dir) - - if ref: - print(f'Checking out {ref}') - checkout_branch_or_commit(clone_dir, ref) - - - for recipe in repo['recipes']: - recipe_path = clone_dir / recipe['path'] + with tempfile.TemporaryDirectory() as tmp_dir: + build_dir = Path(tmp_dir) / "build_outputs" + build_dir.mkdir(exist_ok=True) + + cloned_prefix_dir = Path(tmp_dir) / "cloned" + + build_results = {} + + for repo in config.get("repositories", []): + repo_url = repo["url"] + ref = repo.get("branch") or repo.get("commit") + clone_dir = cloned_prefix_dir.joinpath( + repo_url.split("/")[-1].replace(".git", "") + ) + + if clone_dir.exists(): + shutil.rmtree(clone_dir) + + print(f"Cloning repository: {repo_url}") + + clone_repo(repo_url, clone_dir) + + if ref: + print(f"Checking out {ref}") + checkout_branch_or_commit(clone_dir, ref) + + for recipe in repo["recipes"]: + recipe_path = clone_dir / recipe["path"] + recipe_name = recipe_path.name + + print(f"Building recipe: {recipe_name}") + + # First build + first_build_dir = build_dir / f"{recipe_name}_first" + first_build_dir.mkdir(parents=True, exist_ok=True) + + try: + build_conda_recipe(recipe_path, first_build_dir) + except subprocess.CalledProcessError: + # something went wrong with building it + # for now we record it as not rebuildable + # and skip to next recipe + build_results[str(recipe_path)] = False + continue + + # let's record first hash + conda_file = find_conda_build(first_build_dir) + print(conda_file) + first_build_hash = calculate_hash(conda_file) + + # let's move the build + rebuild_directory = build_dir / f"{recipe_name}_rebuild" + rebuild_file_loc = move_file(conda_file, rebuild_directory) + + # let's rebuild it + try: + rebuild_conda_package(rebuild_file_loc, first_build_dir) + except subprocess.CalledProcessError: + # something went wrong with building it + # for now we record it as not rebuildable + # and skip to next recipe + build_results[str(recipe_path)] = False + continue + + # let's record rebuild hash + re_conda_file = find_conda_build(first_build_dir) + print(re_conda_file) + re_build_hash = calculate_hash(re_conda_file) + + if first_build_hash == re_build_hash: + build_results[str(recipe_path)] = True + print("they are the same!") + else: + print("they are not the same!") + build_results[str(recipe_path)] = False + + for local in config.get("local", []): + recipe_path = Path(local["path"]) recipe_name = recipe_path.name + package_name = get_recipe_name(recipe_path) + print(f"Building recipe: {recipe_name}") - print(f'Building recipe: {recipe_name}') - # First build - first_build_dir = build_dir / f'{recipe_name}_first' + first_build_dir = build_dir / f"{recipe_name}_first" first_build_dir.mkdir(parents=True, exist_ok=True) - build_conda_recipe(recipe_path, first_build_dir) - + try: + build_conda_recipe(recipe_path, first_build_dir) + except subprocess.CalledProcessError: + # something went wrong with building it + # for now we record it as not rebuildable + # and skip to next recipe + build_results[str(recipe_path)] = False + continue # let's record first hash conda_file = find_conda_build(first_build_dir) @@ -59,76 +116,39 @@ first_build_hash = calculate_hash(conda_file) # let's move the build - rebuild_directory = build_dir / f'{recipe_name}_rebuild' + rebuild_directory = build_dir / f"{recipe_name}_rebuild" rebuild_file_loc = move_file(conda_file, rebuild_directory) - # let's rebuild it - rebuild_conda_package(rebuild_file_loc, first_build_dir) + try: + rebuild_conda_package(rebuild_file_loc, first_build_dir) + except subprocess.CalledProcessError: + # something went wrong with building it + # for now we record it as not rebuildable + # and skip to next recipe + build_results[str(recipe_path)] = False + continue # let's record rebuild hash re_conda_file = find_conda_build(first_build_dir) print(re_conda_file) re_build_hash = calculate_hash(re_conda_file) - if first_build_hash == re_build_hash: - build_results[str(recipe_path)] = True print("they are the same!") + build_results[str(recipe_path)] = True + else: print("they are not the same!") build_results[str(recipe_path)] = False - - for local in config['local']: - - recipe_path = Path(local['path']) - recipe_name = recipe_path.name - package_name = get_recipe_name(recipe_path) - print(f'Building recipe: {recipe_name}') - - # First build - first_build_dir = build_dir / f'{recipe_name}_first' - first_build_dir.mkdir(parents=True, exist_ok=True) - build_conda_recipe(recipe_path, first_build_dir) - - - # let's record first hash - conda_file = find_conda_build(first_build_dir) - print(conda_file) - first_build_hash = calculate_hash(conda_file) - - # let's move the build - rebuild_directory = build_dir / f'{recipe_name}_rebuild' - rebuild_file_loc = move_file(conda_file, rebuild_directory) - - - # let's rebuild it - rebuild_conda_package(rebuild_file_loc, first_build_dir) - - # let's record rebuild hash - re_conda_file = find_conda_build(first_build_dir) - print(re_conda_file) - re_build_hash = calculate_hash(re_conda_file) - - - if first_build_hash == re_build_hash: - print("they are the same!") - build_results[str(recipe_path)] = True - - else: - print("they are not the same!") - build_results[str(recipe_path)] = False - - total_packages = len(build_results) reproducible = sum(value for value in build_results.values() if value) not_reproducible = sum(value for value in build_results.values() if not value) - today_date = datetime.datetime.now().strftime("%Y-%m-%d") - with open(f'data/chart_data_{today_date}.txt', 'w') as f: - f.write(f'{total_packages} {reproducible} {not_reproducible}\n') + with open(f"data/chart_data_{today_date}.txt", "w") as f: + f.write(f"{total_packages} {reproducible} {not_reproducible}\n") - with open(f'data/packages_info_{today_date}.json', 'w') as pkg_info: + with open(f"data/packages_info_{today_date}.json", "w") as pkg_info: json.dump(build_results, pkg_info)