Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: failing and non-repro recipes #157

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 21 additions & 17 deletions .github/workflows/build-and-rebuild.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,29 +20,31 @@ env:
PYTHONUTF8: 1

jobs:
run-tests:
continue-on-error: false
# run-tests:
# continue-on-error: false

runs-on: ubuntu-latest
# runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v4
# steps:
# - name: Checkout code
# uses: actions/checkout@v4

- uses: prefix-dev/[email protected]
with:
pixi-version: "latest"
# - uses: prefix-dev/[email protected]
# with:
# pixi-version: "latest"

- name: Build rattler-build
run: |
pixi run test
# - name: Build rattler-build
# run: |
# pixi run test

setup-rattler-build:
needs: run-tests
# needs: run-tests
continue-on-error: false
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
# os: [ubuntu-latest, windows-latest, macos-latest]
os: [macos-latest]


runs-on: ${{ matrix.os }}

Expand Down Expand Up @@ -84,7 +86,7 @@ jobs:
key: v4-${{ runner.os }}-rattler-build-${{ steps.read-yaml.outputs['rattler-build.rev'] }}

generate-recipes:
runs-on: ubuntu-latest
runs-on: macos-latest
needs: setup-rattler-build
outputs:
recipe: ${{ steps.generate-matrix.outputs.recipes }}
Expand All @@ -99,7 +101,7 @@ jobs:
- name: Generate matrix
id: generate-matrix
run: |
message="$(pixi r generate-recipes)"
message="$(pixi r generate-recipes-failed)"
echo "$message"
echo "recipes=$message" >> "$GITHUB_OUTPUT"
env:
Expand All @@ -113,7 +115,9 @@ jobs:
matrix:
recipe_name: ${{ fromJson(needs.generate-recipes.outputs.recipe) }}
# os: [ubuntu-latest]
os: [ubuntu-latest, windows-latest, macos-latest]
# os: [ubuntu-latest, windows-latest, macos-latest]
os: [macos-latest]


runs-on: ${{ matrix.os }}

Expand Down
2 changes: 1 addition & 1 deletion config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ local:
- path: recipes/memory_profiler/recipe.yaml
- path: recipes/ruby/recipe.yaml
rattler-build:
rev: a47f907eaacc345dcd9441fe08d43057b18e2e83
rev: ba692c24ff2fe2448f5f1dc0d342fcbd873ac2dc
url: https://github.com/prefix-dev/rattler-build.git
repositories:
- recipes:
Expand Down
36 changes: 18 additions & 18 deletions pixi.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ generate-html-prod = { cmd = "repror generate-html", env = { REPRO_DB_NAME = "re
] }
# Used to generate build matrix for githubgenerate-html = "repror generate-html"
generate-recipes = "repror --no-output generate-recipes"
generate-recipes-failed = "repror --no-output generate-recipes --only-failed"
# Build recipes in the recipes folder
build-recipe = "repror build-recipe"
# Same as above but skip rattler-build setup
Expand Down
7 changes: 6 additions & 1 deletion src/repror/cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ def main(
- Build/rebuild packages using conda build
- Manage a local rattler-build environment for custom builds
- Rewrite the reproducible-builds README.md file with update statistics

Use REPRO_DB_NAME to set the database name. Default is local.repror.db, and production is repror.db
"""
global_options.no_output = no_output
global_options.config_path = config_path
Expand All @@ -63,9 +65,12 @@ def generate_recipes(
all_: Annotated[
bool, typer.Option("--all", help="Generate all recipe names")
] = False,
only_failed: Annotated[bool, typer.Option()] = False,
):
"""Generate list of recipes from the configuration file. By default it will print only the ones that are not built yet."""
generate.generate_recipes(rattler_build_hash=rattler_build_hash(), all_=all_)
generate.generate_recipes(
rattler_build_hash=rattler_build_hash(), all_=all_, only_failed=only_failed
)


def _check_local_rattler_build():
Expand Down
45 changes: 42 additions & 3 deletions src/repror/cli/generate_recipes.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,47 @@


def _generate_recipes(
rattler_build_hash: str, all_: bool = False, config_path: Path = Path("config.yaml")
rattler_build_hash: str,
all_: bool = False,
only_failed: bool = False,
config_path: Path = Path("config.yaml"),
):
# Prepare the matrix
all_recipes = config.load_all_recipes(config_path=str(config_path))

if all_ and only_failed:
raise ValueError("Cannot use both --all and --only-failed")

if only_failed:
# Get the name and hash of all the recipes
name_and_hash = [(recipe.name, recipe.content_hash) for recipe in all_recipes]

# Current rattler hash
# Latest build with rebuild, a.k.a. finished recipes
finished_recipes = get_latest_build_with_rebuild(
name_and_hash,
# build_tool_hash=rattler_build_hash,
platform_name=platform_name(),
# platform_version=platform_version(),
)
to_run = []

for recipe_name, (build, rebuild) in finished_recipes.items():
if not rebuild or not build:
to_run.append(recipe_name)
elif build.state == "failed" or rebuild.state == "failed":
to_run.append(recipe_name)
elif build.build_hash != rebuild.rebuild_hash:
to_run.append(recipe_name)

for name, hash in name_and_hash:
if name not in finished_recipes:
to_run.append(name)

# Get the recipes that are failed, non-repro, or didn't run yet
# import pdb; pdb.set_trace()
return list(set(to_run))

if not all_:
# Get the name and hash of all the recipes
name_and_hash = [(recipe.name, recipe.content_hash) for recipe in all_recipes]
Expand All @@ -32,9 +68,12 @@ def _generate_recipes(


def generate_recipes(
rattler_build_hash: str, all_: bool = False, config_path: Path = Path("config.yaml")
rattler_build_hash: str,
all_: bool = False,
only_failed: bool = False,
config_path: Path = Path("config.yaml"),
):
"""Generate list of recipes from the configuration file."""
recipe_list = _generate_recipes(rattler_build_hash, all_, config_path)
recipe_list = _generate_recipes(rattler_build_hash, all_, only_failed, config_path)
# Convert the matrix to JSON
print(json.dumps(recipe_list))
4 changes: 2 additions & 2 deletions src/repror/cli/rebuild_recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@ def rebuild_recipe(

latest_build_with_rebuild = get_latest_build_with_rebuild(
recipes_to_find,
rattler_hash,
# rattler_hash,
platform_name,
platform_version,
# platform_version,
)

for recipe in recipes:
Expand Down
8 changes: 4 additions & 4 deletions src/repror/internals/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,9 +226,9 @@ def get_latest_builds(

def get_latest_build_with_rebuild(
recipe_names_and_hash: list[tuple[str, str]],
build_tool_hash: str,
# build_tool_hash: str,
platform_name: str,
platform_version: str,
# platform_version: str,
) -> dict[str, tuple[Build, Optional[Rebuild]]]:
with get_session() as session:
conditions = [
Expand All @@ -243,9 +243,9 @@ def get_latest_build_with_rebuild(
func.max(Build.timestamp).label("max_timestamp"),
)
.where(or_(*conditions))
.where(Build.build_tool_hash == build_tool_hash)
# .where(Build.build_tool_hash == build_tool_hash)
.where(Build.platform_name == platform_name)
.where(Build.platform_version == platform_version)
# .where(Build.platform_version == platform_version)
.group_by(Build.recipe_name, Build.recipe_hash)
).subquery()

Expand Down
Loading