diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ce06fcab4..98eb1696e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,7 +30,46 @@ concurrency: cancel-in-progress: true jobs: + # Generate CLI docs by building the `generate-cli-docs` binary and commiting the result to the PR. + generate-cli-docs: + if: github.event_name == 'pull_request' && github.repository == 'prefix-dev/rattler-build' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + + - uses: prefix-dev/setup-pixi@v0.8.0 + + - name: Build docs for PR + run: | + pixi run generate-cli-docs + + - name: Check for changes + id: git_diff + run: | + git diff --exit-code || echo "has_changes=true" >> $GITHUB_ENV + + - name: Output instructions if there are changes + if: env.has_changes == 'true' + run: | + echo "There are changes in the CLI docs." + echo "Please run the following commands to commit and push the changes:" + echo "" + echo "pixi run commit-cli-docs" + echo "git commit -am 'docs: update CLI docs'" + echo "git push" + echo "" + exit 1 + + - name: Output message if there are no changes + if: env.has_changes != 'true' + run: echo "No changes detected." + build-docs: + needs: generate-cli-docs if: github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest steps: diff --git a/Cargo.lock b/Cargo.lock index adb70c9d3..fad01b4ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -740,6 +740,15 @@ dependencies = [ "clap_derive", ] +[[package]] +name = "clap-markdown" +version = "0.1.3" +source = "git+https://github.com/ruben-arts/clap-markdown?branch=main#bc771f47df47b7b3968c8d08c89372fcd91b5be3" +dependencies = [ + "clap", + "indexmap 2.2.6", +] + [[package]] name = "clap-verbosity-flag" version = "2.2.0" @@ -3400,6 +3409,7 @@ dependencies = [ "bzip2", "chrono", "clap", + "clap-markdown", "clap-verbosity-flag", "clap_complete", "comfy-table", diff --git a/Cargo.toml b/Cargo.toml index b5f299a60..6162742f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,12 +11,14 @@ license = "BSD-3-Clause" readme = "README.md" description = "A fast CLI tool to build conda packages on Windows, macOS and Linux" documentation = "https://prefix-dev.github.io/rattler-build" +default-run = "rattler-build" [features] default = ['native-tls'] native-tls = ['reqwest/native-tls', 'rattler/native-tls', 'rattler_installs_packages/native-tls'] rustls-tls = ['reqwest/rustls-tls', 'reqwest/rustls-tls-native-roots', 'rattler/rustls-tls', 'rattler_installs_packages/rustls-tls'] tui = ['ratatui', 'crossterm', 'ansi-to-tui', 'throbber-widgets-tui', 'tui-input'] +generate-cli-docs = ["clap-markdown"] [dependencies] serde = { version = "1.0.203", features = ["derive"] } @@ -115,6 +117,7 @@ reflink-copy = "0.1.17" rayon = "1.10.0" patch = "0.7.0" regex = "1.10.4" +clap-markdown = { version = "0.1.3", optional = true } [dev-dependencies] insta = { version = "1.39.0", features = ["yaml"] } @@ -148,6 +151,8 @@ rattler_shell = { git = "https://github.com/mamba-org/rattler", branch = "main" rattler_solve = { git = "https://github.com/mamba-org/rattler", branch = "main" } rattler_virtual_packages = { git = "https://github.com/mamba-org/rattler", branch = "main" } rattler_package_streaming = { git = "https://github.com/mamba-org/rattler", branch = "main" } +clap-markdown = { git = "https://github.com/ruben-arts/clap-markdown", branch = "main"} + # rattler = { path = "../rattler/crates/rattler" } # rattler_conda_types = { path = "../rattler/crates/rattler_conda_types" } @@ -159,3 +164,4 @@ rattler_package_streaming = { git = "https://github.com/mamba-org/rattler", bran # rattler_solve = { path = "../rattler/crates/rattler_solve" } # rattler_virtual_packages = { path = "../rattler/crates/rattler_virtual_packages" } # rattler_package_streaming = { path = "../rattler/crates/rattler_package_streaming" } +#clap-markdown = { path = "../clap-markdown" } diff --git a/docs/automatic_linting.md b/docs/automatic_linting.md index bf708dcbb..06be5705a 100644 --- a/docs/automatic_linting.md +++ b/docs/automatic_linting.md @@ -1,16 +1,13 @@ -# Automatic linting in VSCode +# Enabling Automatic Linting in VSCode -The new recipe format comes with a strict JSON scheme. You can find the scheme -[in this repository](https://github.com/prefix-dev/recipe-format). +Our new recipe format adheres to a strict JSON schema, which you can access [here](https://github.com/prefix-dev/recipe-format). -It is implemented with `pydantic` and renders to a JSON schema file. The [YAML -language server extension in -VSCode](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) -can recognize the scheme and give helpful hints during editing. +This schema is implemented using `pydantic` and can be rendered into a JSON schema file. The [YAML language server extension in VSCode](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) is capable of recognizing this schema, providing useful hints during the editing process. -With the YAML language server installed, the automatic linting can be enabled by -adding the following line to the top of the recipe file: +To enable automatic linting with the YAML language server, you need to add the following line at the beginning of your recipe file: ```yaml # yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json ``` + +**Alternatively**, if you prefer not to add this line to your file, you can install the [JSON Schema Store Catalog extension](https://marketplace.visualstudio.com/items?itemName=remcohaszing.schemastore). This extension will also enable automatic linting for your recipe files. \ No newline at end of file diff --git a/docs/highlevel.md b/docs/highlevel.md index d0b1b2fb9..a2ec0f762 100644 --- a/docs/highlevel.md +++ b/docs/highlevel.md @@ -12,29 +12,41 @@ Building of packages consists of several steps. It all begins with a `recipe.yaml` file that specifies how the package is to be built and what the dependencies are. From the recipe file, `rattler-build` executes several steps: -1. Parse the recipe file and evaluate conditional parts (we will see that later, - but parts of the recipe can be conditional, e.g. on Windows vs. macOS) -2. Retrieve all source files specified in the recipe, such as `.tar.gz` files, - `git` repositories or even local paths. Additionally, this step will apply - patches that can be specified alongside the source file. -3. Download and install dependencies into temporary "host" and "build" - workspaces. Any dependencies that are needed at build time are installed in - this step. -4. Execute the build script to build/compile the source code and "install" it - into the host environment. -5. Collect _all_ files that are new in the "host" environment (because the build - script just created them) and apply some transformations if necessary; - specifically, we edit the `rpath` on Linux and macOS to help make binaries - relocatable. -6. Bundle all the files in a package and write out any additional metadata into - the `info/index.json`, `info/about.json`, and `info/paths.json` files. This - also creates the test files that are bundled with the package. -7. If any tests are specified in the recipe, then those tests are run. The package - is considered "done" if it passes all of the tests, otherwise we move it to a - "broken" place. - -After this process, a package is created. This package can be uploaded to somewhere -like a custom [prefix.dev](https://prefix.dev) private or public channel. +1. **Rendering**: + + Parse the recipe file and evaluate conditionals, Jinja expressions, and + variables, and variants. + +2. **Fetch source**: + + Retrieve specified source files, such as `.tar.gz` files, `git` repositories, local paths. + Additionally, this step will apply patches that can be specified alongside the source file. + +3. **Install build environments**: + + Download and install dependencies into temporary "host" and "build" workspaces. + Any dependencies that are needed at build time are installed in this step. + +4. **Build source**: + + Execute the build script to _build/compile_ the source code and _install_ it into the host environment. + +5. **Prepare package files**: + + Collect _all_ files that are new in the "host" environment and apply some transformations if necessary; + specifically, we edit the `rpath` on `Linux` and `macOS` to make binaries relocatable. + +6. **Package**: + + Bundle all the files in a package and write out any additional metadata into the `info/index.json`, `info/about.json`, and `info/paths.json` files. + This also creates the test files that are bundled with the package. + +7. **Test**: + + Run any tests specified in the recipe. + The package is considered _done_ if it passes all the tests, otherwise its moved to `broken/` in the output directory. + +After this process, a package is created. This package can be uploaded to somewhere like a custom [prefix.dev](https://prefix.dev) private or public channel. ### How to run `rattler-build` @@ -109,10 +121,10 @@ requirements: The sections of a recipe are: -| sections | description | -| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| sections | description | +|----------------|------------------------------------------------------------------------------------------------------------------------------------------| | `context` | Defines variables that can be used in the Jinja context later in the recipe (e.g. name and version are commonly interpolated in strings) | -| `package` | This section defines the name and version of the package you are currently building and will be the name of the final output | -| `source` | Defines where the source code is going to be downloaded from and checksums | -| `build` | Settings for the build and the build script | -| `requirements` | Allows the definition of build, host, run and run-constrained dependencies | +| `package` | This section defines the name and version of the package you are currently building and will be the name of the final output | +| `source` | Defines where the source code is going to be downloaded from and checksums | +| `build` | Settings for the build and the build script | +| `requirements` | Allows the definition of build, host, run and run-constrained dependencies | diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 000000000..5ae108b1f --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,664 @@ +# Command-Line Help for `rattler-build` + +This document contains the help content for the `rattler-build` command-line program. + +## `rattler-build` + +**Usage:** `rattler-build [OPTIONS] [COMMAND]` + +##### **Subcommands:** + +* `build` — Build a package from a recipe +* `test` — Run a test for a single package +* `rebuild` — Rebuild a package from a package file instead of a recipe +* `upload` — Upload a package +* `completion` — Generate shell completion script +* `generate-recipe` — Generate a recipe from PyPI or CRAN +* `auth` — Handle authentication to external channels + +##### **Options:** + +- `-v`, `--verbose` + + Increase logging verbosity + + +- `-q`, `--quiet` + + Decrease logging verbosity + + +- `--log-style ` + + Logging style + + - Default value: `fancy` + - Possible values: + - `fancy`: + Use fancy logging output + - `json`: + Use JSON logging output + - `plain`: + Use plain logging output + + +- `--color ` + + Enable or disable colored output from rattler-build. Also honors the `CLICOLOR` and `CLICOLOR_FORCE` environment variable + + - Default value: `auto` + - Possible values: + - `always`: + Always use colors + - `never`: + Never use colors + - `auto`: + Use colors when the output is a terminal + + + + + +### `build` + +Build a package from a recipe + +**Usage:** `rattler-build build [OPTIONS]` + +##### **Options:** + +- `-r`, `--recipe ` + + The recipe file or directory containing `recipe.yaml`. Defaults to the current directory + + - Default value: `.` + +- `--recipe-dir ` + + The directory that contains recipes + + +- `--up-to ` + + Build recipes up to the specified package + + +- `--build-platform ` + + The build platform to use for the build (e.g. for building with emulation, or rendering) + + - Default value: `linux-64` + +- `--target-platform ` + + The target platform for the build + + - Default value: `linux-64` + +- `-c`, `--channel ` + + Add a channel to search for dependencies in + + - Default value: `conda-forge` + +- `-m`, `--variant-config ` + + Variant configuration files for the build + + +- `--render-only` + + Render the recipe files without executing the build + + - Possible values: `true`, `false` + + +- `--with-solve` + + Render the recipe files with solving dependencies + + - Possible values: `true`, `false` + + +- `--keep-build` + + Keep intermediate build artifacts after the build + + - Possible values: `true`, `false` + + +- `--no-build-id` + + Don't use build id(timestamp) when creating build directory name + + - Possible values: `true`, `false` + + +- `--compression-threads ` + + The number of threads to use for compression (only relevant when also using `--package-format conda`) + + +- `--use-zstd` + + Enable support for repodata.json.zst + + - Default value: `true` + - Possible values: `true`, `false` + + +- `--use-bz2` + + Enable support for repodata.json.bz2 + + - Default value: `true` + - Possible values: `true`, `false` + + +- `--experimental` + + Enable experimental features + + - Possible values: `true`, `false` + + +- `--auth-file ` + + Path to an auth-file to read authentication information from + + +- `--tui` + + Launch the terminal user interface + + - Default value: `false` + - Possible values: `true`, `false` + + +###### **Modifying result** + +- `--package-format ` + + The package format to use for the build. Can be one of `tar-bz2` or `conda`. +You can also add a compression level to the package format, e.g. `tar-bz2:` (from 1 to 9) or `conda:` (from -7 to 22). + + - Default value: `conda` + +- `--no-include-recipe` + + Don't store the recipe in the final package + + - Possible values: `true`, `false` + + +- `--no-test` + + Don't run the tests after building the package + + - Default value: `false` + - Possible values: `true`, `false` + + +- `--color-build-log` + + Don't force colors in the output of the build script + + - Default value: `true` + - Possible values: `true`, `false` + + +- `--output-dir ` + + Output directory for build artifacts. + + - Default value: `./output` + +- `--skip-existing ` + + Whether to skip packages that already exist in any channel If set to `none`, do not skip any packages, default when not specified. If set to `local`, only skip packages that already exist locally, default when using `--skip-existing. If set to `all`, skip packages that already exist in any channel + + - Default value: `none` + - Possible values: + - `none`: + Do not skip any packages + - `local`: + Skip packages that already exist locally + - `all`: + Skip packages that already exist in any channel + + + + + +### `test` + +Run a test for a single package + +This creates a temporary directory, copies the package file into it, and then runs the indexing. It then creates a test environment that installs the package and any extra dependencies specified in the package test dependencies file. + +With the activated test environment, the packaged test files are run: + +* `info/test/run_test.sh` or `info/test/run_test.bat` on Windows * `info/test/run_test.py` + +These test files are written at "package creation time" and are part of the package. + +**Usage:** `rattler-build test [OPTIONS] --package-file ` + +##### **Options:** + +- `-c`, `--channel ` + + Channels to use when testing + + +- `-p`, `--package-file ` + + The package file to test + + +- `--use-zstd` + + Enable support for repodata.json.zst + + - Default value: `true` + - Possible values: `true`, `false` + + +- `--use-bz2` + + Enable support for repodata.json.bz2 + + - Default value: `true` + - Possible values: `true`, `false` + + +- `--experimental` + + Enable experimental features + + - Possible values: `true`, `false` + + +- `--auth-file ` + + Path to an auth-file to read authentication information from + + +###### **Modifying result** + +- `--output-dir ` + + Output directory for build artifacts. + + - Default value: `./output` + + + + +### `rebuild` + +Rebuild a package from a package file instead of a recipe + +**Usage:** `rattler-build rebuild [OPTIONS] --package-file ` + +##### **Options:** + +- `-p`, `--package-file ` + + The package file to rebuild + + +- `--no-test` + + Do not run tests after building + + - Default value: `false` + - Possible values: `true`, `false` + + +- `--use-zstd` + + Enable support for repodata.json.zst + + - Default value: `true` + - Possible values: `true`, `false` + + +- `--use-bz2` + + Enable support for repodata.json.bz2 + + - Default value: `true` + - Possible values: `true`, `false` + + +- `--experimental` + + Enable experimental features + + - Possible values: `true`, `false` + + +- `--auth-file ` + + Path to an auth-file to read authentication information from + + +###### **Modifying result** + +- `--output-dir ` + + Output directory for build artifacts. + + - Default value: `./output` + + + + +### `upload` + +Upload a package + +**Usage:** `rattler-build upload [OPTIONS] [PACKAGE_FILES]... ` + +##### **Subcommands:** + +* `quetz` — Upload to aQuetz server. Authentication is used from the keychain / auth-file +* `artifactory` — Options for uploading to a Artifactory channel. Authentication is used from the keychain / auth-file +* `prefix` — Options for uploading to a prefix.dev server. Authentication is used from the keychain / auth-file +* `anaconda` — Options for uploading to a Anaconda.org server + +##### **Arguments:** + +- `` + + The package file to upload + + + +##### **Options:** + +- `--use-zstd` + + Enable support for repodata.json.zst + + - Default value: `true` + - Possible values: `true`, `false` + + +- `--use-bz2` + + Enable support for repodata.json.bz2 + + - Default value: `true` + - Possible values: `true`, `false` + + +- `--experimental` + + Enable experimental features + + - Possible values: `true`, `false` + + +- `--auth-file ` + + Path to an auth-file to read authentication information from + + +###### **Modifying result** + +- `--output-dir ` + + Output directory for build artifacts. + + - Default value: `./output` + + + + +#### `quetz` + +Upload to aQuetz server. Authentication is used from the keychain / auth-file + +**Usage:** `rattler-build upload quetz [OPTIONS] --url --channel ` + +##### **Options:** + +- `-u`, `--url ` + + The URL to your Quetz server + + +- `-c`, `--channel ` + + The URL to your channel + + +- `-a`, `--api-key ` + + The Quetz API key, if none is provided, the token is read from the keychain / auth-file + + + + + +#### `artifactory` + +Options for uploading to a Artifactory channel. Authentication is used from the keychain / auth-file + +**Usage:** `rattler-build upload artifactory [OPTIONS] --url --channel ` + +##### **Options:** + +- `-u`, `--url ` + + The URL to your Artifactory server + + +- `-c`, `--channel ` + + The URL to your channel + + +- `-r`, `--username ` + + Your Artifactory username + + +- `-p`, `--password ` + + Your Artifactory password + + + + + +#### `prefix` + +Options for uploading to a prefix.dev server. Authentication is used from the keychain / auth-file + +**Usage:** `rattler-build upload prefix [OPTIONS] --channel ` + +##### **Options:** + +- `-u`, `--url ` + + The URL to the prefix.dev server (only necessary for self-hosted instances) + + - Default value: `https://prefix.dev` + +- `-c`, `--channel ` + + The channel to upload the package to + + +- `-a`, `--api-key ` + + The prefix.dev API key, if none is provided, the token is read from the keychain / auth-file + + + + + +#### `anaconda` + +Options for uploading to a Anaconda.org server + +**Usage:** `rattler-build upload anaconda [OPTIONS] --owner ` + +##### **Options:** + +- `-o`, `--owner ` + + The owner of the distribution (e.g. conda-forge or your username) + + +- `-c`, `--channel ` + + The channel / label to upload the package to (e.g. main / rc) + + - Default value: `main` + +- `-a`, `--api-key ` + + The Anaconda API key, if none is provided, the token is read from the keychain / auth-file + + +- `-u`, `--url ` + + The URL to the Anaconda server + + - Default value: `https://api.anaconda.org` + +- `-f`, `--force` + + Replace files on conflict + + - Default value: `false` + - Possible values: `true`, `false` + + + + + +### `completion` + +Generate shell completion script + +**Usage:** `rattler-build completion [OPTIONS]` + +##### **Options:** + +- `-s`, `--shell ` + + Shell + + - Possible values: `bash`, `elvish`, `fish`, `powershell`, `zsh` + + + + + +### `generate-recipe` + +Generate a recipe from PyPI or CRAN + +**Usage:** `rattler-build generate-recipe ` + +##### **Arguments:** + +- `` + + Type of package to generate a recipe for + + - Possible values: + - `pypi`: + Generate a recipe for a Python package from PyPI + - `cran`: + Generate a recipe for an R package from CRAN + + +- `` + + Name of the package to generate + + + + + +### `auth` + +Handle authentication to external channels + +**Usage:** `rattler-build auth ` + +##### **Subcommands:** + +* `login` — Store authentication information for a given host +* `logout` — Remove authentication information for a given host + + + +#### `login` + +Store authentication information for a given host + +**Usage:** `rattler-build auth login [OPTIONS] ` + +##### **Arguments:** + +- `` + + The host to authenticate with (e.g. repo.prefix.dev) + + + +##### **Options:** + +- `--token ` + + The token to use (for authentication with prefix.dev) + + +- `--username ` + + The username to use (for basic HTTP authentication) + + +- `--password ` + + The password to use (for basic HTTP authentication) + + +- `--conda-token ` + + The token to use on anaconda.org / quetz authentication + + + + + +#### `logout` + +Remove authentication information for a given host + +**Usage:** `rattler-build auth logout ` + +##### **Arguments:** + +- `` + + The host to remove authentication for + + + + + +
+ + + This document was generated automatically by + clap-markdown. + + diff --git a/docs/available_jinja.md b/docs/reference/jinja.md similarity index 100% rename from docs/available_jinja.md rename to docs/reference/jinja.md diff --git a/docs/recipe_file.md b/docs/reference/recipe_file.md similarity index 99% rename from docs/recipe_file.md rename to docs/reference/recipe_file.md index c6af1004d..49ac1b0f1 100644 --- a/docs/recipe_file.md +++ b/docs/reference/recipe_file.md @@ -79,7 +79,7 @@ The schema (and `pydantic` source file) can be found in this repository: Read more about this [here](https://github.com/redhat-developer/yaml-language-server). -See more in the [automatic linting](./automatic_linting.md) chapter. +See more in the [automatic linting](../automatic_linting.md) chapter.