From fa091d4536904a9674c2eb6f4c5016d12f9bd60a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Tr=C4=85tnowiecki?= Date: Tue, 10 Dec 2024 21:43:09 +0100 Subject: [PATCH] Add include field to the manifest fixes #1770 commit-id:d297a417 --- scarb/src/core/manifest/mod.rs | 1 + scarb/src/core/manifest/toml_manifest.rs | 11 ++ .../core/publishing/manifest_normalization.rs | 1 + scarb/src/core/publishing/source.rs | 9 +- scarb/src/ops/package.rs | 5 +- scarb/tests/package.rs | 111 ++++++++++++++++++ website/docs/reference/manifest.md | 14 +++ website/docs/registries/publishing.md | 32 ++++- 8 files changed, 175 insertions(+), 9 deletions(-) diff --git a/scarb/src/core/manifest/mod.rs b/scarb/src/core/manifest/mod.rs index bdeb0c175..013916113 100644 --- a/scarb/src/core/manifest/mod.rs +++ b/scarb/src/core/manifest/mod.rs @@ -75,6 +75,7 @@ pub struct ManifestMetadata { pub license_file: Option, pub readme: Option, pub repository: Option, + pub include: Option>, #[serde(rename = "tool")] pub tool_metadata: Option>, pub cairo_version: Option, diff --git a/scarb/src/core/manifest/toml_manifest.rs b/scarb/src/core/manifest/toml_manifest.rs index 64485e847..1e0c688bd 100644 --- a/scarb/src/core/manifest/toml_manifest.rs +++ b/scarb/src/core/manifest/toml_manifest.rs @@ -197,6 +197,7 @@ pub struct TomlPackage { pub license_file: Option>, pub readme: Option>, pub repository: Option>, + pub include: Option>, /// **UNSTABLE** This package does not depend on Cairo's `core`. pub no_core: Option, pub cairo_version: Option>, @@ -571,6 +572,16 @@ impl TomlManifest { .clone() .map(|mw| mw.resolve("repository", || inheritable_package.repository())) .transpose()?, + include: package + .include + .clone() + .map(|include| { + include + .into_iter() + .map(|path| abs_canonical_path("include path", manifest_path, &path)) + .collect::>>() + }) + .transpose()?, cairo_version: package .cairo_version .clone() diff --git a/scarb/src/core/publishing/manifest_normalization.rs b/scarb/src/core/publishing/manifest_normalization.rs index 14e41e0ac..6fd868eaf 100644 --- a/scarb/src/core/publishing/manifest_normalization.rs +++ b/scarb/src/core/publishing/manifest_normalization.rs @@ -73,6 +73,7 @@ fn generate_package(pkg: &Package) -> Box { .clone() .map(|_| MaybeWorkspace::Defined((Utf8PathBuf::from(DEFAULT_README_FILE_NAME)).into())), repository: metadata.repository.clone().map(MaybeWorkspace::Defined), + include: None, no_core: summary.no_core.then_some(true), cairo_version: metadata.cairo_version.clone().map(MaybeWorkspace::Defined), experimental_features: pkg.manifest.experimental_features.clone(), diff --git a/scarb/src/core/publishing/source.rs b/scarb/src/core/publishing/source.rs index 2416bb232..e04d17c8f 100644 --- a/scarb/src/core/publishing/source.rs +++ b/scarb/src/core/publishing/source.rs @@ -77,8 +77,13 @@ fn push_worktree_files(pkg: &Package, ret: &mut Vec) -> Result<()> true } }; - - WalkBuilder::new(pkg.root()) + let mut builder = WalkBuilder::new(pkg.root()); + if let Some(include) = pkg.manifest.metadata.include.as_ref() { + for path in include { + builder.add(path); + } + } + builder .follow_links(true) .standard_filters(true) .parents(false) diff --git a/scarb/src/ops/package.rs b/scarb/src/ops/package.rs index c01ceda23..8e81898a2 100644 --- a/scarb/src/ops/package.rs +++ b/scarb/src/ops/package.rs @@ -402,7 +402,10 @@ fn source_files(pkg: &Package) -> Result { list_source_files(pkg)? .into_iter() .map(|on_disk| { - let path = on_disk.strip_prefix(pkg.root())?.to_owned(); + let path = on_disk + .strip_prefix(pkg.root()) + .with_context(|| format!("file `{on_disk}` is not part of `{}`", pkg.id.name))? + .to_owned(); Ok(ArchiveFile { path, contents: ArchiveFileContents::OnDisk(on_disk), diff --git a/scarb/tests/package.rs b/scarb/tests/package.rs index 00d3cb279..3b6475650 100644 --- a/scarb/tests/package.rs +++ b/scarb/tests/package.rs @@ -1525,3 +1525,114 @@ fn package_with_publish_disabled() { [..]Packaged [..] files, [..] ([..] compressed) "#}); } + +#[test] +fn can_include_additional_files() { + let t = TempDir::new().unwrap(); + simple_project() + .manifest_package_extra(indoc! {r#" + include = ["target/file.txt", "target/some/", "other/file.txt", "other/some"] + "#}) + .build(&t); + + t.child("target/file.txt") + .write_str("some file content") + .unwrap(); + t.child("target/some/file.txt") + .write_str("some file content") + .unwrap(); + t.child("other/file.txt") + .write_str("some file content") + .unwrap(); + t.child("other/some/dir/file.txt") + .write_str("some file content") + .unwrap(); + t.child("other/some/dir/other.txt") + .write_str("some file content") + .unwrap(); + + t.child(".gitignore").write_str("target").unwrap(); + t.child(".scarbignore").write_str("other").unwrap(); + + Scarb::quick_snapbox() + .arg("package") + .arg("--no-metadata") + .current_dir(&t) + .assert() + .success() + .stdout_matches(indoc! {r#" + [..] Packaging foo v1.0.0 [..] + [..] Verifying foo-1.0.0.tar.zst + [..] Compiling foo v1.0.0 ([..]) + [..] Finished `dev` profile target(s) in [..] + [..] Packaged [..] files, [..] ([..] compressed) + "#}); + + PackageChecker::assert(&t.child("target/package/foo-1.0.0.tar.zst")) + .name_and_version("foo", "1.0.0") + .contents(&[ + "VERSION", + "Scarb.orig.toml", + "Scarb.toml", + "src/lib.cairo", + "src/foo.cairo", + "other/some/dir/other.txt", + "other/some/dir/file.txt", + "other/file.txt", + "target/some/file.txt", + "target/file.txt", + ]) + .file_eq("VERSION", "1") + .file_eq_path("Scarb.orig.toml", t.child("Scarb.toml")) + .file_eq_path("src/lib.cairo", t.child("src/lib.cairo")) + .file_eq_path("src/foo.cairo", t.child("src/foo.cairo")) + .file_matches_nl( + "Scarb.toml", + indoc! {r#" + # Code generated by scarb package -p foo; DO NOT EDIT. + # + # When uploading packages to the registry Scarb will automatically + # "normalize" Scarb.toml files for maximal compatibility + # with all versions of Scarb and also rewrite `path` dependencies + # to registry dependencies. + # + # If you are reading this file be aware that the original Scarb.toml + # will likely look very different (and much more reasonable). + # See Scarb.orig.toml for the original contents. + + [package] + name = "foo" + version = "1.0.0" + edition = "2023_01" + + [dependencies] + "#}, + ); +} + +#[test] +fn files_outside_package_cannot_be_included() { + let t = TempDir::new().unwrap(); + let pkg = t.child("pkg"); + simple_project() + .manifest_package_extra(indoc! {r#" + include = ["../some/file.txt"] + "#}) + .build(&pkg); + t.child("some/file.txt") + .write_str("some file content") + .unwrap(); + Scarb::quick_snapbox() + .arg("package") + .arg("--no-metadata") + .current_dir(&pkg) + .assert() + .failure() + .stdout_matches(indoc! {r#" + [..] Packaging foo v1.0.0 [..] + error: file `[..]file.txt` is not part of `foo` + + Caused by: + prefix not found + "#}); +} diff --git a/website/docs/reference/manifest.md b/website/docs/reference/manifest.md index 9bcb3bbcd..7ecc6e9dc 100644 --- a/website/docs/reference/manifest.md +++ b/website/docs/reference/manifest.md @@ -100,6 +100,20 @@ Setting the `cairo-version` key in `[package]` will affect all targets in the pa The value in this field will not affect the version of the compiler run by Scarb. Scarb always uses its built-in version of the Cairo compiler. +### `include` + +When packaging a package with `scarb package` command (see +[packaging your package](../registries/publishing.md#packaging-your-package)), all files excluded with rules from +`.gitignore` or `.scarbignore` files are not included in the resulting package tarball. +This field can be used mark files and subdirectories that should be included in the package tarball, even if those files +would be excluded by rules from ignore files. +The paths are relative to the package root and cannot point to files outside the package. + +```toml +[package] +include = ["target/some/file.txt"] +``` + ### `authors` This optional field lists the people or organizations that are considered the "authors" of the package. diff --git a/website/docs/registries/publishing.md b/website/docs/registries/publishing.md index 6a400ed74..a8a2faef1 100644 --- a/website/docs/registries/publishing.md +++ b/website/docs/registries/publishing.md @@ -7,9 +7,11 @@ Once uploaded, it will be available for other users to download and use. To upload your package, use the scarb publish command. By default, this command will publish your package to the official [scarbs.xyz](https://scarbs.xyz) registry. -The publish command automatically [packages and verifies](#packaging-your-package) your package, so there is no need to run `scarb package` beforehand. +The publish command automatically [packages and verifies](#packaging-your-package) your package, so there is no need to +run `scarb package` beforehand. -To publish your package to a registry that supports package publishing, you need to authenticate using an API token with the `publish` scope. +To publish your package to a registry that supports package publishing, you need to authenticate using an API token with +the `publish` scope. First, log in to the registry and [in the dashboard](https://scarbs.xyz/dashboard) generate the API token. Scarb will use the token to authenticate and complete the publishing process. The token must be provided via the `SCARB_REGISTRY_AUTH_TOKEN` environment variable. @@ -44,17 +46,35 @@ publish = false Use the `scarb package` command to create an archive of your package. You can read about the package compression algorithm and contents in the [Package tarball](./package-tarball) section. -Basically when you run the command, Scarb gathers the source code of your package along with metadata files, such as the manifest file, and places them in an archive in `target/package` directory. +Basically when you run the command, Scarb gathers the source code of your package along with metadata files, such as the +manifest file, and places them in an archive in `target/package` directory. -If you are in a Git repository, Scarb will first check if the repo state is clean and error out in case of any changes present in the Git working directory. +If you are in a Git repository, Scarb will first check if the repo state is clean and error out in case of any changes +present in the Git working directory. To bypass this check, you can use the `--allow-dirty` flag. The next step is package verification. -After creating the initial archive, Scarb will attempt to unpack it and compile to check for any corruptions in the packaging process. +After creating the initial archive, Scarb will attempt to unpack it and compile to check for any corruptions in the +packaging process. If you want to speed up the packaging process, you can disable this step using the `--no-verify` flag. > [!WARNING] > This is a dangerous operation as it can lead to uploading a corrupted package to the registry. > Please use with caution. -After successfully completing the whole process, the `{name}-{version}.tar.zst` archive waits in the `target/package` directory for being uploaded, where both `name` and `version` correspond to the values in `Scarb.toml`. +After successfully completing the whole process, the `{name}-{version}.tar.zst` archive waits in the `target/package` +directory for being uploaded, where both `name` and `version` correspond to the values in `Scarb.toml`. + +### Files included in the package + +All files in the package directory are included in the resulting tarball, except for the following: + +- Files excluded with rules defined in any `.scarbignore`, `.gitignore` or `.ignore` files. +- The `.git` directory. +- Any subdirectories containing `Scarb.toml` file. +- The `/target` directory. +- Symlinks within the package directory are followed, while symlinks outside are ignored. +- File system boundaries are not crossed. + +Files that would be otherwise ignored by the rules listed above, can still be included +with [include](../reference/manifest.md#include) field.