From 719f105ec1b0fe0c939089f8d08d6328fac22df1 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 9 Dec 2022 22:40:09 +0000 Subject: [PATCH 01/71] Functions MVP --- .github/CODEOWNERS | 1 + .github/dependabot.yml | 10 + .github/workflows/check_changelog.yml | 24 + .github/workflows/ci.yml | 61 + .gitignore | 3 + Cargo.lock | 1331 +++++++++++++++++ Cargo.toml | 30 + README.md | 2 + buildpack.toml | 21 + notes.md | 330 ++++ src/errors.rs | 306 ++++ src/functions.rs | 117 ++ src/layers/mod.rs | 3 + src/layers/pip_cache.rs | 75 + src/layers/pip_dependencies.rs | 160 ++ src/layers/python.rs | 306 ++++ src/main.rs | 153 ++ src/package_manager.rs | 33 + src/project_descriptor.rs | 281 ++++ src/python_version.rs | 104 ++ src/runtime_txt.rs | 188 +++ src/utils.rs | 80 + test-fixtures/default/requirements.txt | 0 test-fixtures/empty/.gitkeep | 0 .../function_invalid_not_async/main.py | 5 + .../function_invalid_not_async/project.toml | 2 + .../requirements.txt | 5 + .../main.py | 5 + .../project.toml | 2 + .../requirements.txt | 1 + test-fixtures/function_python_3.10/main.py | 20 + .../function_python_3.10/project.toml | 2 + .../function_python_3.10/requirements.txt | 5 + .../function_python_3.10/runtime.txt | 1 + .../function_python_version_invalid/main.py | 5 + .../project.toml | 2 + .../requirements.txt | 5 + .../runtime.txt | 1 + .../function_python_version_too_old/main.py | 5 + .../project.toml | 2 + .../requirements.txt | 5 + .../runtime.txt | 1 + .../main.py | 5 + .../project.toml | 2 + .../requirements.txt | 5 + .../runtime.txt | 1 + test-fixtures/function_template/README.md | 3 + test-fixtures/function_template/main.py | 20 + test-fixtures/function_template/payload.json | 1 + test-fixtures/function_template/project.toml | 9 + .../function_template/requirements.txt | 5 + .../project_toml_invalid/project.toml | 5 + .../project_toml_non_salesforce/project.toml | 8 + tests/integration.rs | 369 +++++ 54 files changed, 4126 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/check_changelog.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 buildpack.toml create mode 100644 notes.md create mode 100644 src/errors.rs create mode 100644 src/functions.rs create mode 100644 src/layers/mod.rs create mode 100644 src/layers/pip_cache.rs create mode 100644 src/layers/pip_dependencies.rs create mode 100644 src/layers/python.rs create mode 100644 src/main.rs create mode 100644 src/package_manager.rs create mode 100644 src/project_descriptor.rs create mode 100644 src/python_version.rs create mode 100644 src/runtime_txt.rs create mode 100644 src/utils.rs create mode 100644 test-fixtures/default/requirements.txt create mode 100644 test-fixtures/empty/.gitkeep create mode 100644 test-fixtures/function_invalid_not_async/main.py create mode 100644 test-fixtures/function_invalid_not_async/project.toml create mode 100644 test-fixtures/function_invalid_not_async/requirements.txt create mode 100644 test-fixtures/function_missing_functions_package/main.py create mode 100644 test-fixtures/function_missing_functions_package/project.toml create mode 100644 test-fixtures/function_missing_functions_package/requirements.txt create mode 100644 test-fixtures/function_python_3.10/main.py create mode 100644 test-fixtures/function_python_3.10/project.toml create mode 100644 test-fixtures/function_python_3.10/requirements.txt create mode 100644 test-fixtures/function_python_3.10/runtime.txt create mode 100644 test-fixtures/function_python_version_invalid/main.py create mode 100644 test-fixtures/function_python_version_invalid/project.toml create mode 100644 test-fixtures/function_python_version_invalid/requirements.txt create mode 100644 test-fixtures/function_python_version_invalid/runtime.txt create mode 100644 test-fixtures/function_python_version_too_old/main.py create mode 100644 test-fixtures/function_python_version_too_old/project.toml create mode 100644 test-fixtures/function_python_version_too_old/requirements.txt create mode 100644 test-fixtures/function_python_version_too_old/runtime.txt create mode 100644 test-fixtures/function_python_version_unavailable/main.py create mode 100644 test-fixtures/function_python_version_unavailable/project.toml create mode 100644 test-fixtures/function_python_version_unavailable/requirements.txt create mode 100644 test-fixtures/function_python_version_unavailable/runtime.txt create mode 100644 test-fixtures/function_template/README.md create mode 100644 test-fixtures/function_template/main.py create mode 100644 test-fixtures/function_template/payload.json create mode 100644 test-fixtures/function_template/project.toml create mode 100644 test-fixtures/function_template/requirements.txt create mode 100644 test-fixtures/project_toml_invalid/project.toml create mode 100644 test-fixtures/project_toml_non_salesforce/project.toml create mode 100644 tests/integration.rs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..25f6404 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @heroku/languages diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..98e44ee --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/check_changelog.yml b/.github/workflows/check_changelog.yml new file mode 100644 index 0000000..d9967a8 --- /dev/null +++ b/.github/workflows/check_changelog.yml @@ -0,0 +1,24 @@ +name: Check Changelog + +on: + pull_request: + types: [opened, reopened, edited, labeled, unlabeled, synchronize] + +permissions: + contents: read + +jobs: + check-changelog: + runs-on: ubuntu-22.04 + if: | + !contains(github.event.pull_request.body, '[skip changelog]') && + !contains(github.event.pull_request.body, '[changelog skip]') && + !contains(github.event.pull_request.body, '[skip ci]') && + !contains(github.event.pull_request.labels.*.name, 'skip changelog') + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Check that CHANGELOG is touched + run: | + git fetch origin ${{ github.base_ref }} --depth 1 && \ + git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f149776 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,61 @@ +name: CI + +on: + push: + # Avoid duplicate builds on PRs. + # TODO: Uncomment once this is merged to `main`. + # branches: + # - main + pull_request: + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + lint: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Update Rust toolchain + run: rustup update + - name: Rust Cache + uses: Swatinem/rust-cache@v2.2.0 + - name: Clippy + run: cargo clippy --all-targets --locked -- --deny warnings + - name: rustfmt + run: cargo fmt -- --check + + unit-test: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Update Rust toolchain + run: rustup update + - name: Rust Cache + uses: Swatinem/rust-cache@v2.2.0 + - name: Run unit tests + run: cargo test --locked + + integration-test: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install musl-tools + run: sudo apt-get install musl-tools --no-install-recommends + - name: Update Rust toolchain + run: rustup update + - name: Install Rust linux-musl target + run: rustup target add x86_64-unknown-linux-musl + - name: Rust Cache + uses: Swatinem/rust-cache@v2.2.0 + - name: Install Pack CLI + uses: buildpacks/github-actions/setup-pack@v4.9.0 + - name: Run integration tests + # Runs only tests annotated with the `ignore` attribute (which in this repo, are the integration tests). + run: cargo test --locked -- --ignored diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..280b2ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +target/ +.DS_Store diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..58912fe --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1331 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bollard" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82e7850583ead5f8bbef247e2a3c37a19bd576e8420cd262a6711921827e1e5" +dependencies = [ + "base64", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http", + "hyper", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.42.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed59b5c00048f48d7af971b71f800fdf23e858844a6f9e4d32ca72e9399e7864" +dependencies = [ + "serde", + "serde_with", +] + +[[package]] +name = "bumpalo" +version = "3.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" + +[[package]] +name = "bytes" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" + +[[package]] +name = "camino" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ad0e1e3e88dd237a156ab9f571021b8a158caa0ae44b1968a241efb5144c1e" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982a0cf6a99c350d7246035613882e376d58cebe571785abc5da4f648d53ac0a" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cc" +version = "1.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + +[[package]] +name = "fancy-regex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0678ab2d46fa5195aaf59ad034c083d351377d4af57f3e073c074d0da3e3c766" +dependencies = [ + "bit-set", + "regex", +] + +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] +name = "filetime" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys", +] + +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "libz-sys", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + +[[package]] +name = "futures-channel" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" + +[[package]] +name = "futures-macro" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" + +[[package]] +name = "futures-task" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" + +[[package]] +name = "futures-util" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "h2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyperlocal" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" +dependencies = [ + "futures-util", + "hex", + "hyper", + "pin-project", + "tokio", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adab1eaa3408fb7f0c777a73e7465fd5656136fc93b670eb6df3c88c2c1344e3" + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" + +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.138" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" + +[[package]] +name = "libcnb" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88e7663908798e4c9a6ce419220e9493ad19dd12a70fa605dd4e927e3fdc1fc9" +dependencies = [ + "libcnb-data", + "libcnb-proc-macros", + "serde", + "stacker", + "thiserror", + "toml", +] + +[[package]] +name = "libcnb-data" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "066632abe99f4ca2de170a3c3f5946253e63c715e9b7b1a31341de883cb03246" +dependencies = [ + "fancy-regex", + "libcnb-proc-macros", + "serde", + "thiserror", + "toml", +] + +[[package]] +name = "libcnb-package" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa1d102e9d744212b2bfe650a5012fb93609700642fde5e778dc27103fc1b26e" +dependencies = [ + "cargo_metadata", + "libcnb-data", + "toml", + "which", +] + +[[package]] +name = "libcnb-proc-macros" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092a9091dc629e2fafb9a24e7b9050ca3e84565f82c5a67aecaf8afa24234d32" +dependencies = [ + "cargo_metadata", + "fancy-regex", + "quote", + "syn", +] + +[[package]] +name = "libcnb-test" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "318a615aed63ac85117eb0c149105ad9ca956cd3133d11d1a83fb011941802f8" +dependencies = [ + "bollard", + "cargo_metadata", + "fastrand", + "fs_extra", + "libcnb-data", + "libcnb-package", + "serde", + "tempfile", + "tokio", + "tokio-stream", +] + +[[package]] +name = "libherokubuildpack" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bffa56c2d7cf1c9265126f368343d6d5bf34c33da7ddf0d5ac245da9e999265" +dependencies = [ + "termcolor", +] + +[[package]] +name = "libz-sys" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys", +] + +[[package]] +name = "num_cpus" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + +[[package]] +name = "python-buildpack" +version = "0.0.0" +dependencies = [ + "flate2", + "indoc", + "libcnb", + "libcnb-test", + "libherokubuildpack", + "serde", + "tar", + "toml", + "ureq", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustls" +version = "0.20.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "semver" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +dependencies = [ + "serde", +] + +[[package]] +name = "serde" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "stacker" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "winapi", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "tokio-stream" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f" +dependencies = [ + "base64", + "chunked_transfer", + "log", + "once_cell", + "rustls", + "url", + "webpki", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" +dependencies = [ + "webpki", +] + +[[package]] +name = "which" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +dependencies = [ + "either", + "libc", + "once_cell", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..c7e282b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "python-buildpack" +version = "0.0.0" +edition = "2021" +rust-version = "1.65" +publish = false + +[dependencies] +# The default `miniz_oxide` flate2 backend has poor performance in debug/under QEMU: +# https://github.com/rust-lang/flate2-rs/issues/297 +# Ideally we'd use the fastest `zlib-ng` backend, however it fails to cross-compile: +# https://github.com/rust-lang/libz-sys/issues/93 +# As such we have to use the next best alternate backend, which is `zlib`. +flate2 = { version = "1.0.25", default-features = false, features = ["zlib"] } +indoc = "1.0.7" +libcnb = "0.11.1" +libherokubuildpack = { version = "0.11.1", default-features = false, features = ["log"] } +serde = "1.0.149" +tar = "0.4.38" +toml = "0.5.9" +ureq = { version = "2.5.0", default-features = false, features = ["tls"] } + +[dev-dependencies] +libcnb-test = "0.11.1" + +# [profile.dev] +# Speed up downloading/extraction of Python during integration tests. +# TODO: Test again to see if it's still worth it, now that the Python archives are smaller + using alternate flate2 backend. +# (now only seems to change the E2E pack build time of an app using urllib3 from 23.4s to 21.8s on M1?) +# opt-level = 1 diff --git a/README.md b/README.md index 757b374..dd6ba99 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ # Heroku Cloud Native Buildpack for Python +[![CI](https://github.com/heroku/buildpacks-python/actions/workflows/ci.yml/badge.svg)](https://github.com/heroku/buildpacks-python/actions/workflows/ci.yml) + Heroku's official [Cloud Native Buildpack](https://buildpacks.io) for the Python ecosystem. diff --git a/buildpack.toml b/buildpack.toml new file mode 100644 index 0000000..6b5ecfa --- /dev/null +++ b/buildpack.toml @@ -0,0 +1,21 @@ +api = "0.8" + +[buildpack] +# The buildpack ID here is temporary, for the Python functions alpha/beta. +# TODO: Change it back to `heroku/python` once the buildpack is ready for non-functions use. +id = "heroku/python-functions-experimental" +version = "0.1.0" +name = "Python" +homepage = "https://github.com/heroku/buildpacks-python" +description = "Heroku's official Python Cloud Native Buildpack." +keywords = ["python", "heroku"] +clear-env = true + +[[buildpack.licenses]] +type = "BSD-3-Clause" + +[[stacks]] +id = "heroku-20" + +[[stacks]] +id = "heroku-22" diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..236b1a1 --- /dev/null +++ b/notes.md @@ -0,0 +1,330 @@ +## Python package resolution +What are all of the ways packages end up on sys.path? And in what order? +-> Script/current dir, PYTHONPATH, user site-packages (incl `.pth` and `usercustomize`), system site-packages (incl `.pth` and `sitecustomize`) +Can system site-packages location be overridden? +-> Not really, since needs to be same as libs etc +Can user site-packages be overridden? +-> Yes, using `PYTHONUSERBASE` +What deps do Pip, Poetry and pipenv have? Can the tools be installed outside of the env they are managing? +-> Pip: None (it vendors). Managing deps outside of a venv is not supported (other than `--target` and perhaps `--prefix`). See: https://github.com/pypa/pip/issues/5472 +-> Poetry: lots! However installing in a venv is both supported and recommended. +-> Pipenv: lots! However installing in a venv is supported and kinda recommended. +Is pip needed when installing Poetry/pipenv? +-> Poetry: Yes +-> Pipenv: Yes +What deps will the Python invoker have? (ie can cause conflicts) Or fully vendored / in Rust? +-> TBD +How do user installs work when there are conflicting dependencies? Can they be used inside a virtualenv? +-> Seems to work well. And no, can't be used in a venv. See https://pip.pypa.io/en/stable/user_guide/#user-installs +What approaches do other CNBs use? +-> GCP: Other buildpacks put their `requirements.txt` files into the build plan and then a single pip-install CNB installs it. Prior to that that they tried using `--prefix` and `--target` with `PYTHONPATH`. They cannot use `PYTHONUSERBASE` fully due to compatibility issues with their GAE image using system Python and having to use virtualenvs (which don't support user installs). +-> Paketo: `PYTHONUSERBASE` to set install location during pip install of pip/deps, but then `PYTHONPATH` afterwards. They used to use `PYTHONUSERBASE` for both but changed in https://github.com/paketo-buildpacks/pip-install/pull/58 to "allow other buildpacks to use `PYTHONUSERBASE`" -- seems like they perhaps haven't realised about the `PYTHONPATH` shadowing stdlib issues? +What are the issues with using `--target` and `--prefix` that meant GCP stopped using them? +-> https://github.com/GoogleCloudPlatform/buildpacks/commit/7768ebe4d5f300598b86328f607eeb70ab7b7131 +-> https://github.com/GoogleCloudPlatform/buildpacks/commit/410b552aba55404bdb45acb638112feb271de01f +-> https://github.com/GoogleCloudPlatform/buildpacks/commit/b93391cd653eef7336bc154466fa6d3de4ed337b +-> https://github.com/pypa/pip/issues/8799 +So what are the alternatives for where to install packages? +-> New venv (w/wo Pip / system site-packages) +-> Arbitrary directory and point at it with `PYTHONPATH` +-> Arbitrary directory used as user install location with `PYTHONUSERBASE` +-> System site-packages in same layer as Python runtime +-> Arbitrary directory and point at it with `.pth` file from user/system site-packages +Resources: +https://peps.python.org/pep-0370/ +https://docs.python.org/3.10/library/site.html +https://docs.python.org/3.10/install/index.html#alternate-installation +https://docs.python.org/3.10/using/cmdline.html#envvar-PYTHONNOUSERSITE +https://docs.python.org/3.10/using/cmdline.html#envvar-PYTHONPATH +https://docs.python.org/3.10/library/sys.html#sys.path +https://docs.python.org/3.10/library/sysconfig.html#installation-paths +https://docs.python.org/3.11/library/sys_path_init.html#sys-path-init + +## Installation locations +- Pip/setuptools/wheel: System site-packages in same layer as Python runtime +- Poetry/Pipenv (if applicable): Venv using `--symlinks --system-site-packages --without-pip` (using `--without-pip` saves ~8.5 MB and 1.6s on macOS). Must install using `python -m pip`. +- App dependencies: User site-packages +- Function invoker (if in Python): Arbitrary directory added to `PYTHONPATH` or make the user install + +## Installing dependencies with pip +- Do we support having no package manager being used? +-> TBD +- Does a single layer handle all install types, or separate layer per package manager? +-> Separate +- When to cache/invalidate site-packages? +-> Invalidation needed to clean up removed packages (otherwise have to manually remove), and ensure unpinned deps are updated (if not using --upgrade) +- Should the pip cache also be cached? If so, when to invalidate that? +-> Helps when cached site-packages invalidated, or if a previously used package added back +- Should we use `--upgrade`? +-> Pros: Ensures unpinned deps stay up to date. Might mean we don't need to invalidate site-packages as often. +-> Cons: Causes pip to still query PyPI even for `==` deps. +-> Are people using `--upgrade` locally? +- What is the perf impact of caching site-packages vs pip cache? What about `--upgrade`? +- Options: `pip install --user --disable-pip-version-check --cache-dir --no-input` +- What about `requirements.txt` files with an include? +- Do we need to use `--exists-action`? +- No way to purge pip cache of items older than X (https://github.com/pypa/pip/issues/8355) + +curl -O https://raw.githubusercontent.com/mozilla/treeherder/master/requirements/common.txt +rm -rf venv /root/.cache/pip/ && python -m venv --symlink venv && time venv/bin/pip install --disable-pip-version-check -r common.txt -q --no-cache-dir + +## When does site-packages need invalidating? +- Python version changed (any, or just major?) +-> Yes, perhaps any? +- Stack changed +-> Yes +- Pip/setuptools/wheel version changed? +-> Don't think so +- requirements.txt changes + +## Should we use `--upgrade` or `--upgrade --upgrade-strategy eager`? +- Pros: + - Means updated versions of unpinned packages (or unspecified transitive deps) are pulled in (without invalidating site-packages) + - Means pip logs show what changed (vs invalidating site-packages) +- Cons: + - Pip still queries PyPI for `==` pinned deps, slowing otherwise no-op runs. + - If an updated package drops a dep, then that dep isn't uninstalled (vs invalidating site-packages). + - Using `--upgrade --upgrade-strategy eager` results in errors for projects using hashes where a dependency has a transitive dep on setuptools (such as gunicorn) +- Other: + - Updates are pulled in immediately rather than after a delay + - Does `--upgrade` match what people are using locally? + - Does pip handle transitive dep updates any differently from empty site-packages? + +## Should we invalidate on root requirements.txt changes +- Yes! Have to otherwise package removals don't work. + +## What isn't handled when invalidating on root requirements.txt changes when not using `--upgrade`? +- Updated versions of unpinned packages (or unspecified transitive deps) are not pulled in +- Removals from transitive requirements.txt files (unless we scan for those too) +- Explicit package updates that drop a dep, in transitive requirements.txt files (unless we scan for those too) + +## What isn't handled when invalidating on root requirements.txt changes when using `--upgrade`? +- If an implicitly updated package drops a dep, then that dep isn't uninstalled (vs invalidating site-packages). +- Removals from transitive requirements.txt files (unless we scan for those too) +- Explicit package updates that drop a dep, in transitive requirements.txt files (unless we scan for those too) + +## How could we handle transitive requirements.txt files? +- Scan root requirements.txt for `-r ...` usages and check for changes to those too +- Output a warning if `-r ...` usages found and encourage users to stop using them or switch to eg Poetry +- Offer alternative locations to just the repo root, hoping people would use those instead of includes? (But doesn't cover all use-cases eg common deps) + +## Timings for treeherder's common.txt (Python 3.9, in venv, wheel installed, --disable-pip-version-check) +- Clean install, --no-cache-dir: 37.3s +- Clean install, cold cache: 37.8s +- Clean install, warm cache (all): 33.7s (however zstandard cached built wheel not used due to hashes) +- No-op repeat install, --no-cache, no upgrade: 0.61s +- No-op repeat install, warm cache, no upgrade: 0.61s +- No-op repeat install, --no-cache, --upgrade: 3.3s +- No-op repeat install, warm cache, --upgrade: 3.3s + +## Timings for treeherder's common.txt with hashes removed (Python 3.9, in venv, wheel installed, --disable-pip-version-check) +- Clean install, --no-cache-dir: 37.8s +- Clean install, cold cache: 37.8s +- Clean install, warm cache (all): 9.0s (without wheel installed this increases to 12.9s) +- Clean install, warm cache (3 MB wheel dir only): 12.8s +- Clean install, warm cache (72 MB http dir only): 33.9s + +## Timings for getting-started-guide's requirements.txt (Python 3.9, in venv, wheel installed, --disable-pip-version-check) +- Clean install, --no-cache-dir: 5.6s +- Clean install, cold cache: 5.7s +- Clean install, warm cache (all): 1.4s +- Clean install, warm cache (0.5 MB wheel dir only): 1.9s +- Clean install, warm cache (8.7 MB http dir only): 5.1s +- No-op repeat install, warm cache, no upgrade: 0.28s + +## Pip cache conclusions +- Wheel generation is where most of the time is spent (on a fast connection at least) +- If caching pip cache must have wheel installed or wheels won't be cached properly +- Could just cache wheels directory of pip cache since fraction of the size for most of the benefit. But wouldn't help slow connections. +- Invalidating site-packages increases install time from: 0.25s -> 1.4s (small project), 0.6s -> 9s (large project), 0.6s -> 34s (large project using hashes) +- Invalidating pip cache too increases install time from: 1.4s -> 5.7s (small project), 9s -> 38s (large project), 34s -> 38s (large project using hashes) +- Pip hashes really impact caching - should we output a warning? + +## Possible layer invalidation conditions +- Python version (either only when the major version changes, or also including minor version changes) +- Stack +- pip/setuptools/wheel version +- Poetry/pipenv version +- Input files from app (eg requirements.txt/Poetry.lock hash) +- Time since layer created +- Buildpack changes that aren't backwards compatible with old caches + +## Layer scenarios +- Initial install: `build()` -> `create()` +- Keeping cached layer: `build()` -> `existing_layer_strategy()` +- Recreating cached layer: `build()` -> `existing_layer_strategy()` -> `create()` +- Updating cached layer: `build()` -> `existing_layer_strategy()` -> `update()` + +## Logging +- What do users care about in the logs? + - If something went wrong, what it was, whether it was their fault or not, and how to resolve + - What is happening in general, so it doesn't seem like a black box + - How behaviour can be customised + - Why has behaviour changed since last build, particularly if something is now broken. +- When to use headings vs not? +- Should there always be a "doing thing" and "finished thing" message or just one or the other? +- How verbose should the logs be (particularly for output from subprocesses)? +- Should the verbosity be user controllable? Should we ask for a standard env var upstream? +- What should the logs show for using cache vs invalidating cache? + +## Errors +- Remove unwraps throughout and replace with new error enum variants +- How fine grained should the io::Error instances be? +- should layer errors be flattened into the top level buildpack error enum, or have their own error enums? +- Should the error `From` implementations live with the error enums (eg in the layer), or in errors.rs? +- What if anything should be covered by retries? Presumably only things involving network I/O? How well do pip's retries work? + +## Misc +- Utils for calling subprocesses +- Clear the env when calling subprocesses too (for most of them at least) +- What logic lives in the layer vs outside? +- Need to make Procfile mandatory given no default entrypoint. Although don't want to fail detect? +- Should set User Agent on outbound network requests +- Should we use https://docs.gunicorn.org/en/stable/settings.html#preload-app by default? + +## Unit tests +- What things do/don't need a unit test? +- Should the unit test cover lower down functions or their parents? + +## Integration tests +- Check Python static library works +- Check behaviour if buildpack run twice + +## Poetry +- Should it use a different layer name for the `site-packages` layer? + +## Improvements/decisions deferred to the future +- SHA256 checking of Python download. +- Decide whether to move pip/setuptools/wheel requirements to a requirements file so Dependabot can update them. + - However then means it's harder for us to list versions. + - Also, if integration tests include versions in log output and it's hardcoded, then Dependabot PRs will need manual updates anyway. +- Decide whether to use hashes for pip/setuptools/wheel requirements. + +## Python version support +- Do we support "3.*" / "*"", or just "3.x.*"? +- Do we support major version syntax in runtime.txt? +- Which of these other formats do we support? + - pyproject.toml's project.requires-python + - a new pyproject.toml table/property + - .python-version (with or w/o major version support?) + - tool.poetry.dependencies.python in pyproject.toml + - CNB project.toml file + +### pyproject.toml +[project] +requires-python = ">=3.8" +requires-python = "~=3.8" (means >=3.8, <4.0) +requires-python = "~=3.8.2" (means >=3.8.2, <3.9) +requires-python = "==3.8" (means ==3.8.0) +requires-python = "==3.8.*" +https://www.python.org/dev/peps/pep-0621/#requires-python +https://www.python.org/dev/peps/pep-0440/#version-specifiers +~=: Compatible release clause +==: Version matching clause +!=: Version exclusion clause +<=, >=: Inclusive ordered comparison clause +<, >: Exclusive ordered comparison clause +===: Arbitrary equality clause. + +### pyproject.toml +[tool.poetry.dependencies] +python = "^3.9" + +### .python-version +X.Y.Z +didn't used to support X.Y unless using a plugin, but now does: https://github.com/pyenv/pyenv#prefix-auto-resolution + +# pyc locations +- python stdlib +- pip/setuptools/wheel install in system site-packages +- app dependencies installed by pip in user site-packages +- poetry install in venv +- app dependencies installed by poetry in user site-packages +- app python files themselves in app dir + +# pyc alternatives +- timestamp (default) +- checked hash by disabling automatic compileall then running manually +- checked hash by setting SOURCE_DATE_EPOCH (only works via py_compile not by just running) +- unchecked hash by disabling automatic compileall then running manually +- delete the pyc files and let them be generated at build and/or app boot + +# pyc timings +- `python:3-slim`, native, `pip --version`, no pycs (creating timestamp): 0.628s +- `python:3-slim`, native, `pip --version`, no pycs (creating none): 0.571s +- `python:3-slim`, native, `pip --version`, existing timestamp: 0.151s +- `python:3-slim`, native, `pip --version`, existing checked: 0.161s +- `python:3-slim`, native, `pip --version`, existing unchecked: 0.152s +- `python:3-slim`, native, compileall pip dir, timestamp: 0.565s +- `python:3-slim`, native, compileall site-packages, checked: 0.637s +- `python:3-slim`, native, compileall site-packages, checked, workers=0: 0.199s +- `python:3-slim`, native, compileall python lib dir, timestamp: 1.277s +- `python:3-slim`, native, compileall python lib dir, checked: 1.275s +- `python:3-slim`, native, compileall python lib dir, checked, workers=0: 0.423s +- `python:3-slim`, qemu, `pip --version`, no pycs (creating timestamp): 5.475s +- `python:3-slim`, qemu, `pip --version`, no pycs (creating none): 5.357s +- `python:3-slim`, qemu, `pip --version`, existing timestamp: 1.360s +- `python:3-slim`, qemu, `pip --version`, existing checked: 1.386s +- `python:3-slim`, qemu, `pip --version`, existing unchecked: 1.356s +- `python:3-slim`, qemu, compileall pip dir, timestamp: 4.883s +- `python:3-slim`, qemu, compileall pip dir, checked: 4.869s +- `python:3-slim`, qemu, compileall python lib dir, timestamp: 11.682s +- `python:3-slim`, qemu, compileall python lib dir, checked: 11.708s +- `python:3-slim`, qemu, compileall python lib dir, checked, workers=0: 3.436s +- heroku gsg-ci, Perf-M, `pip --version`, existing timestamp: 0.202s +- heroku gsg-ci, Perf-M, `pip --version`, existing checked: 0.211s +- heroku gsg-ci, Perf-M, `pip --version`, existing unchecked: 0.202s +- heroku gsg-ci, Perf-M, `manage.py check`, existing timestamp: 0.283s +- heroku gsg-ci, Perf-M, `manage.py check`, existing checked: 0.299s +- heroku gsg-ci, Perf-M, `manage.py check`, existing unchecked: 0.282s + +Tested using: + +``` +find /app/.heroku/python/lib/python3.10/ -depth -type f -name "*.pyc" -delete +time python -m compileall -qq --invalidation-mode timestamp /app/.heroku/python/lib/python3.10/ +time python -m compileall -qq --invalidation-mode checked-hash /app/.heroku/python/lib/python3.10/ +time python -m compileall -qq --invalidation-mode unchecked-hash /app/.heroku/python/lib/python3.10/ +``` + +``` +find /usr/local -depth -type f -name "*.pyc" -delete +time python -m compileall -qq --invalidation-mode timestamp /usr/local/lib/python3.10/ +time python -m compileall -qq --invalidation-mode checked-hash /usr/local/lib/python3.10/ +time python -m compileall -qq --invalidation-mode unchecked-hash /usr/local/lib/python3.10/ +while true; do time pip --version; done +export SOURCE_DATE_EPOCH=1 +``` + +# Summary of runtime perf impact of checked vs unchecked pycs +- Native Docker, pip --version: +9ms on 152ms = +5.9% +- QEMU Docker, pip --version: +30ms on 1,356ms = +2.2% +- Heroku, pip --version: +9ms on 202ms = +4.5% +- Heroku, gsg manage.py check: +17ms on 282ms = +6.0% + +# pyc conclusion +- For Python runtime archive: delete all pycs, then regenerate using unchecked-hash +- For pip/setuptools/wheel: install using --no-compile, generate using unchecked-hash + concurrency +- For app dependencies installed using pip, either: + - Install using --no-compile, generate using unchecked-hash + concurrency + - Install using --no-compile, generate using checked-hash + concurrency + - Install normally, but ensure checked-hash by setting SOURCE_DATE_EPOCH +- For app dependencies installed using poetry (which doesn't support --no-compile), either: + - Install normally, but ensure checked-hash by setting SOURCE_DATE_EPOCH + - Install normally, then regenerate using unchecked-hash + concurrency + - Install normally, then regenerate using checked-hash + concurrency + +# bundled pip timings +- Bundled pip qemu: 5.2s for `--version` +- Bundled pip native: 0.6s for `--version` +- Unpacked pip qemu, without pycs: 3.3s for `--version` +- Unpacked pip native, without pycs: 0.4s for `--version` +- Unpacked pip qemu, with pycs: 1.4s for `--version` +- Unpacked pip native, with pycs: 0.2s for `--version` + +// before: +// time until pip install completed: 14.65s +// time until all completed (incl pycs): 16.65s +// after: +// time until pip install completed: 9.15s +// time until all completed (incl pycs): 11.15s diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..7f54893 --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,306 @@ +use crate::functions::{CheckFunctionError, FUNCTION_RUNTIME_PROGRAM_NAME}; +use crate::layers::pip_dependencies::PipDependenciesLayerError; +use crate::layers::python::PythonLayerError; +use crate::package_manager::DeterminePackageManagerError; +use crate::project_descriptor::ReadProjectDescriptorError; +use crate::python_version::{PythonVersionError, DEFAULT_PYTHON_VERSION}; +use crate::runtime_txt::{ParseRuntimeTxtError, ReadRuntimeTxtError}; +use crate::utils::{CommandError, DownloadUnpackArchiveError}; +use crate::BuildpackError; +use indoc::{formatdoc, indoc}; +use libherokubuildpack::log::log_error; +use std::io; + +/// Handle any non-recoverable buildpack or libcnb errors that occur. +/// +/// The buildpack will exit non-zero after this handler has run, so all that needs to be +/// performed here is the logging of an error message - and in the future, emitting metrics. +/// +/// We're intentionally not using `libherokubuildpack::error::on_error` since: +/// - It doesn't currently do anything other than logging an internal error for the libcnb +/// error case, and by inlining that here it's easier to keep the output consistent with +/// the messages emitted for buildpack-specific errors. +/// - Using it causes trait mismatch errors when Dependabot PRs incrementally update crates. +/// - When we want to add metrics to our buildpacks, it's going to need a rewrite of +/// `Buildpack::on_error` anyway (we'll need to write out metrics not log them, so will need +/// access to the `BuildContext`), at which point we can re-evaluate. +pub(crate) fn on_error(error: libcnb::Error) { + match error { + libcnb::Error::BuildpackError(buildpack_error) => on_buildpack_error(buildpack_error), + libcnb_error => log_error( + "Internal buildpack error", + formatdoc! {" + An unexpected internal error was reported by the framework used by this buildpack. + + Please open a support ticket and include the full log output of this build. + + Details: {libcnb_error} + "}, + ), + }; +} + +fn on_buildpack_error(buildpack_error: BuildpackError) { + match buildpack_error { + BuildpackError::CheckFunction(error) => on_check_function_error(error), + BuildpackError::DetectIo(io_error) => log_io_error( + "Unable to complete buildpack detection", + "determining if the Python buildpack should be run for this application", + &io_error, + ), + BuildpackError::DeterminePackageManager(error) => on_determine_package_manager_error(error), + BuildpackError::PipLayer(error) => on_pip_dependencies_layer_error(error), + BuildpackError::ProjectDescriptor(error) => on_project_descriptor_error(error), + BuildpackError::PythonLayer(error) => on_python_layer_error(error), + BuildpackError::PythonVersion(error) => on_python_version_error(error), + }; +} + +fn on_project_descriptor_error(project_descriptor_error: ReadProjectDescriptorError) { + match project_descriptor_error { + ReadProjectDescriptorError::Io(io_error) => log_io_error( + "Unable to read project.toml", + "reading the (optional) project.toml file", + &io_error, + ), + // TODO: Add more detail here, like example file contents for functions? + ReadProjectDescriptorError::Parse(toml_error) => log_error( + "Invalid project.toml", + formatdoc! {" + A parsing/validation error error occurred whilst loading the project.toml file. + + Details: {toml_error} + "}, + ), + }; +} + +fn on_determine_package_manager_error( + determine_package_manager_error: DeterminePackageManagerError, +) { + match determine_package_manager_error { + DeterminePackageManagerError::Io(io_error) => log_io_error( + "Unable to determine the package manager", + "determining which Python package manager to use for this project", + &io_error, + ), + // TODO: Should this mention the setup.py / pyproject.toml case? + DeterminePackageManagerError::NoneFound => log_error( + "No Python package manager files were found", + indoc! {" + A Pip requirements file was not found in your application's source code. + This file is required so that your application's dependencies can be installed. + + Please add a file named exactly 'requirements.txt' to the root directory of your + application, containing a list of the packages required by your application. + + For more information on what this file should contain, see: + https://pip.pypa.io/en/stable/reference/requirements-file-format/ + "}, + ), + }; +} + +fn on_python_version_error(python_version_error: PythonVersionError) { + match python_version_error { + PythonVersionError::RuntimeTxt(error) => match error { + ReadRuntimeTxtError::Io(io_error) => log_io_error( + "Unable to read runtime.txt", + "reading the (optional) runtime.txt file", + &io_error, + ), + // TODO: Write the supported Python versions inline, instead of linking out to Dev Center. + ReadRuntimeTxtError::Parse(ParseRuntimeTxtError { cleaned_contents }) => log_error( + "Invalid Python version in runtime.txt", + formatdoc! {" + The Python version specified in 'runtime.txt' is not in the correct format. + + The following file contents were found: + {cleaned_contents} + + However, the file contents must begin with a 'python-' prefix, followed by the + version specified as '..'. Comments are not supported. + + For example, to request Python {DEFAULT_PYTHON_VERSION}, the correct version format is: + python-{major}.{minor}.{patch} + + Please update 'runtime.txt' to use the correct version format, or else remove + the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). + + For a list of the supported Python versions, see: + https://devcenter.heroku.com/articles/python-support#supported-runtimes + ", + major = DEFAULT_PYTHON_VERSION.major, + minor = DEFAULT_PYTHON_VERSION.minor, + patch = DEFAULT_PYTHON_VERSION.patch + }, + ), + }, + }; +} + +fn on_python_layer_error(python_layer_error: PythonLayerError) { + match python_layer_error { + PythonLayerError::BootstrapPipCommand(error) => match error { + CommandError::Io(io_error) => log_io_error( + "Unable to bootstrap pip", + "running the command to install pip, setuptools and wheel", + &io_error, + ), + CommandError::NonZeroExitStatus(exit_status) => log_error( + "Unable to bootstrap pip", + formatdoc! {" + The command to install pip, setuptools and wheel did not exit successfully ({exit_status}). + + See the log output above for more information. + + In some cases, this happens due to an unstable network connection. + Please try again to see if the error resolves itself. + + If that does not help, check the status of PyPI (the upstream Python + package repository service), here: + https://status.python.org + "}, + ), + }, + PythonLayerError::CompileByteCodeCommand(error) => match error { + CommandError::Io(io_error) => log_io_error( + "Unable to compile Python byte-code", + "running the 'python -m compileall' command", + &io_error, + ), + CommandError::NonZeroExitStatus(exit_status) => log_error( + "Unable to compile Python byte-code", + formatdoc! {" + The 'python -m compileall' command used to compile Python byte-code + for the system 'site-packages' directory failed ({exit_status}). + + See the log output above for more information. + "}, + ), + }, + PythonLayerError::DownloadUnpackArchive(error) => match error { + DownloadUnpackArchiveError::Io(io_error) => log_io_error( + "Unable to unpack the Python archive", + "unpacking the downloaded Python runtime archive and writing it to disk", + &io_error, + ), + DownloadUnpackArchiveError::Request(ureq_error) => log_error( + "Unable to download Python", + formatdoc! {" + An error occurred whilst downloading the Python runtime archive. + + In some cases, this happens due to an unstable network connection. + Please try again and to see if the error resolves itself. + + Details: {ureq_error} + "}, + ), + }, + PythonLayerError::LocateBundledPipIo(io_error) => log_io_error( + "Unable to locate the bundled copy of pip", + "locating the pip wheel file bundled inside the Python 'ensurepip' module", + &io_error, + ), + PythonLayerError::MakeSitePackagesReadOnlyIo(io_error) => log_io_error( + "Unable to make site-packages directory read-only", + "modifying the permissions on Python's 'site-packages' directory", + &io_error, + ), + // This error will change once the Python version is validated against a manifest. + // TODO: Write the supported Python versions inline, instead of linking out to Dev Center. + PythonLayerError::PythonVersionNotFound { + python_version, + stack, + } => log_error( + "Requested Python version is not available", + formatdoc! {" + The requested Python version ({python_version}) is not available for this stack ({stack}). + + Please update the version in 'runtime.txt' to a supported Python version, or else + remove the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). + + For a list of the supported Python versions, see: + https://devcenter.heroku.com/articles/python-support#supported-runtimes + "}, + ), + }; +} + +fn on_pip_dependencies_layer_error(pip_dependencies_layer_error: PipDependenciesLayerError) { + match pip_dependencies_layer_error { + PipDependenciesLayerError::CreateSrcDirIo(io_error) => log_io_error( + "Unable to create 'src' directory required for pip install", + "creating the 'src' directory in the pip layer, prior to running pip install", + &io_error, + ), + PipDependenciesLayerError::PipInstallCommand(error) => match error { + CommandError::Io(io_error) => log_io_error( + "Unable to install dependencies using pip", + "running the 'pip install' command to install the application's dependencies", + &io_error, + ), + // TODO: Add more suggestions here as to causes (eg network, invalid requirements.txt, + // package broken or not compatible with version of Python, missing system dependencies etc) + CommandError::NonZeroExitStatus(exit_status) => log_error( + "Unable to install dependencies using pip", + formatdoc! {" + The 'pip install' command to install the application's dependencies from + 'requirements.txt' failed ({exit_status}). + + See the log output above for more information. + "}, + ), + }, + }; +} + +fn on_check_function_error(check_function_error: CheckFunctionError) { + match check_function_error { + CheckFunctionError::Io(io_error) => log_io_error( + "Unable to run the Salesforce Functions self-check command", + &format!("running the '{FUNCTION_RUNTIME_PROGRAM_NAME} check' command"), + &io_error, + ), + // TOOO: Clean up the error message from the check command. + CheckFunctionError::NonZeroExitStatus(output) => log_error( + "The Salesforce Functions self-check failed", + formatdoc! {" + The '{FUNCTION_RUNTIME_PROGRAM_NAME} check' command failed ({exit_status}), indicating + there is a problem with the Python Salesforce Function in this project. + + Details: + {stderr} + ", + exit_status = output.status, + stderr = String::from_utf8_lossy(&output.stderr), + }, + ), + CheckFunctionError::ProgramNotFound => log_error( + "The Salesforce Functions package is not installed", + formatdoc! {" + The '{FUNCTION_RUNTIME_PROGRAM_NAME}' program that is required for Python Salesforce + Functions could not be found. + + Check that the 'salesforce-functions' Python package is listed as a + dependency in 'requirements.txt'. + + If this project is not intended to be a Salesforce Function, remove the + 'type = \"function\"' declaration from 'project.toml' to skip this check. + "}, + ), + }; +} + +fn log_io_error(header: &str, occurred_whilst: &str, io_error: &io::Error) { + // We don't suggest opening a support ticket, since a subset of I/O errors can be caused + // by issues in the application. In the future, perhaps we should try and split these out? + log_error( + header, + formatdoc! {" + An unexpected error occurred whilst {occurred_whilst}. + + Details: I/O Error: {io_error} + "}, + ); +} diff --git a/src/functions.rs b/src/functions.rs new file mode 100644 index 0000000..0bf0c5a --- /dev/null +++ b/src/functions.rs @@ -0,0 +1,117 @@ +use crate::project_descriptor::{self, ReadProjectDescriptorError, SalesforceProjectType}; +use libcnb::data::launch::{Launch, LaunchBuilder, ProcessBuilder}; +use libcnb::data::process_type; +use libcnb::Env; +use std::io; +use std::path::Path; +use std::process::{Command, Output}; + +pub const FUNCTION_RUNTIME_PROGRAM_NAME: &str = "sf-functions-python"; + +// TODO: Decide default number of workers. +const SERVE_SUBCOMMAND: &str = "serve --host 0.0.0.0 --port \"${PORT:-8080}\" --workers 4 ."; + +/// Detect whether the specified project directory is that of a Salesforce Function. +/// +/// Returns `Ok(true)` if the specified project directory contains a `project.toml` file with a +/// `com.salesforce.type` of "function". +/// +/// It is permitted for the `project.toml` file not to exist, or for there to be no `com.salesforce` +/// TOML table within the file, in which case `Ok(false)` will be returned. +/// +/// However, an error will be returned if any other IO error occurred, if the `project.toml` file +/// is not valid TOML, or the TOML document does not adhere to the schema. +pub(crate) fn is_function_project(app_dir: &Path) -> Result { + project_descriptor::read_salesforce_project_type(app_dir) + .map(|project_type| project_type == Some(SalesforceProjectType::Function)) +} + +/// Validate the function using the `sf-functions-python check` command. +// TODO: Add support for checking the function meets a minimum version, like the CLI does: +// - Explore pros/cons of version command vs looking up package version. +// - Version command failure cases: Not found / io error / exit code / invalid version (unparsable) / too old version +// TODO: Should we output the version of the salesforce-functions package in the CNB build, locally, at runtime etc? +// TODO: Should we inform that a new version is available, as a less strict complement to the minimum version? +pub(crate) fn check_function(env: &Env) -> Result<(), CheckFunctionError> { + // Not using `utils::run_command` since we want to capture output and only + // display it if the check command fails. + Command::new(FUNCTION_RUNTIME_PROGRAM_NAME) + .args(["check", "."]) + .envs(env) + .output() + .map_err(|io_error| match io_error.kind() { + io::ErrorKind::NotFound => CheckFunctionError::ProgramNotFound, + _ => CheckFunctionError::Io(io_error), + }) + .and_then(|output| { + if output.status.success() { + Ok(()) + } else { + Err(CheckFunctionError::NonZeroExitStatus(output)) + } + }) +} + +/// Generate a `launch.toml` configuration for running Python Salesforce Functions. +/// +/// Runs the `sf-functions-python serve` command with suitable options for production. +pub(crate) fn launch_config() -> Launch { + LaunchBuilder::new() + .process( + // TODO: Stop running via bash once direct processes support env var interpolation: + // https://github.com/buildpacks/rfcs/issues/258 + ProcessBuilder::new(process_type!("web"), "bash") + .args([ + "-c", + &format!("exec {FUNCTION_RUNTIME_PROGRAM_NAME} {SERVE_SUBCOMMAND}"), + ]) + .default(true) + .direct(true) + .build(), + ) + .build() +} + +/// Errors that can occur when running the `sf-functions-python check` command. +#[derive(Debug)] +pub(crate) enum CheckFunctionError { + Io(io::Error), + NonZeroExitStatus(Output), + ProgramNotFound, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_function_project_no_project_toml() { + let app_dir = Path::new("test-fixtures/empty"); + + assert!(!is_function_project(app_dir).unwrap()); + } + + #[test] + fn is_function_project_non_salesforce_project_toml() { + let app_dir = Path::new("test-fixtures/project_toml_non_salesforce"); + + assert!(!is_function_project(app_dir).unwrap()); + } + + #[test] + fn is_function_project_function_project_toml() { + let app_dir = Path::new("test-fixtures/function_template"); + + assert!(is_function_project(app_dir).unwrap()); + } + + #[test] + fn is_function_project_invalid_project_toml() { + let app_dir = Path::new("test-fixtures/project_toml_invalid"); + + assert!(matches!( + is_function_project(app_dir).unwrap_err(), + ReadProjectDescriptorError::Parse(_) + )); + } +} diff --git a/src/layers/mod.rs b/src/layers/mod.rs new file mode 100644 index 0000000..74c1faa --- /dev/null +++ b/src/layers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod pip_cache; +pub(crate) mod pip_dependencies; +pub(crate) mod python; diff --git a/src/layers/pip_cache.rs b/src/layers/pip_cache.rs new file mode 100644 index 0000000..55993a5 --- /dev/null +++ b/src/layers/pip_cache.rs @@ -0,0 +1,75 @@ +use crate::python_version::PythonVersion; +use crate::PythonBuildpack; +use libcnb::build::BuildContext; +use libcnb::data::buildpack::StackId; +use libcnb::data::layer_content_metadata::LayerTypes; +use libcnb::layer::{ExistingLayerStrategy, Layer, LayerData, LayerResult, LayerResultBuilder}; +use libcnb::Buildpack; +use libherokubuildpack::log::log_info; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +pub(crate) struct PipCacheLayer<'a> { + pub python_version: &'a PythonVersion, +} + +#[derive(Clone, Deserialize, PartialEq, Serialize)] +pub(crate) struct PipCacheLayerMetadata { + python_version: String, + stack: StackId, +} + +impl Layer for PipCacheLayer<'_> { + type Buildpack = PythonBuildpack; + type Metadata = PipCacheLayerMetadata; + + fn types(&self) -> LayerTypes { + LayerTypes { + build: false, + cache: true, + launch: false, + } + } + + fn create( + &self, + context: &BuildContext, + _layer_path: &Path, + ) -> Result, ::Error> { + log_info("Pip cache created"); + let layer_metadata = generate_layer_metadata(&context.stack_id, self.python_version); + LayerResultBuilder::new(layer_metadata).build() + } + + fn existing_layer_strategy( + &self, + context: &BuildContext, + layer_data: &LayerData, + ) -> Result::Error> { + // TODO: Also invalidate based on time since layer creation? + // TODO: Decide what should be logged + if layer_data.content_metadata.metadata + == generate_layer_metadata(&context.stack_id, self.python_version) + { + log_info("Re-using cached pip-cache"); + Ok(ExistingLayerStrategy::Keep) + } else { + log_info("Discarding cached pip-cache"); + Ok(ExistingLayerStrategy::Recreate) + } + } +} + +fn generate_layer_metadata( + stack_id: &StackId, + python_version: &PythonVersion, +) -> PipCacheLayerMetadata { + // TODO: Add timestamp field or similar (maybe not necessary if invalidating on pip/python change?) + // TODO: Invalidate on pip version change? + PipCacheLayerMetadata { + python_version: python_version.to_string(), + stack: stack_id.clone(), + } +} + +// TODO: Unit tests for cache invalidation handling? diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs new file mode 100644 index 0000000..56b2584 --- /dev/null +++ b/src/layers/pip_dependencies.rs @@ -0,0 +1,160 @@ +use crate::python_version::PythonVersion; +use crate::utils::{self, CommandError}; +use crate::{BuildpackError, PythonBuildpack}; +use libcnb::build::BuildContext; +use libcnb::data::buildpack::StackId; +use libcnb::data::layer_content_metadata::LayerTypes; +use libcnb::layer::{Layer, LayerResult, LayerResultBuilder}; +use libcnb::layer_env::{LayerEnv, ModificationBehavior, Scope}; +use libcnb::{Buildpack, Env}; +use libherokubuildpack::log::log_info; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{fs, io}; + +pub(crate) struct PipDependenciesLayer<'a> { + pub env: &'a Env, + pub pip_cache_dir: PathBuf, + pub python_version: &'a PythonVersion, +} + +#[derive(Clone, Deserialize, PartialEq, Serialize)] +pub(crate) struct PipDependenciesLayerMetadata { + python_version: String, + stack: StackId, +} + +impl Layer for PipDependenciesLayer<'_> { + type Buildpack = PythonBuildpack; + type Metadata = PipDependenciesLayerMetadata; + + fn types(&self) -> LayerTypes { + LayerTypes { + build: true, + // TODO: Re-enabling caching once remaining invalidation logic finished. + cache: false, + launch: true, + } + } + + fn create( + &self, + context: &BuildContext, + layer_path: &Path, + ) -> Result, ::Error> { + // TODO: Explain PYTHONUSERBASE and that it will contain bin/, lib/.../site-packages/ + // etc and so does not need to be nested due to the env/ directory. + let layer_env = LayerEnv::new().chainable_insert( + Scope::All, + ModificationBehavior::Override, + "PYTHONUSERBASE", + layer_path, + ); + let env = layer_env.apply(Scope::Build, self.env); + + let src_dir = layer_path.join("src"); + fs::create_dir(&src_dir).map_err(PipDependenciesLayerError::CreateSrcDirIo)?; + + log_info("Running pip install"); + + // TODO: Explain why we're using user install + // TODO: Refactor this out so it can be shared with `update()` + // TODO: Mention that we're intentionally not using env_clear() otherwise + // PATH won't be set, and Pip won't be able to find things like Git. + utils::run_command( + Command::new("pip") + .args([ + "install", + "--cache-dir", + &self.pip_cache_dir.to_string_lossy(), + "--no-input", + // Prevent warning about the `bin/` directory not being on `PATH`, since it + // will be added automatically by libcnb/lifecycle later. + "--no-warn-script-location", + "--progress", + "off", + "--user", + "--requirement", + "requirements.txt", + // Make pip clone any VCS repositories installed in editable mode into a directory in this layer, + // rather than the default of the current working directory (the app dir). + "--src", + &src_dir.to_string_lossy(), + ]) + .envs(&env) + // TODO: Decide whether to use this or `--no-compile` + `compileall`. + // If using compileall will need different strategy for `update()`. + // See also: https://github.com/pypa/pip/blob/3820b0e52c7fed2b2c43ba731b718f316e6816d1/src/pip/_internal/operations/install/wheel.py#L616 + // Using 1980-01-01T00:00:01Z to avoid: + // ValueError: ZIP does not support timestamps before 1980 + .env("SOURCE_DATE_EPOCH", "315532800"), + ) + .map_err(PipDependenciesLayerError::PipInstallCommand)?; + + log_info("Pip install completed"); + + let layer_metadata = generate_layer_metadata(&context.stack_id, self.python_version); + LayerResultBuilder::new(layer_metadata) + .env(layer_env) + .build() + } + + // TODO: Re-enabling caching once remaining invalidation logic finished. + // fn update( + // &self, + // _context: &BuildContext, + // _layer_data: &LayerData, + // ) -> Result, ::Error> { + // // TODO + // unimplemented!() + // } + // + // fn existing_layer_strategy( + // &self, + // context: &BuildContext, + // layer_data: &LayerData, + // ) -> Result::Error> { + // // TODO: Also invalidate based on requirements.txt contents + // // TODO: Decide whether sub-requirements files should also invalidate? If not, should we warn? + // // TODO: Also invalidate based on time since layer creation + // // TODO: Decide what should be logged + // // TODO: Re-test the performance of caching site-modules vs only caching Pip's cache. + // #[allow(unreachable_code)] + // if layer_data.content_metadata.metadata + // == generate_layer_metadata(&context.stack_id, self.python_version) + // { + // log_info("Re-using cached dependencies"); + // Ok(ExistingLayerStrategy::Update) + // } else { + // log_info("Discarding cached dependencies"); + // Ok(ExistingLayerStrategy::Recreate) + // } + // } +} + +fn generate_layer_metadata( + stack_id: &StackId, + python_version: &PythonVersion, +) -> PipDependenciesLayerMetadata { + // TODO: Add requirements.txt SHA256 or similar + // TODO: Add timestamp field or similar + PipDependenciesLayerMetadata { + python_version: python_version.to_string(), + stack: stack_id.clone(), + } +} + +#[derive(Debug)] +pub(crate) enum PipDependenciesLayerError { + CreateSrcDirIo(io::Error), + PipInstallCommand(CommandError), +} + +impl From for BuildpackError { + fn from(error: PipDependenciesLayerError) -> Self { + Self::PipLayer(error) + } +} + +// TODO: Unit tests for cache invalidation handling? diff --git a/src/layers/python.rs b/src/layers/python.rs new file mode 100644 index 0000000..9e1d2aa --- /dev/null +++ b/src/layers/python.rs @@ -0,0 +1,306 @@ +use crate::python_version::PythonVersion; +use crate::utils::{self, CommandError, DownloadUnpackArchiveError}; +use crate::{BuildpackError, PythonBuildpack}; +use libcnb::build::BuildContext; +use libcnb::data::buildpack::StackId; +use libcnb::data::layer_content_metadata::LayerTypes; +use libcnb::layer::{ExistingLayerStrategy, Layer, LayerData, LayerResult, LayerResultBuilder}; +use libcnb::layer_env::{LayerEnv, ModificationBehavior, Scope}; +use libcnb::{Buildpack, Env}; +use libherokubuildpack::log::{log_header, log_info}; +use serde::{Deserialize, Serialize}; +use std::fs::Permissions; +use std::os::unix::prelude::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{fs, io}; + +const PIP_VERSION: &str = "22.3.1"; +const SETUPTOOLS_VERSION: &str = "65.6.3"; +const WHEEL_VERSION: &str = "0.38.3"; + +pub(crate) struct PythonLayer<'a> { + pub env: &'a Env, + pub python_version: &'a PythonVersion, +} + +#[derive(Clone, Deserialize, PartialEq, Serialize)] +pub(crate) struct PythonLayerMetadata { + stack: StackId, + python_version: String, + pip_version: String, + setuptools_version: String, + wheel_version: String, +} + +impl Layer for PythonLayer<'_> { + type Buildpack = PythonBuildpack; + type Metadata = PythonLayerMetadata; + + fn types(&self) -> LayerTypes { + LayerTypes { + build: true, + cache: true, + launch: true, + } + } + + #[allow(clippy::too_many_lines)] + fn create( + &self, + context: &BuildContext, + layer_path: &Path, + ) -> Result, ::Error> { + log_header("Installing Python"); + + // TODO: Move this URL generation somewhere else (ie manifest etc). + let archive_url = format!( + "https://heroku-buildpack-python.s3.us-east-1.amazonaws.com/{}/runtimes/python-{}.tar.gz", + context.stack_id, self.python_version + ); + + log_info(format!("Downloading Python {}", self.python_version)); + utils::download_and_unpack_gzipped_archive(&archive_url, layer_path).map_err(|error| { + match error { + // TODO: Remove this once the Python version is validated against a manifest (at which + // point 404s can be treated as an internal error, instead of user error) + DownloadUnpackArchiveError::Request(ureq::Error::Status(404, _)) => { + PythonLayerError::PythonVersionNotFound { + stack: context.stack_id.clone(), + python_version: self.python_version.clone(), + } + } + other_error => PythonLayerError::DownloadUnpackArchive(other_error), + } + })?; + log_info("Python installation successful"); + + // Remember to force invalidation of the cached layer if this list ever changes. + let layer_env = LayerEnv::new() + // We have to set `CPATH` explicitly, since the automatic path set by lifecycle/libcnb is + // `/include/` whereas Python's header files are at `/include/pythonX.Y/` + // (and compilers don't recursively search). + .chainable_insert( + Scope::All, + ModificationBehavior::Prepend, + "CPATH", + layer_path.join(format!( + "include/python{}.{}", + self.python_version.major, self.python_version.minor + )), + ) + .chainable_insert(Scope::All, ModificationBehavior::Delimiter, "CPATH", ":") + // Ensure Python uses a Unicode locate, to prevent the issues described in: + // https://github.com/docker-library/python/pull/570 + .chainable_insert( + Scope::All, + ModificationBehavior::Override, + "LANG", + "C.UTF-8", + ) + // We have to set `PKG_CONFIG_PATH` explicitly, since the automatic path set by lifecycle/libcnb + // is `/pkgconfig/`, whereas Python's pkgconfig files are at `/lib/pkgconfig/`. + .chainable_insert( + Scope::All, + ModificationBehavior::Prepend, + "PKG_CONFIG_PATH", + layer_path.join("lib/pkgconfig"), + ) + .chainable_insert( + Scope::All, + ModificationBehavior::Delimiter, + "PKG_CONFIG_PATH", + ":", + ) + // We use a curated Pip version, so skip the update check to speed up Pip invocations, + // reduce build log spam and prevent users from thinking they need to manually upgrade. + .chainable_insert( + Scope::All, + ModificationBehavior::Override, + "PIP_DISABLE_PIP_VERSION_CHECK", + "1", + ) + // Disable Python's output buffering to ensure logs aren't dropped if an app crashes. + .chainable_insert( + Scope::All, + ModificationBehavior::Override, + "PYTHONUNBUFFERED", + "1", + ); + let mut env = layer_env.apply(Scope::Build, self.env); + + // The Python binaries are built using `--shared`, and since they're being installed at a + // different location from their original `--prefix`, they need `LD_LIBRARY_PATH` to be set + // in order to find `libpython3`. Whilst `LD_LIBRARY_PATH` will be automatically set later by + // lifecycle/libcnb, it's not set by libcnb until this `Layer` has ended, and so we have to + // explicitly set it for the Python invocations within this layer. + env.insert("LD_LIBRARY_PATH", layer_path.join("lib")); + + log_header("Installing Pip"); + log_info(format!("Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION}")); + + let python_binary = layer_path.join("bin/python"); + let python_stdlib_dir = layer_path.join(format!( + "lib/python{}.{}", + self.python_version.major, self.python_version.minor + )); + let site_packages_dir = python_stdlib_dir.join("site-packages"); + + // TODO: Explain what's happening here + let bundled_pip_module = + bundled_pip_module(&python_stdlib_dir).map_err(PythonLayerError::LocateBundledPipIo)?; + utils::run_command( + Command::new(&python_binary) + .args([ + &bundled_pip_module.to_string_lossy(), + "install", + "--no-cache-dir", + "--no-compile", + "--no-input", + "--quiet", + format!("pip=={PIP_VERSION}").as_str(), + format!("setuptools=={SETUPTOOLS_VERSION}").as_str(), + format!("wheel=={WHEEL_VERSION}").as_str(), + ]) + .envs(&env), + ) + .map_err(PythonLayerError::BootstrapPipCommand)?; + + // TODO: Add comment explaining why we're doing this vs pip default compile. + // (on M1 this reduces the time taken for the pip bootstrap from 17.6s to 13.4s) + // TODO: Test performance difference when not running under QEMU + utils::run_command( + Command::new(python_binary) + .args([ + "-m", + "compileall", + "-f", + "-q", + "--invalidation-mode", + "unchecked-hash", + "--workers", + "0", + &site_packages_dir.to_string_lossy(), + ]) + .envs(&env), + ) + .map_err(PythonLayerError::CompileByteCodeCommand)?; + + // By default Pip will install into the system site-packages directory if it is writeable + // by the current user. Whilst the buildpack's own `pip install` invocations always use + // `--user` to ensure application dependencies are instead installed into the user + // site-packages, it's possible other buildpacks or custom scripts may forget to do so. + // By making the system site-packages directory read-only, Pip will automatically use + // user installs in such cases: + // https://github.com/pypa/pip/blob/22.3.1/src/pip/_internal/commands/install.py#L706-L764 + fs::set_permissions(&site_packages_dir, Permissions::from_mode(0o555)) + .map_err(PythonLayerError::MakeSitePackagesReadOnlyIo)?; + + log_info("Installation completed"); + + let layer_metadata = generate_layer_metadata(&context.stack_id, self.python_version); + LayerResultBuilder::new(layer_metadata) + .env(layer_env) + .build() + } + + fn existing_layer_strategy( + &self, + context: &BuildContext, + layer_data: &LayerData, + ) -> Result::Error> { + // TODO: Decide what should be logged in the cached case (+more granular reason?) + // Worth including what changed not only for cache invalidation, but also + // to help debug any issues (eg changed pip version causing issues) + let old_metadata = &layer_data.content_metadata.metadata; + let new_metadata = generate_layer_metadata(&context.stack_id, self.python_version); + if new_metadata == *old_metadata { + log_header("Installing Python"); + log_info(format!( + "Re-using cached Python {}", + old_metadata.python_version + )); + + log_header("Installing Pip"); + log_info(format!( + "Re-using cached pip {}, setuptools {} and wheel {}", + new_metadata.pip_version, + new_metadata.setuptools_version, + new_metadata.wheel_version + )); + + Ok(ExistingLayerStrategy::Keep) + } else { + log_info(format!( + "Discarding cached Python {}", + old_metadata.python_version + )); + log_info(format!( + "Discarding cached pip {}, setuptools {} and wheel {}", + old_metadata.pip_version, + old_metadata.setuptools_version, + old_metadata.wheel_version + )); + Ok(ExistingLayerStrategy::Recreate) + } + } +} + +// TODO: Explain what's happening here +// The bundled version of Pip (and thus the wheel filename) varies across Python versions, +// so we have to search the bundled wheels directory for the appropriate file. +// TODO: This returns a module path rather than a wheel path - change? +fn bundled_pip_module(python_stdlib_dir: &Path) -> io::Result { + let bundled_wheels_dir = python_stdlib_dir.join("ensurepip/_bundled"); + let pip_wheel_filename_prefix = "pip-"; + + for entry in fs::read_dir(bundled_wheels_dir)? { + let entry = entry?; + if entry + .file_name() + .to_string_lossy() + .starts_with(pip_wheel_filename_prefix) + { + return Ok(entry.path().join("pip")); + } + } + + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("No files found matching the filename prefix of '{pip_wheel_filename_prefix}'"), + )) +} + +fn generate_layer_metadata( + stack_id: &StackId, + python_version: &PythonVersion, +) -> PythonLayerMetadata { + PythonLayerMetadata { + stack: stack_id.clone(), + python_version: python_version.to_string(), + pip_version: PIP_VERSION.to_string(), + setuptools_version: SETUPTOOLS_VERSION.to_string(), + wheel_version: WHEEL_VERSION.to_string(), + } +} + +#[derive(Debug)] +pub(crate) enum PythonLayerError { + BootstrapPipCommand(CommandError), + CompileByteCodeCommand(CommandError), + DownloadUnpackArchive(DownloadUnpackArchiveError), + LocateBundledPipIo(io::Error), + MakeSitePackagesReadOnlyIo(io::Error), + PythonVersionNotFound { + python_version: PythonVersion, + stack: StackId, + }, +} + +impl From for BuildpackError { + fn from(error: PythonLayerError) -> Self { + Self::PythonLayer(error) + } +} + +// TODO: Unit tests for cache invalidation handling? diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..9308299 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,153 @@ +#![warn(clippy::pedantic)] +#![warn(unused_crate_dependencies)] +// Prevent warnings caused by the large size of `ureq::Error` in error enums, +// where it is not worth boxing since the enum size doesn't affect performance. +#![allow(clippy::large_enum_variant)] +#![allow(clippy::result_large_err)] + +mod errors; +mod functions; +mod layers; +mod package_manager; +mod project_descriptor; +mod python_version; +mod runtime_txt; +mod utils; + +use crate::functions::CheckFunctionError; +use crate::layers::pip_cache::PipCacheLayer; +use crate::layers::pip_dependencies::{PipDependenciesLayer, PipDependenciesLayerError}; +use crate::layers::python::{PythonLayer, PythonLayerError}; +use crate::package_manager::{DeterminePackageManagerError, PackageManager}; +use crate::project_descriptor::ReadProjectDescriptorError; +use crate::python_version::PythonVersionError; +use libcnb::build::{BuildContext, BuildResult, BuildResultBuilder}; +use libcnb::data::layer_name; +use libcnb::detect::{DetectContext, DetectResult, DetectResultBuilder}; +use libcnb::generic::{GenericMetadata, GenericPlatform}; +use libcnb::layer_env::Scope; +use libcnb::{buildpack_main, Buildpack, Env}; +use libherokubuildpack::log::{log_header, log_info}; +use std::io; + +struct PythonBuildpack; + +impl Buildpack for PythonBuildpack { + type Platform = GenericPlatform; + type Metadata = GenericMetadata; + type Error = BuildpackError; + + fn detect(&self, context: DetectContext) -> libcnb::Result { + // For the functions alpha/beta, we need to release the CNB into the main builder image, + // however only want to make it available for functions for now, since the CNB is still + // experimental and not feature-complete for non-function use-cases. + // TODO: Remove this once the buildpack is ready for non-functions use. + if !functions::is_function_project(&context.app_dir) + .map_err(BuildpackError::ProjectDescriptor)? + { + log_info("A project.toml file containing a suitable Salesforce Function configuration was not found."); + return DetectResultBuilder::fail().build(); + } + + // In the future we will add support for requiring this buildpack through the build plan, + // but we first need a better understanding of real-world use-cases, so that we can work + // out how best to support them without sacrificing existing error handling UX (such as + // wanting to show a clear error when requirements.txt is missing). + if utils::is_python_project(&context.app_dir).map_err(BuildpackError::DetectIo)? { + DetectResultBuilder::pass().build() + } else { + log_info("No Python project files found (such as requirements.txt)."); + DetectResultBuilder::fail().build() + } + } + + fn build(&self, context: BuildContext) -> libcnb::Result { + // We perform all project analysis up front, so the build can fail early if the config is invalid. + // TODO: Add a "Build config" header and list all config in one place? + let is_function = functions::is_function_project(&context.app_dir) + .map_err(BuildpackError::ProjectDescriptor)?; + let package_manager = package_manager::determine_package_manager(&context.app_dir) + .map_err(BuildpackError::DeterminePackageManager)?; + + log_header("Determining Python version"); + let python_version = python_version::determine_python_version(&context.app_dir) + .map_err(BuildpackError::PythonVersion)?; + + // We inherit the current process's env vars, since we want `PATH` and `HOME` to be set + // so that later commands can find tools like Git in the stack image. Any user-provided + // env vars will still be excluded, due to the use of `clear-env` in `buildpack.toml`. + let mut env = Env::from_current(); + + let python_layer = context.handle_layer( + layer_name!("python"), + PythonLayer { + env: &env, + python_version: &python_version, + }, + )?; + env = python_layer.env.apply(Scope::Build, &env); + + let dependencies_layer_env = match package_manager { + PackageManager::Pip => { + log_header("Installing dependencies using Pip"); + let pip_cache_layer = context.handle_layer( + layer_name!("pip-cache"), + PipCacheLayer { + python_version: &python_version, + }, + )?; + let pip_layer = context.handle_layer( + layer_name!("dependencies"), + PipDependenciesLayer { + env: &env, + pip_cache_dir: pip_cache_layer.path, + python_version: &python_version, + }, + )?; + pip_layer.env + } + }; + env = dependencies_layer_env.apply(Scope::Build, &env); + + if is_function { + log_header("Validating Salesforce Function"); + functions::check_function(&env).map_err(BuildpackError::CheckFunction)?; + log_info("Function passed validation."); + + BuildResultBuilder::new() + .launch(functions::launch_config()) + .build() + } else { + BuildResultBuilder::new().build() + } + } + + fn on_error(&self, error: libcnb::Error) { + errors::on_error(error); + } +} + +#[derive(Debug)] +pub(crate) enum BuildpackError { + CheckFunction(CheckFunctionError), + DetectIo(io::Error), + DeterminePackageManager(DeterminePackageManagerError), + PipLayer(PipDependenciesLayerError), + ProjectDescriptor(ReadProjectDescriptorError), + PythonLayer(PythonLayerError), + PythonVersion(PythonVersionError), +} + +impl From for libcnb::Error { + fn from(error: BuildpackError) -> Self { + Self::BuildpackError(error) + } +} + +buildpack_main!(PythonBuildpack); + +#[cfg(test)] +mod tests { + // Suppress warnings due to the `unused_crate_dependencies` lint not handling integration tests well. + use libcnb_test as _; +} diff --git a/src/package_manager.rs b/src/package_manager.rs new file mode 100644 index 0000000..50cf745 --- /dev/null +++ b/src/package_manager.rs @@ -0,0 +1,33 @@ +use std::io; +use std::path::Path; + +pub(crate) enum PackageManager { + Pip, +} + +const PACKAGE_MANAGER_FILE_MAPPING: [(&str, PackageManager); 1] = + [("requirements.txt", PackageManager::Pip)]; + +// TODO: Unit test +pub(crate) fn determine_package_manager( + app_dir: &Path, +) -> Result { + // Until `Iterator::try_find` is stabilised, this is cleaner as a for loop. + for (filename, package_manager) in PACKAGE_MANAGER_FILE_MAPPING { + if app_dir + .join(filename) + .try_exists() + .map_err(DeterminePackageManagerError::Io)? + { + return Ok(package_manager); + } + } + + Err(DeterminePackageManagerError::NoneFound) +} + +#[derive(Debug)] +pub(crate) enum DeterminePackageManagerError { + Io(io::Error), + NoneFound, +} diff --git a/src/project_descriptor.rs b/src/project_descriptor.rs new file mode 100644 index 0000000..fa91dab --- /dev/null +++ b/src/project_descriptor.rs @@ -0,0 +1,281 @@ +use crate::utils; +use serde::Deserialize; +use std::io; +use std::path::Path; + +/// Reads the `com.salesforce.type` field from any `project.toml` in the specified directory. +/// +/// It is permitted for the `project.toml` file not to exist, or for there to be no `com.salesforce` +/// table within the TOML document, in which case `Ok(None)` will be returned. +/// +/// However, an error will be returned if any other IO error occurred, the file is not valid TOML, +/// or the TOML document does not adhere to the schema. +pub(crate) fn read_salesforce_project_type( + app_dir: &Path, +) -> Result, ReadProjectDescriptorError> { + read_project_descriptor(app_dir).map(|descriptor| { + descriptor + .unwrap_or_default() + .com + .unwrap_or_default() + .salesforce + .map(|salesforce| salesforce.project_type) + }) +} + +/// Reads any `project.toml` file in the specified directory, parsing it into a [`ProjectDescriptor`]. +/// +/// It is permitted for the `project.toml` file not to exist, in which case `Ok(None)` will be returned. +/// +/// However, an error will be returned if any other IO error occurred, the file is not valid TOML, +/// or the TOML document does not adhere to the schema. +fn read_project_descriptor( + app_dir: &Path, +) -> Result, ReadProjectDescriptorError> { + let project_descriptor_path = app_dir.join("project.toml"); + + utils::read_optional_file(&project_descriptor_path) + .map_err(ReadProjectDescriptorError::Io)? + .map(|contents| parse(&contents).map_err(ReadProjectDescriptorError::Parse)) + .transpose() +} + +/// Parse the contents of a project descriptor TOML file into a [`ProjectDescriptor`]. +/// +/// An error will be returned if the string is not valid TOML, or the TOML document does not +/// adhere to the schema. +fn parse(contents: &str) -> Result { + toml::from_str::(contents) +} + +/// Represents a Cloud Native Buildpack project descriptor file (`project.toml`). +/// +/// Currently only fields used by the buildpack are enforced, so this represents only a +/// subset of the upstream CNB project descriptor schema. +/// +/// See: +#[derive(Debug, Default, Deserialize, PartialEq)] +struct ProjectDescriptor { + com: Option, +} + +/// Represents the `com` table in the project descriptor. +#[derive(Debug, Default, Deserialize, PartialEq)] +struct ComTable { + salesforce: Option, +} + +/// Represents the `com.salesforce` table in the project descriptor. +/// +/// Currently only fields used by the buildpack are enforced, so this represents only a +/// subset of the Salesforce-specific project descriptor schema. +/// +/// See: +#[derive(Debug, Deserialize, PartialEq)] +struct SalesforceTable { + #[serde(rename = "type")] + project_type: SalesforceProjectType, +} + +/// The type of a Salesforce project. +/// +/// For now `Function` is the only valid type, however others will be added in the future. +/// +/// Unknown project types are intentionally rejected, since we're prioritising the UX for +/// functions projects where the type may have been mis-spelt, over forward-compatibility. +#[derive(Debug, Deserialize, PartialEq)] +pub(crate) enum SalesforceProjectType { + #[serde(rename = "function")] + Function, +} + +/// Errors that can occur when reading and parsing a `project.toml` file. +#[derive(Debug)] +pub(crate) enum ReadProjectDescriptorError { + Io(io::Error), + Parse(toml::de::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deserialize_empty_descriptor() { + assert_eq!(parse("").unwrap(), ProjectDescriptor { com: None }); + } + + #[test] + fn deserialize_non_salesforce_descriptor() { + let toml_str = r#" + [_] + schema-version = "0.2" + + [io.buildpacks] + builder = "my-builder" + + [com.example] + key = "value" + "#; + + assert_eq!( + parse(toml_str), + Ok(ProjectDescriptor { + com: Some(ComTable { salesforce: None }) + }) + ); + } + + #[test] + fn deserialize_function_descriptor() { + let toml_str = r#" + [_] + schema-version = "0.2" + + [com.salesforce] + schema-version = "0.1" + id = "example" + description = "Example function" + type = "function" + salesforce-api-version = "56.0" + "#; + + assert_eq!( + parse(toml_str), + Ok(ProjectDescriptor { + com: Some(ComTable { + salesforce: Some(SalesforceTable { + project_type: SalesforceProjectType::Function + }) + }) + }) + ); + } + + #[test] + fn deserialize_minimal_function_descriptor() { + let toml_str = r#" + [com.salesforce] + type = "function" + "#; + + assert_eq!( + parse(toml_str), + Ok(ProjectDescriptor { + com: Some(ComTable { + salesforce: Some(SalesforceTable { + project_type: SalesforceProjectType::Function + }) + }) + }) + ); + } + + #[test] + fn reject_salesforce_table_with_no_project_type() { + let toml_str = r#" + [com.salesforce] + schema-version = "0.1" + id = "example" + "#; + + let error = parse(toml_str).unwrap_err(); + assert_eq!( + error.to_string(), + "missing field `type` for key `com.salesforce` at line 2 column 13" + ); + } + + #[test] + fn reject_unknown_salesforce_project_type() { + let toml_str = r#" + [com.salesforce] + type = "some_unknown_type" + "#; + + let error = parse(toml_str).unwrap_err(); + assert_eq!( + error.to_string(), + "unknown variant `some_unknown_type`, expected `function` for key `com.salesforce.type` at line 2 column 13" + ); + } + + #[test] + fn read_project_descriptor_no_project_toml_file() { + let app_dir = Path::new("test-fixtures/empty"); + + assert_eq!(read_project_descriptor(app_dir).unwrap(), None); + } + + #[test] + fn read_project_descriptor_non_salesforce() { + let app_dir = Path::new("test-fixtures/project_toml_non_salesforce"); + + assert_eq!( + read_project_descriptor(app_dir).unwrap(), + Some(ProjectDescriptor { + com: Some(ComTable { salesforce: None }) + }) + ); + } + + #[test] + fn read_project_descriptor_function() { + let app_dir = Path::new("test-fixtures/function_template"); + + assert_eq!( + read_project_descriptor(app_dir).unwrap(), + Some(ProjectDescriptor { + com: Some(ComTable { + salesforce: Some(SalesforceTable { + project_type: SalesforceProjectType::Function + }) + }) + }) + ); + } + + #[test] + fn read_project_descriptor_invalid_project_toml_file() { + let app_dir = Path::new("test-fixtures/project_toml_invalid"); + + assert!(matches!( + read_project_descriptor(app_dir).unwrap_err(), + ReadProjectDescriptorError::Parse(_) + )); + } + + #[test] + fn get_salesforce_project_type_missing() { + let app_dir = Path::new("test-fixtures/empty"); + + assert_eq!(read_salesforce_project_type(app_dir).unwrap(), None); + } + + #[test] + fn get_salesforce_project_type_non_salesforce() { + let app_dir = Path::new("test-fixtures/project_toml_non_salesforce"); + + assert_eq!(read_salesforce_project_type(app_dir).unwrap(), None); + } + + #[test] + fn get_salesforce_project_type_function() { + let app_dir = Path::new("test-fixtures/function_template"); + + assert_eq!( + read_salesforce_project_type(app_dir).unwrap(), + Some(SalesforceProjectType::Function) + ); + } + + #[test] + fn get_salesforce_project_type_invalid_project_toml_file() { + let app_dir = Path::new("test-fixtures/project_toml_invalid"); + + assert!(matches!( + read_salesforce_project_type(app_dir).unwrap_err(), + ReadProjectDescriptorError::Parse(_) + )); + } +} diff --git a/src/python_version.rs b/src/python_version.rs new file mode 100644 index 0000000..ac91e62 --- /dev/null +++ b/src/python_version.rs @@ -0,0 +1,104 @@ +use crate::runtime_txt::{self, ReadRuntimeTxtError}; +use indoc::formatdoc; +use libherokubuildpack::log::log_info; +use std::fmt::{self, Display}; +use std::path::Path; + +pub(crate) const DEFAULT_PYTHON_VERSION: PythonVersion = PythonVersion { + major: 3, + minor: 11, + patch: 1, +}; + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct PythonVersion { + pub major: u16, + pub minor: u16, + pub patch: u16, +} + +impl PythonVersion { + pub fn new(major: u16, minor: u16, patch: u16) -> Self { + Self { + major, + minor, + patch, + } + } +} + +impl Display for PythonVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}.{}.{}", self.major, self.minor, self.patch) + } +} + +// string -> requested python version -> exact python version -> python runtime (incl URL etc) + +// resolving python version: +// failure modes: Nonsensical, unknown to buildpack, known but not supported, known and used to be supported but no longer +// Does this occur inside each `get_version` / creation of `PythonVersion`? +// But then each error type needs 3-4 additional enum variants +// Depends on whether we want different error messages for each? +// Though could still vary error message by using `PythonVersion.source` etc + +// Questions: +// How should Python version detection precedence work? + +// TODO: Add tests for `get_version`? Or test caller? Or integration test? +// +// Possible tests: +// - some IO error -> Err(RuntimeTxtError::Io) +// - file present but invalid -> Err(RuntimeTxtError::Parse) +// - file present and valid -> Ok(Some(python_version)) +// - file not present -> Ok(None) + +// warnings: +// EOL major version, non-latest minor version, deprecated version specifier? +// output warnings as found during build, or at end of the build log? +// does EOL warnings use requested Python version or resolved version? I suppose resolved since needs EOL date etc, plus range version might still be outdated? + +// logging: +// Do we log for version specifier files not found? Or only when found? +// where do we log? In get_version, determine_python_version, or in the caller and have to store the version source in `PythonVersion`? + +pub(crate) fn determine_python_version( + app_dir: &Path, +) -> Result { + if let Some(runtime_txt_version) = + runtime_txt::read_version(app_dir).map_err(PythonVersionError::RuntimeTxt)? + { + // TODO: Consider passing this back as a `source` field on PythonVersion + // so this can be logged by the caller. + log_info(format!( + "Using Python version {runtime_txt_version} specified in runtime.txt" + )); + return Ok(runtime_txt_version); + } + + // TODO: Write this content inline, instead of linking out to Dev Center. + // Also adjust wording to mention pinning as a use-case, not just using a different version. + log_info(formatdoc! {" + No Python version specified, using the current default of {DEFAULT_PYTHON_VERSION}. + To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes"}); + Ok(DEFAULT_PYTHON_VERSION) +} + +pub(crate) fn _determine_python_version2( + app_dir: &Path, +) -> Result { + runtime_txt::read_version(app_dir) + .map_err(PythonVersionError::RuntimeTxt) + .transpose() + .or_else(|| { + runtime_txt::read_version(app_dir) + .map_err(PythonVersionError::RuntimeTxt) + .transpose() + }) + .unwrap_or(Ok(DEFAULT_PYTHON_VERSION)) +} + +#[derive(Debug)] +pub(crate) enum PythonVersionError { + RuntimeTxt(ReadRuntimeTxtError), +} diff --git a/src/runtime_txt.rs b/src/runtime_txt.rs new file mode 100644 index 0000000..e19d403 --- /dev/null +++ b/src/runtime_txt.rs @@ -0,0 +1,188 @@ +use crate::python_version::PythonVersion; +use crate::utils; +use std::io; +use std::path::Path; + +/// TODO +pub(crate) fn read_version(app_dir: &Path) -> Result, ReadRuntimeTxtError> { + let runtime_txt_path = app_dir.join("runtime.txt"); + + utils::read_optional_file(&runtime_txt_path) + .map_err(ReadRuntimeTxtError::Io)? + .map(|contents| parse(&contents).map_err(ReadRuntimeTxtError::Parse)) + .transpose() +} + +/// Parse the contents of a `runtime.txt` file into a [`PythonVersion`]. +/// +/// The file is expected to contain a string of form `python-X.Y.Z`. +/// Any leading or trailing whitespace will be removed. +fn parse(contents: &str) -> Result { + // All leading/trailing whitespace is trimmed, since that's what the classic buildpack + // permitted (however it's primarily trailing newlines that we need to support). The + // string is then escaped, to aid debugging when non-ascii characters have inadvertently + // been used, such as when an editor has auto-corrected the hyphen to an en/em dash. + let cleaned_contents = contents.trim().escape_default().to_string(); + + let version_substring = + cleaned_contents + .strip_prefix("python-") + .ok_or_else(|| ParseRuntimeTxtError { + cleaned_contents: cleaned_contents.clone(), + })?; + + match version_substring + .split('.') + .map(str::parse) + .collect::, _>>() + .unwrap_or_default() + .as_slice() + { + &[major, minor, patch] => Ok(PythonVersion::new(major, minor, patch)), + _ => Err(ParseRuntimeTxtError { + cleaned_contents: cleaned_contents.clone(), + }), + } +} + +#[derive(Debug)] +pub(crate) enum ReadRuntimeTxtError { + Io(io::Error), + Parse(ParseRuntimeTxtError), +} + +#[derive(Debug, PartialEq)] +pub(crate) struct ParseRuntimeTxtError { + pub cleaned_contents: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_valid() { + assert_eq!(parse("python-1.2.3"), Ok(PythonVersion::new(1, 2, 3))); + assert_eq!( + parse("python-987.654.3210"), + Ok(PythonVersion::new(987, 654, 3210)) + ); + assert_eq!( + parse("\n python-1.2.3 \n"), + Ok(PythonVersion::new(1, 2, 3)) + ); + } + + #[test] + fn parse_invalid_prefix() { + assert_eq!( + parse(""), + Err(ParseRuntimeTxtError { + cleaned_contents: String::new() + }) + ); + assert_eq!( + parse("1.2.3"), + Err(ParseRuntimeTxtError { + cleaned_contents: "1.2.3".to_string() + }) + ); + assert_eq!( + parse("python 1.2.3"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python 1.2.3".to_string() + }) + ); + assert_eq!( + parse("python -1.2.3"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python -1.2.3".to_string() + }) + ); + assert_eq!( + parse("abc-1.2.3"), + Err(ParseRuntimeTxtError { + cleaned_contents: "abc-1.2.3".to_string() + }) + ); + assert_eq!( + parse("\n -1.2.3 \n"), + Err(ParseRuntimeTxtError { + cleaned_contents: "-1.2.3".to_string() + }) + ); + assert_eq!( + // En dash. + parse("python–1.2.3"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python\\u{2013}1.2.3".to_string() + }) + ); + assert_eq!( + // Em dash. + parse("python—1.2.3"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python\\u{2014}1.2.3".to_string() + }) + ); + } + + #[test] + fn parse_invalid_version() { + assert_eq!( + parse("python-1"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python-1".to_string(), + }) + ); + assert_eq!( + parse("python-1.2"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python-1.2".to_string(), + }) + ); + assert_eq!( + parse("python-1.2.3.4"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python-1.2.3.4".to_string(), + }) + ); + assert_eq!( + parse("python-1..3"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python-1..3".to_string(), + }) + ); + assert_eq!( + parse("python-1.2.3."), + Err(ParseRuntimeTxtError { + cleaned_contents: "python-1.2.3.".to_string(), + }) + ); + assert_eq!( + parse("python- 1.2.3"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python- 1.2.3".to_string(), + }) + ); + assert_eq!( + parse("\n python-1.2.3a \n"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python-1.2.3a".to_string(), + }) + ); + // These are valid semver versions, but not supported Python versions. + assert_eq!( + parse("python-1.2.3-dev"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python-1.2.3-dev".to_string(), + }) + ); + assert_eq!( + parse("python-1.2.3+abc"), + Err(ParseRuntimeTxtError { + cleaned_contents: "python-1.2.3+abc".to_string(), + }) + ); + } +} diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..fc0836e --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,80 @@ +use flate2::read::GzDecoder; +use std::path::Path; +use std::process::{Command, ExitStatus}; +use std::{fs, io}; +use tar::Archive; + +// TODO: Unit test that all files from PACKAGE_MANAGER_FILES are in here. +const KNOWN_PYTHON_PROJECT_FILES: [&str; 9] = [ + ".python-version", + "main.py", + "manage.py", + "Pipfile", + "poetry.lock", + "pyproject.toml", + "requirements.txt", + "runtime.txt", + "setup.py", +]; + +// TODO: Unit test +pub(crate) fn is_python_project(app_dir: &Path) -> io::Result { + // Until `Iterator::try_find` is stabilised, this is cleaner as a for loop. + for filename in KNOWN_PYTHON_PROJECT_FILES { + if app_dir.join(filename).try_exists()? { + return Ok(true); + } + } + + Ok(false) +} + +// TODO: Unit test +pub(crate) fn read_optional_file(path: &Path) -> io::Result> { + fs::read_to_string(path) + .map(Some) + .or_else(|io_error| match io_error.kind() { + io::ErrorKind::NotFound => Ok(None), + _ => Err(io_error), + }) +} + +pub(crate) fn download_and_unpack_gzipped_archive( + uri: &str, + destination: &Path, +) -> Result<(), DownloadUnpackArchiveError> { + // TODO: Timeouts: https://docs.rs/ureq/latest/ureq/struct.AgentBuilder.html?search=timeout + // TODO: Retries + let response = ureq::get(uri) + .call() + .map_err(DownloadUnpackArchiveError::Request)?; + let gzip_decoder = GzDecoder::new(response.into_reader()); + Archive::new(gzip_decoder) + .unpack(destination) + .map_err(DownloadUnpackArchiveError::Io) +} + +#[derive(Debug)] +pub(crate) enum DownloadUnpackArchiveError { + Io(io::Error), + Request(ureq::Error), +} + +pub(crate) fn run_command(command: &mut Command) -> Result<(), CommandError> { + command + .status() + .map_err(CommandError::Io) + .and_then(|exit_status| { + if exit_status.success() { + Ok(()) + } else { + Err(CommandError::NonZeroExitStatus(exit_status)) + } + }) +} + +#[derive(Debug)] +pub(crate) enum CommandError { + Io(io::Error), + NonZeroExitStatus(ExitStatus), +} diff --git a/test-fixtures/default/requirements.txt b/test-fixtures/default/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/test-fixtures/empty/.gitkeep b/test-fixtures/empty/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test-fixtures/function_invalid_not_async/main.py b/test-fixtures/function_invalid_not_async/main.py new file mode 100644 index 0000000..b46ee45 --- /dev/null +++ b/test-fixtures/function_invalid_not_async/main.py @@ -0,0 +1,5 @@ +from salesforce_functions import Context, InvocationEvent + + +def function(_event: InvocationEvent[None], _context: Context) -> None: + return None diff --git a/test-fixtures/function_invalid_not_async/project.toml b/test-fixtures/function_invalid_not_async/project.toml new file mode 100644 index 0000000..ef6d5f8 --- /dev/null +++ b/test-fixtures/function_invalid_not_async/project.toml @@ -0,0 +1,2 @@ +[com.salesforce] +type = "function" diff --git a/test-fixtures/function_invalid_not_async/requirements.txt b/test-fixtures/function_invalid_not_async/requirements.txt new file mode 100644 index 0000000..f291837 --- /dev/null +++ b/test-fixtures/function_invalid_not_async/requirements.txt @@ -0,0 +1,5 @@ +# Once Python support for Salesforce Functions is in beta, the salesforce-functions +# package will be published to PyPI, and the GitHub URL here can be replaced by the +# PyPI package name instead. For example: +# salesforce-functions>=0.1.0,<0.2.0 +salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git diff --git a/test-fixtures/function_missing_functions_package/main.py b/test-fixtures/function_missing_functions_package/main.py new file mode 100644 index 0000000..80920de --- /dev/null +++ b/test-fixtures/function_missing_functions_package/main.py @@ -0,0 +1,5 @@ +from salesforce_functions import Context, InvocationEvent + + +async def function(_event: InvocationEvent[None], _context: Context) -> None: + return None diff --git a/test-fixtures/function_missing_functions_package/project.toml b/test-fixtures/function_missing_functions_package/project.toml new file mode 100644 index 0000000..ef6d5f8 --- /dev/null +++ b/test-fixtures/function_missing_functions_package/project.toml @@ -0,0 +1,2 @@ +[com.salesforce] +type = "function" diff --git a/test-fixtures/function_missing_functions_package/requirements.txt b/test-fixtures/function_missing_functions_package/requirements.txt new file mode 100644 index 0000000..fbb9d22 --- /dev/null +++ b/test-fixtures/function_missing_functions_package/requirements.txt @@ -0,0 +1 @@ +# The salesforce-functions package is missing from here. diff --git a/test-fixtures/function_python_3.10/main.py b/test-fixtures/function_python_3.10/main.py new file mode 100644 index 0000000..4c05d20 --- /dev/null +++ b/test-fixtures/function_python_3.10/main.py @@ -0,0 +1,20 @@ +from typing import Any + +from salesforce_functions import Context, InvocationEvent, get_logger + +# The type of the data payload sent with the invocation event. +# Change this to a more specific type matching the expected payload for +# improved IDE auto-completion and linting coverage. For example: +# `EventPayloadType = dict[str, Any]` +EventPayloadType = Any + +logger = get_logger() + + +async def function(event: InvocationEvent[EventPayloadType], context: Context): + """Describe the function here.""" + + result = await context.org.data_api.query("SELECT Id, Name FROM Account") + logger.info(f"Function successfully queried {result.total_size} account records!") + + return result.records diff --git a/test-fixtures/function_python_3.10/project.toml b/test-fixtures/function_python_3.10/project.toml new file mode 100644 index 0000000..ef6d5f8 --- /dev/null +++ b/test-fixtures/function_python_3.10/project.toml @@ -0,0 +1,2 @@ +[com.salesforce] +type = "function" diff --git a/test-fixtures/function_python_3.10/requirements.txt b/test-fixtures/function_python_3.10/requirements.txt new file mode 100644 index 0000000..f291837 --- /dev/null +++ b/test-fixtures/function_python_3.10/requirements.txt @@ -0,0 +1,5 @@ +# Once Python support for Salesforce Functions is in beta, the salesforce-functions +# package will be published to PyPI, and the GitHub URL here can be replaced by the +# PyPI package name instead. For example: +# salesforce-functions>=0.1.0,<0.2.0 +salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git diff --git a/test-fixtures/function_python_3.10/runtime.txt b/test-fixtures/function_python_3.10/runtime.txt new file mode 100644 index 0000000..19c64f2 --- /dev/null +++ b/test-fixtures/function_python_3.10/runtime.txt @@ -0,0 +1 @@ +python-3.10.9 diff --git a/test-fixtures/function_python_version_invalid/main.py b/test-fixtures/function_python_version_invalid/main.py new file mode 100644 index 0000000..80920de --- /dev/null +++ b/test-fixtures/function_python_version_invalid/main.py @@ -0,0 +1,5 @@ +from salesforce_functions import Context, InvocationEvent + + +async def function(_event: InvocationEvent[None], _context: Context) -> None: + return None diff --git a/test-fixtures/function_python_version_invalid/project.toml b/test-fixtures/function_python_version_invalid/project.toml new file mode 100644 index 0000000..ef6d5f8 --- /dev/null +++ b/test-fixtures/function_python_version_invalid/project.toml @@ -0,0 +1,2 @@ +[com.salesforce] +type = "function" diff --git a/test-fixtures/function_python_version_invalid/requirements.txt b/test-fixtures/function_python_version_invalid/requirements.txt new file mode 100644 index 0000000..f291837 --- /dev/null +++ b/test-fixtures/function_python_version_invalid/requirements.txt @@ -0,0 +1,5 @@ +# Once Python support for Salesforce Functions is in beta, the salesforce-functions +# package will be published to PyPI, and the GitHub URL here can be replaced by the +# PyPI package name instead. For example: +# salesforce-functions>=0.1.0,<0.2.0 +salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git diff --git a/test-fixtures/function_python_version_invalid/runtime.txt b/test-fixtures/function_python_version_invalid/runtime.txt new file mode 100644 index 0000000..606a469 --- /dev/null +++ b/test-fixtures/function_python_version_invalid/runtime.txt @@ -0,0 +1 @@ +python-an.invalid.version diff --git a/test-fixtures/function_python_version_too_old/main.py b/test-fixtures/function_python_version_too_old/main.py new file mode 100644 index 0000000..80920de --- /dev/null +++ b/test-fixtures/function_python_version_too_old/main.py @@ -0,0 +1,5 @@ +from salesforce_functions import Context, InvocationEvent + + +async def function(_event: InvocationEvent[None], _context: Context) -> None: + return None diff --git a/test-fixtures/function_python_version_too_old/project.toml b/test-fixtures/function_python_version_too_old/project.toml new file mode 100644 index 0000000..ef6d5f8 --- /dev/null +++ b/test-fixtures/function_python_version_too_old/project.toml @@ -0,0 +1,2 @@ +[com.salesforce] +type = "function" diff --git a/test-fixtures/function_python_version_too_old/requirements.txt b/test-fixtures/function_python_version_too_old/requirements.txt new file mode 100644 index 0000000..f291837 --- /dev/null +++ b/test-fixtures/function_python_version_too_old/requirements.txt @@ -0,0 +1,5 @@ +# Once Python support for Salesforce Functions is in beta, the salesforce-functions +# package will be published to PyPI, and the GitHub URL here can be replaced by the +# PyPI package name instead. For example: +# salesforce-functions>=0.1.0,<0.2.0 +salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git diff --git a/test-fixtures/function_python_version_too_old/runtime.txt b/test-fixtures/function_python_version_too_old/runtime.txt new file mode 100644 index 0000000..c9cbcea --- /dev/null +++ b/test-fixtures/function_python_version_too_old/runtime.txt @@ -0,0 +1 @@ +python-3.9.16 diff --git a/test-fixtures/function_python_version_unavailable/main.py b/test-fixtures/function_python_version_unavailable/main.py new file mode 100644 index 0000000..80920de --- /dev/null +++ b/test-fixtures/function_python_version_unavailable/main.py @@ -0,0 +1,5 @@ +from salesforce_functions import Context, InvocationEvent + + +async def function(_event: InvocationEvent[None], _context: Context) -> None: + return None diff --git a/test-fixtures/function_python_version_unavailable/project.toml b/test-fixtures/function_python_version_unavailable/project.toml new file mode 100644 index 0000000..ef6d5f8 --- /dev/null +++ b/test-fixtures/function_python_version_unavailable/project.toml @@ -0,0 +1,2 @@ +[com.salesforce] +type = "function" diff --git a/test-fixtures/function_python_version_unavailable/requirements.txt b/test-fixtures/function_python_version_unavailable/requirements.txt new file mode 100644 index 0000000..f291837 --- /dev/null +++ b/test-fixtures/function_python_version_unavailable/requirements.txt @@ -0,0 +1,5 @@ +# Once Python support for Salesforce Functions is in beta, the salesforce-functions +# package will be published to PyPI, and the GitHub URL here can be replaced by the +# PyPI package name instead. For example: +# salesforce-functions>=0.1.0,<0.2.0 +salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git diff --git a/test-fixtures/function_python_version_unavailable/runtime.txt b/test-fixtures/function_python_version_unavailable/runtime.txt new file mode 100644 index 0000000..e67d1c2 --- /dev/null +++ b/test-fixtures/function_python_version_unavailable/runtime.txt @@ -0,0 +1 @@ +python-999.999.999 diff --git a/test-fixtures/function_template/README.md b/test-fixtures/function_template/README.md new file mode 100644 index 0000000..4bcafa9 --- /dev/null +++ b/test-fixtures/function_template/README.md @@ -0,0 +1,3 @@ +# Pythonexample Function + + diff --git a/test-fixtures/function_template/main.py b/test-fixtures/function_template/main.py new file mode 100644 index 0000000..4c05d20 --- /dev/null +++ b/test-fixtures/function_template/main.py @@ -0,0 +1,20 @@ +from typing import Any + +from salesforce_functions import Context, InvocationEvent, get_logger + +# The type of the data payload sent with the invocation event. +# Change this to a more specific type matching the expected payload for +# improved IDE auto-completion and linting coverage. For example: +# `EventPayloadType = dict[str, Any]` +EventPayloadType = Any + +logger = get_logger() + + +async def function(event: InvocationEvent[EventPayloadType], context: Context): + """Describe the function here.""" + + result = await context.org.data_api.query("SELECT Id, Name FROM Account") + logger.info(f"Function successfully queried {result.total_size} account records!") + + return result.records diff --git a/test-fixtures/function_template/payload.json b/test-fixtures/function_template/payload.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test-fixtures/function_template/payload.json @@ -0,0 +1 @@ +{} diff --git a/test-fixtures/function_template/project.toml b/test-fixtures/function_template/project.toml new file mode 100644 index 0000000..ac505f2 --- /dev/null +++ b/test-fixtures/function_template/project.toml @@ -0,0 +1,9 @@ +[_] +schema-version = "0.2" + +[com.salesforce] +schema-version = "0.1" +id = "pythonexample" +description = "A Salesforce Function" +type = "function" +salesforce-api-version = "56.0" diff --git a/test-fixtures/function_template/requirements.txt b/test-fixtures/function_template/requirements.txt new file mode 100644 index 0000000..f291837 --- /dev/null +++ b/test-fixtures/function_template/requirements.txt @@ -0,0 +1,5 @@ +# Once Python support for Salesforce Functions is in beta, the salesforce-functions +# package will be published to PyPI, and the GitHub URL here can be replaced by the +# PyPI package name instead. For example: +# salesforce-functions>=0.1.0,<0.2.0 +salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git diff --git a/test-fixtures/project_toml_invalid/project.toml b/test-fixtures/project_toml_invalid/project.toml new file mode 100644 index 0000000..7a6399a --- /dev/null +++ b/test-fixtures/project_toml_invalid/project.toml @@ -0,0 +1,5 @@ +[_] +schema-version = "0.2" + +[com.salesforce] +# The required fields here are missing. diff --git a/test-fixtures/project_toml_non_salesforce/project.toml b/test-fixtures/project_toml_non_salesforce/project.toml new file mode 100644 index 0000000..dd8b5ef --- /dev/null +++ b/test-fixtures/project_toml_non_salesforce/project.toml @@ -0,0 +1,8 @@ +[_] +schema-version = "0.2" + +[io.buildpacks] +builder = "my-builder" + +[com.example] +key = "value" diff --git a/tests/integration.rs b/tests/integration.rs new file mode 100644 index 0000000..e88f751 --- /dev/null +++ b/tests/integration.rs @@ -0,0 +1,369 @@ +//! All integration tests are skipped by default (using the `ignore` attribute), +//! since performing builds is slow. To run the tests use: `cargo test -- --ignored` + +#![warn(clippy::pedantic)] + +use indoc::indoc; +use libcnb_test::{assert_contains, BuildConfig, ContainerConfig, PackResult, TestRunner}; +use std::thread; +use std::time::Duration; + +const TEST_PORT: u16 = 12345; + +// For now, these integration tests only cover functions, since: +// - that's what needs to ship first +// - the buildpack's detect by design rejects anything but a function, so for now +// all tests here need to actually be a function to get past detect + +#[test] +#[ignore = "integration test"] +fn detect_rejects_non_functions() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/default") + .expected_pack_result(PackResult::Failure), + |context| { + // We can't test the detect failure reason, since by default pack CLI only shows output for + // non-zero, non-100 exit codes, and `libcnb-test` support enabling pack build's verbose mode: + // https://github.com/heroku/libcnb.rs/issues/383 + assert_contains!( + context.pack_stdout, + "ERROR: No buildpack groups passed detection." + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn function_template() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/function_template"), + |context| { + // Pip outputs git clone output to stderr for some reason, so stderr isn't empty. + // TODO: Decide whether this is a bug in pip and/or if we should work around it. + // assert_empty!(context.pack_stderr); + + assert_contains!( + context.pack_stdout, + indoc! {" + [Determining Python version] + No Python version specified, using the current default of 3.11.1. + To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes + + [Installing Python] + Downloading Python 3.11.1 + Python installation successful + + [Installing Pip] + Installing pip 22.3.1, setuptools 65.6.3 and wheel 0.38.3 + Installation completed + + [Installing dependencies using Pip] + Pip cache created + Running pip install + Collecting salesforce-functions@ git+https://github.com/heroku/sf-functions-python.git + "} + ); + + assert_contains!( + context.pack_stdout, + indoc! {" + Pip install completed + + [Validating Salesforce Function] + Function passed validation. + "} + ); + + context.start_container( + ContainerConfig::new() + .env("PORT", TEST_PORT.to_string()) + .expose_port(TEST_PORT), + |container| { + let address_on_host = container.address_for_port(TEST_PORT).unwrap(); + let url = format!("http://{}:{}", address_on_host.ip(), address_on_host.port()); + + // Retries needed since the server takes a moment to start up. + let mut attempts_remaining = 5; + let response = loop { + let response = ureq::post(&url).set("x-health-check", "true").call(); + if response.is_ok() || attempts_remaining == 0 { + break response; + } + attempts_remaining -= 1; + thread::sleep(Duration::from_secs(1)); + }; + + let server_log_output = container.logs_now(); + assert_contains!( + server_log_output.stderr, + &format!("Uvicorn running on http://0.0.0.0:{TEST_PORT}") + ); + + let body = response.unwrap().into_string().unwrap(); + assert_eq!(body, r#""OK""#); + }, + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn function_repeat_build() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/function_template"), + |context| { + let config = context.config.clone(); + context.rebuild(config, |rebuild_context| { + assert_contains!( + rebuild_context.pack_stdout, + indoc! {" + [Determining Python version] + No Python version specified, using the current default of 3.11.1. + To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes + + [Installing Python] + Re-using cached Python 3.11.1 + + [Installing Pip] + Re-using cached pip 22.3.1, setuptools 65.6.3 and wheel 0.38.3 + + [Installing dependencies using Pip] + Re-using cached pip-cache + Running pip install + Collecting salesforce-functions@ git+https://github.com/heroku/sf-functions-python.git + "} + ); + }); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn function_python_3_10() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "test-fixtures/function_python_3.10"), + |context| { + assert_contains!( + context.pack_stdout, + indoc! {" + [Determining Python version] + Using Python version 3.10.9 specified in runtime.txt + + [Installing Python] + Downloading Python 3.10.9 + Python installation successful + "} + ); + + assert_contains!( + context.pack_stdout, + indoc! {" + Pip install completed + + [Validating Salesforce Function] + Function passed validation. + "} + ); + + context.start_container( + ContainerConfig::new() + .env("PORT", TEST_PORT.to_string()) + .expose_port(TEST_PORT), + |container| { + let address_on_host = container.address_for_port(TEST_PORT).unwrap(); + let url = format!("http://{}:{}", address_on_host.ip(), address_on_host.port()); + + // Retries needed since the server takes a moment to start up. + let mut attempts_remaining = 5; + let response = loop { + let response = ureq::post(&url).set("x-health-check", "true").call(); + if response.is_ok() || attempts_remaining == 0 { + break response; + } + attempts_remaining -= 1; + thread::sleep(Duration::from_secs(1)); + }; + + let server_log_output = container.logs_now(); + assert_contains!( + server_log_output.stderr, + &format!("Uvicorn running on http://0.0.0.0:{TEST_PORT}") + ); + + let body = response.unwrap().into_string().unwrap(); + assert_eq!(body, r#""OK""#); + }, + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn function_python_version_too_old() { + TestRunner::default().build( + BuildConfig::new( + "heroku/builder:22", + "test-fixtures/function_python_version_too_old", + ) + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {" + ERROR: Package 'salesforce-functions' requires a different Python: 3.9.16 not in '>=3.10' + + [Error: Unable to install dependencies using pip] + The 'pip install' command to install the application's dependencies from + 'requirements.txt' failed (exit status: 1). + + See the log output above for more information. + "} + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn function_python_version_unavailable() { + TestRunner::default().build( + BuildConfig::new( + "heroku/builder:22", + "test-fixtures/function_python_version_unavailable", + ) + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {" + [Error: Requested Python version is not available] + The requested Python version (999.999.999) is not available for this stack (heroku-22). + + Please update the version in 'runtime.txt' to a supported Python version, or else + remove the file to instead use the default version (currently Python 3.11.1). + + For a list of the supported Python versions, see: + https://devcenter.heroku.com/articles/python-support#supported-runtimes + "} + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn function_python_version_invalid() { + TestRunner::default().build( + BuildConfig::new( + "heroku/builder:22", + "test-fixtures/function_python_version_invalid", + ) + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {" + [Error: Invalid Python version in runtime.txt] + The Python version specified in 'runtime.txt' is not in the correct format. + + The following file contents were found: + python-an.invalid.version + + However, the file contents must begin with a 'python-' prefix, followed by the + version specified as '..'. Comments are not supported. + + For example, to request Python 3.11.1, the correct version format is: + python-3.11.1 + + Please update 'runtime.txt' to use the correct version format, or else remove + the file to instead use the default version (currently Python 3.11.1). + + For a list of the supported Python versions, see: + https://devcenter.heroku.com/articles/python-support#supported-runtimes + "} + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn function_missing_functions_package() { + TestRunner::default().build( + BuildConfig::new( + "heroku/builder:22", + "test-fixtures/function_missing_functions_package", + ) + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {r#" + [Error: The Salesforce Functions package is not installed] + The 'sf-functions-python' program that is required for Python Salesforce + Functions could not be found. + + Check that the 'salesforce-functions' Python package is listed as a + dependency in 'requirements.txt'. + + If this project is not intended to be a Salesforce Function, remove the + 'type = "function"' declaration from 'project.toml' to skip this check. + "#} + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn function_fails_self_check() { + TestRunner::default().build( + BuildConfig::new( + "heroku/builder:22", + "test-fixtures/function_invalid_not_async", + ) + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {" + [Error: The Salesforce Functions self-check failed] + The 'sf-functions-python check' command failed (exit status: 1), indicating + there is a problem with the Python Salesforce Function in this project. + + Details: + Error: Function failed validation! The function named 'function' must be an async function. Change the function definition from 'def function' to 'async def function'. + "} + ); + }, + ); +} + +// TODO: +// +// Detect +// - no Python files +// +// Python versions +// - Default +// - 3.11. +// - 3.11. (show update warning) +// - 3.10. +// - 3.9. +// - 3.8 (unsupported, show reason) +// - 3.7 (unsupported, show reason) +// - 3.6 (unsupported, explain EOL) +// - various invalid version strings +// +// Caching +// - Python version change +// - Stack change +// - Various Pip cache invalidation types (package additions/removals etc) +// - No-op +// +// Other +// - that pip install can find Python headers From d87a78d73306e92878c9b8d64af0213eefb4420e Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 14 Dec 2022 22:36:46 +0000 Subject: [PATCH 02/71] Install salesforce-functions from PyPI now it's been published --- test-fixtures/function_invalid_not_async/requirements.txt | 6 +----- test-fixtures/function_python_3.10/requirements.txt | 6 +----- .../function_python_version_invalid/requirements.txt | 6 +----- .../function_python_version_too_old/requirements.txt | 6 +----- .../function_python_version_unavailable/requirements.txt | 6 +----- test-fixtures/function_template/requirements.txt | 6 +----- tests/integration.rs | 8 +++++--- 7 files changed, 11 insertions(+), 33 deletions(-) diff --git a/test-fixtures/function_invalid_not_async/requirements.txt b/test-fixtures/function_invalid_not_async/requirements.txt index f291837..ced5be3 100644 --- a/test-fixtures/function_invalid_not_async/requirements.txt +++ b/test-fixtures/function_invalid_not_async/requirements.txt @@ -1,5 +1 @@ -# Once Python support for Salesforce Functions is in beta, the salesforce-functions -# package will be published to PyPI, and the GitHub URL here can be replaced by the -# PyPI package name instead. For example: -# salesforce-functions>=0.1.0,<0.2.0 -salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git +salesforce-functions diff --git a/test-fixtures/function_python_3.10/requirements.txt b/test-fixtures/function_python_3.10/requirements.txt index f291837..ced5be3 100644 --- a/test-fixtures/function_python_3.10/requirements.txt +++ b/test-fixtures/function_python_3.10/requirements.txt @@ -1,5 +1 @@ -# Once Python support for Salesforce Functions is in beta, the salesforce-functions -# package will be published to PyPI, and the GitHub URL here can be replaced by the -# PyPI package name instead. For example: -# salesforce-functions>=0.1.0,<0.2.0 -salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git +salesforce-functions diff --git a/test-fixtures/function_python_version_invalid/requirements.txt b/test-fixtures/function_python_version_invalid/requirements.txt index f291837..ced5be3 100644 --- a/test-fixtures/function_python_version_invalid/requirements.txt +++ b/test-fixtures/function_python_version_invalid/requirements.txt @@ -1,5 +1 @@ -# Once Python support for Salesforce Functions is in beta, the salesforce-functions -# package will be published to PyPI, and the GitHub URL here can be replaced by the -# PyPI package name instead. For example: -# salesforce-functions>=0.1.0,<0.2.0 -salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git +salesforce-functions diff --git a/test-fixtures/function_python_version_too_old/requirements.txt b/test-fixtures/function_python_version_too_old/requirements.txt index f291837..ced5be3 100644 --- a/test-fixtures/function_python_version_too_old/requirements.txt +++ b/test-fixtures/function_python_version_too_old/requirements.txt @@ -1,5 +1 @@ -# Once Python support for Salesforce Functions is in beta, the salesforce-functions -# package will be published to PyPI, and the GitHub URL here can be replaced by the -# PyPI package name instead. For example: -# salesforce-functions>=0.1.0,<0.2.0 -salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git +salesforce-functions diff --git a/test-fixtures/function_python_version_unavailable/requirements.txt b/test-fixtures/function_python_version_unavailable/requirements.txt index f291837..ced5be3 100644 --- a/test-fixtures/function_python_version_unavailable/requirements.txt +++ b/test-fixtures/function_python_version_unavailable/requirements.txt @@ -1,5 +1 @@ -# Once Python support for Salesforce Functions is in beta, the salesforce-functions -# package will be published to PyPI, and the GitHub URL here can be replaced by the -# PyPI package name instead. For example: -# salesforce-functions>=0.1.0,<0.2.0 -salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git +salesforce-functions diff --git a/test-fixtures/function_template/requirements.txt b/test-fixtures/function_template/requirements.txt index f291837..ced5be3 100644 --- a/test-fixtures/function_template/requirements.txt +++ b/test-fixtures/function_template/requirements.txt @@ -1,5 +1 @@ -# Once Python support for Salesforce Functions is in beta, the salesforce-functions -# package will be published to PyPI, and the GitHub URL here can be replaced by the -# PyPI package name instead. For example: -# salesforce-functions>=0.1.0,<0.2.0 -salesforce-functions @ git+https://github.com/heroku/sf-functions-python.git +salesforce-functions diff --git a/tests/integration.rs b/tests/integration.rs index e88f751..c2f8702 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -61,7 +61,7 @@ fn function_template() { [Installing dependencies using Pip] Pip cache created Running pip install - Collecting salesforce-functions@ git+https://github.com/heroku/sf-functions-python.git + Collecting salesforce-functions "} ); @@ -132,7 +132,7 @@ fn function_repeat_build() { [Installing dependencies using Pip] Re-using cached pip-cache Running pip install - Collecting salesforce-functions@ git+https://github.com/heroku/sf-functions-python.git + Collecting salesforce-functions "} ); }); @@ -214,7 +214,9 @@ fn function_python_version_too_old() { assert_contains!( context.pack_stderr, indoc! {" - ERROR: Package 'salesforce-functions' requires a different Python: 3.9.16 not in '>=3.10' + ERROR: Ignored the following versions that require a different python version: 0.1.0 Requires-Python >=3.10 + ERROR: Could not find a version that satisfies the requirement salesforce-functions (from versions: none) + ERROR: No matching distribution found for salesforce-functions [Error: Unable to install dependencies using pip] The 'pip install' command to install the application's dependencies from From 4e5ba1e0a203fd6fe60152c256e1439461fda7ae Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 2 Jan 2023 19:04:26 +0000 Subject: [PATCH 03/71] Fix CI after salesforce-functions 0.2.0 release --- src/errors.rs | 1 - .../main.py | 0 .../project.toml | 1 + .../requirements.txt | 0 test-fixtures/function_python_3.10/project.toml | 1 + tests/integration.rs | 6 +++--- 6 files changed, 5 insertions(+), 4 deletions(-) rename test-fixtures/{function_invalid_not_async => function_fails_self_check}/main.py (100%) rename test-fixtures/{function_invalid_not_async => function_fails_self_check}/project.toml (50%) rename test-fixtures/{function_invalid_not_async => function_fails_self_check}/requirements.txt (100%) diff --git a/src/errors.rs b/src/errors.rs index 7f54893..ea79944 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -262,7 +262,6 @@ fn on_check_function_error(check_function_error: CheckFunctionError) { &format!("running the '{FUNCTION_RUNTIME_PROGRAM_NAME} check' command"), &io_error, ), - // TOOO: Clean up the error message from the check command. CheckFunctionError::NonZeroExitStatus(output) => log_error( "The Salesforce Functions self-check failed", formatdoc! {" diff --git a/test-fixtures/function_invalid_not_async/main.py b/test-fixtures/function_fails_self_check/main.py similarity index 100% rename from test-fixtures/function_invalid_not_async/main.py rename to test-fixtures/function_fails_self_check/main.py diff --git a/test-fixtures/function_invalid_not_async/project.toml b/test-fixtures/function_fails_self_check/project.toml similarity index 50% rename from test-fixtures/function_invalid_not_async/project.toml rename to test-fixtures/function_fails_self_check/project.toml index ef6d5f8..2a44a3a 100644 --- a/test-fixtures/function_invalid_not_async/project.toml +++ b/test-fixtures/function_fails_self_check/project.toml @@ -1,2 +1,3 @@ [com.salesforce] type = "function" +salesforce-api-version = "invalid" diff --git a/test-fixtures/function_invalid_not_async/requirements.txt b/test-fixtures/function_fails_self_check/requirements.txt similarity index 100% rename from test-fixtures/function_invalid_not_async/requirements.txt rename to test-fixtures/function_fails_self_check/requirements.txt diff --git a/test-fixtures/function_python_3.10/project.toml b/test-fixtures/function_python_3.10/project.toml index ef6d5f8..332e751 100644 --- a/test-fixtures/function_python_3.10/project.toml +++ b/test-fixtures/function_python_3.10/project.toml @@ -1,2 +1,3 @@ [com.salesforce] type = "function" +salesforce-api-version = "56.0" diff --git a/tests/integration.rs b/tests/integration.rs index c2f8702..0dcacbb 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -214,7 +214,7 @@ fn function_python_version_too_old() { assert_contains!( context.pack_stderr, indoc! {" - ERROR: Ignored the following versions that require a different python version: 0.1.0 Requires-Python >=3.10 + ERROR: Ignored the following versions that require a different python version: 0.1.0 Requires-Python >=3.10; 0.2.0 Requires-Python >=3.10 ERROR: Could not find a version that satisfies the requirement salesforce-functions (from versions: none) ERROR: No matching distribution found for salesforce-functions @@ -326,7 +326,7 @@ fn function_fails_self_check() { TestRunner::default().build( BuildConfig::new( "heroku/builder:22", - "test-fixtures/function_invalid_not_async", + "test-fixtures/function_fails_self_check", ) .expected_pack_result(PackResult::Failure), |context| { @@ -338,7 +338,7 @@ fn function_fails_self_check() { there is a problem with the Python Salesforce Function in this project. Details: - Error: Function failed validation! The function named 'function' must be an async function. Change the function definition from 'def function' to 'async def function'. + Function failed validation: 'invalid' is not a valid Salesforce REST API version. Update 'salesforce-api-version' in project.toml to a version of form 'X.Y'. "} ); }, From fca044cabcab29bde07c27a5ca84121c62ce9e95 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 2 Jan 2023 19:04:42 +0000 Subject: [PATCH 04/71] Bump minimum Rust version to 1.66 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c7e282b..ded9bb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "python-buildpack" version = "0.0.0" edition = "2021" -rust-version = "1.65" +rust-version = "1.66" publish = false [dependencies] From 406f255c78b24e32dca664bbf33b8b70a764c5f8 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 2 Jan 2023 19:05:16 +0000 Subject: [PATCH 05/71] Only use major versions in Cargo.toml Since cargo already treats them as semver version ranges (so this is effectively a no-op), and this way it avoids conflicts/churn in this file from Dependabot PRs. --- Cargo.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ded9bb1..a139f19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,17 +11,17 @@ publish = false # Ideally we'd use the fastest `zlib-ng` backend, however it fails to cross-compile: # https://github.com/rust-lang/libz-sys/issues/93 # As such we have to use the next best alternate backend, which is `zlib`. -flate2 = { version = "1.0.25", default-features = false, features = ["zlib"] } -indoc = "1.0.7" -libcnb = "0.11.1" -libherokubuildpack = { version = "0.11.1", default-features = false, features = ["log"] } -serde = "1.0.149" -tar = "0.4.38" -toml = "0.5.9" -ureq = { version = "2.5.0", default-features = false, features = ["tls"] } +flate2 = { version = "1", default-features = false, features = ["zlib"] } +indoc = "1" +libcnb = "0.11" +libherokubuildpack = { version = "0.11", default-features = false, features = ["log"] } +serde = "1" +tar = "0.4" +toml = "0.5" +ureq = { version = "2", default-features = false, features = ["tls"] } [dev-dependencies] -libcnb-test = "0.11.1" +libcnb-test = "0.11" # [profile.dev] # Speed up downloading/extraction of Python during integration tests. From f687559913a5c0ee4315e8c57b98bb1a72f7d127 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 2 Jan 2023 19:05:48 +0000 Subject: [PATCH 06/71] Skip check changelog for Dependabot PRs Since the vast majority do not need a changelog entry. --- .github/workflows/check_changelog.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check_changelog.yml b/.github/workflows/check_changelog.yml index d9967a8..71acf20 100644 --- a/.github/workflows/check_changelog.yml +++ b/.github/workflows/check_changelog.yml @@ -14,7 +14,8 @@ jobs: !contains(github.event.pull_request.body, '[skip changelog]') && !contains(github.event.pull_request.body, '[changelog skip]') && !contains(github.event.pull_request.body, '[skip ci]') && - !contains(github.event.pull_request.labels.*.name, 'skip changelog') + !contains(github.event.pull_request.labels.*.name, 'skip changelog') && + !contains(github.event.pull_request.labels.*.name, 'dependencies') steps: - name: Checkout uses: actions/checkout@v3 From 35a6dde4aacff9ceeb3ba100078cccabffcada44 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 2 Jan 2023 19:12:34 +0000 Subject: [PATCH 07/71] Refresh lockfile --- Cargo.lock | 138 +++++++++++++++++++++-------------------------------- 1 file changed, 54 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58912fe..67a8767 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" [[package]] name = "cfg-if" @@ -135,12 +135,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chunked_transfer" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" - [[package]] name = "crc32fast" version = "1.3.2" @@ -333,9 +327,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" dependencies = [ "libc", ] @@ -445,9 +439,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adab1eaa3408fb7f0c777a73e7465fd5656136fc93b670eb6df3c88c2c1344e3" +checksum = "da2d6f23ffea9d7e76c53eee25dfb67bcd8fde7f1198b0855350698c9f07c780" [[package]] name = "instant" @@ -460,9 +454,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "js-sys" @@ -475,29 +469,28 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.138" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libcnb" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88e7663908798e4c9a6ce419220e9493ad19dd12a70fa605dd4e927e3fdc1fc9" +checksum = "525c69469fb63994037d42e13fe0218b95efeaa4f7c7da50ec20b8a0f4133d6e" dependencies = [ "libcnb-data", "libcnb-proc-macros", "serde", - "stacker", "thiserror", "toml", ] [[package]] name = "libcnb-data" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "066632abe99f4ca2de170a3c3f5946253e63c715e9b7b1a31341de883cb03246" +checksum = "ad960f5527b27ca85ec621876d7b4d03f17cc5c752727ad9db76727c8962afb0" dependencies = [ "fancy-regex", "libcnb-proc-macros", @@ -508,9 +501,9 @@ dependencies = [ [[package]] name = "libcnb-package" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa1d102e9d744212b2bfe650a5012fb93609700642fde5e778dc27103fc1b26e" +checksum = "b19bfe059955e9c0bc79335e4f2a24d6d1fb1bc7af66d6273b5b717535a27446" dependencies = [ "cargo_metadata", "libcnb-data", @@ -520,9 +513,9 @@ dependencies = [ [[package]] name = "libcnb-proc-macros" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "092a9091dc629e2fafb9a24e7b9050ca3e84565f82c5a67aecaf8afa24234d32" +checksum = "0818a0b0a3ff34b0d585ab2180aec1ad701593d16ceb0be7f87aa6b57a37b6fa" dependencies = [ "cargo_metadata", "fancy-regex", @@ -532,9 +525,9 @@ dependencies = [ [[package]] name = "libcnb-test" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "318a615aed63ac85117eb0c149105ad9ca956cd3133d11d1a83fb011941802f8" +checksum = "a26403ea43f48643acc19561491ca9e7b738eec0be2b02f96ea0c74518f9a63e" dependencies = [ "bollard", "cargo_metadata", @@ -550,9 +543,9 @@ dependencies = [ [[package]] name = "libherokubuildpack" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bffa56c2d7cf1c9265126f368343d6d5bf34c33da7ddf0d5ac245da9e999265" +checksum = "808d58945c03f51376de7e359e979f6f259473ee27c03188dcedd7ef41224ba5" dependencies = [ "termcolor", ] @@ -606,9 +599,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ "hermit-abi", "libc", @@ -616,9 +609,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "percent-encoding" @@ -666,22 +659,13 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] -[[package]] -name = "psm" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" -dependencies = [ - "cc", -] - [[package]] name = "python-buildpack" version = "0.0.0" @@ -699,9 +683,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -768,9 +752,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "sct" @@ -784,27 +768,27 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.149" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.149" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -813,9 +797,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "itoa", "ryu", @@ -881,19 +865,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "stacker" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" -dependencies = [ - "cc", - "cfg-if", - "libc", - "psm", - "winapi", -] - [[package]] name = "strsim" version = "0.10.0" @@ -902,9 +873,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -947,18 +918,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -1024,9 +995,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" dependencies = [ "serde", ] @@ -1071,9 +1042,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" @@ -1092,12 +1063,11 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "ureq" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f" +checksum = "cc0c46e911514c4edd735f38d2e493c182c1d4f7a1f89022e14ea3f9833be24b" dependencies = [ "base64", - "chunked_transfer", "log", "once_cell", "rustls", @@ -1215,9 +1185,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ "webpki", ] From 11efc7231e688d58ac2609350eeb79777812f3f2 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 6 Jan 2023 08:56:19 +0000 Subject: [PATCH 08/71] Remove support for skipping check-changelog using the PR description --- .github/workflows/check_changelog.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/check_changelog.yml b/.github/workflows/check_changelog.yml index 71acf20..c590644 100644 --- a/.github/workflows/check_changelog.yml +++ b/.github/workflows/check_changelog.yml @@ -2,7 +2,7 @@ name: Check Changelog on: pull_request: - types: [opened, reopened, edited, labeled, unlabeled, synchronize] + types: [opened, reopened, labeled, unlabeled, synchronize] permissions: contents: read @@ -11,9 +11,6 @@ jobs: check-changelog: runs-on: ubuntu-22.04 if: | - !contains(github.event.pull_request.body, '[skip changelog]') && - !contains(github.event.pull_request.body, '[changelog skip]') && - !contains(github.event.pull_request.body, '[skip ci]') && !contains(github.event.pull_request.labels.*.name, 'skip changelog') && !contains(github.event.pull_request.labels.*.name, 'dependencies') steps: From f518418d9de601f66309d2d4b3450cc523bd8602 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 6 Jan 2023 21:32:43 +0000 Subject: [PATCH 09/71] Update buildpacks/github-actions to 5.0.1 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f149776..c813a82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Rust Cache uses: Swatinem/rust-cache@v2.2.0 - name: Install Pack CLI - uses: buildpacks/github-actions/setup-pack@v4.9.0 + uses: buildpacks/github-actions/setup-pack@v5.0.1 - name: Run integration tests # Runs only tests annotated with the `ignore` attribute (which in this repo, are the integration tests). run: cargo test --locked -- --ignored From cc2fb09e9fe63216b3b251f2f6b23307cf69285a Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Sat, 14 Jan 2023 14:03:03 +0000 Subject: [PATCH 10/71] Remove workarounds for slow M1 performance Since with the new beta Docker for macOS Rosetta feature the build times on M1 have significantly improved. --- Cargo.toml | 6 ------ src/errors.rs | 16 ---------------- src/layers/pip_dependencies.rs | 4 +--- src/layers/python.rs | 32 +++++++------------------------- 4 files changed, 8 insertions(+), 50 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a139f19..0e2a95e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,9 +22,3 @@ ureq = { version = "2", default-features = false, features = ["tls"] } [dev-dependencies] libcnb-test = "0.11" - -# [profile.dev] -# Speed up downloading/extraction of Python during integration tests. -# TODO: Test again to see if it's still worth it, now that the Python archives are smaller + using alternate flate2 backend. -# (now only seems to change the E2E pack build time of an app using urllib3 from 23.4s to 21.8s on M1?) -# opt-level = 1 diff --git a/src/errors.rs b/src/errors.rs index ea79944..83089f9 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -163,22 +163,6 @@ fn on_python_layer_error(python_layer_error: PythonLayerError) { "}, ), }, - PythonLayerError::CompileByteCodeCommand(error) => match error { - CommandError::Io(io_error) => log_io_error( - "Unable to compile Python byte-code", - "running the 'python -m compileall' command", - &io_error, - ), - CommandError::NonZeroExitStatus(exit_status) => log_error( - "Unable to compile Python byte-code", - formatdoc! {" - The 'python -m compileall' command used to compile Python byte-code - for the system 'site-packages' directory failed ({exit_status}). - - See the log output above for more information. - "}, - ), - }, PythonLayerError::DownloadUnpackArchive(error) => match error { DownloadUnpackArchiveError::Io(io_error) => log_io_error( "Unable to unpack the Python archive", diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 56b2584..f098d6e 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -83,9 +83,7 @@ impl Layer for PipDependenciesLayer<'_> { &src_dir.to_string_lossy(), ]) .envs(&env) - // TODO: Decide whether to use this or `--no-compile` + `compileall`. - // If using compileall will need different strategy for `update()`. - // See also: https://github.com/pypa/pip/blob/3820b0e52c7fed2b2c43ba731b718f316e6816d1/src/pip/_internal/operations/install/wheel.py#L616 + // TODO: Explain why we're setting this // Using 1980-01-01T00:00:01Z to avoid: // ValueError: ZIP does not support timestamps before 1980 .env("SOURCE_DATE_EPOCH", "315532800"), diff --git a/src/layers/python.rs b/src/layers/python.rs index 9e1d2aa..a3293bd 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -150,42 +150,25 @@ impl Layer for PythonLayer<'_> { let bundled_pip_module = bundled_pip_module(&python_stdlib_dir).map_err(PythonLayerError::LocateBundledPipIo)?; utils::run_command( - Command::new(&python_binary) + Command::new(python_binary) .args([ &bundled_pip_module.to_string_lossy(), "install", "--no-cache-dir", - "--no-compile", "--no-input", "--quiet", format!("pip=={PIP_VERSION}").as_str(), format!("setuptools=={SETUPTOOLS_VERSION}").as_str(), format!("wheel=={WHEEL_VERSION}").as_str(), ]) - .envs(&env), + .envs(&env) + // TODO: Explain why we're setting this + // Using 1980-01-01T00:00:01Z to avoid: + // ValueError: ZIP does not support timestamps before 1980 + .env("SOURCE_DATE_EPOCH", "315532800"), ) .map_err(PythonLayerError::BootstrapPipCommand)?; - // TODO: Add comment explaining why we're doing this vs pip default compile. - // (on M1 this reduces the time taken for the pip bootstrap from 17.6s to 13.4s) - // TODO: Test performance difference when not running under QEMU - utils::run_command( - Command::new(python_binary) - .args([ - "-m", - "compileall", - "-f", - "-q", - "--invalidation-mode", - "unchecked-hash", - "--workers", - "0", - &site_packages_dir.to_string_lossy(), - ]) - .envs(&env), - ) - .map_err(PythonLayerError::CompileByteCodeCommand)?; - // By default Pip will install into the system site-packages directory if it is writeable // by the current user. Whilst the buildpack's own `pip install` invocations always use // `--user` to ensure application dependencies are instead installed into the user @@ -193,7 +176,7 @@ impl Layer for PythonLayer<'_> { // By making the system site-packages directory read-only, Pip will automatically use // user installs in such cases: // https://github.com/pypa/pip/blob/22.3.1/src/pip/_internal/commands/install.py#L706-L764 - fs::set_permissions(&site_packages_dir, Permissions::from_mode(0o555)) + fs::set_permissions(site_packages_dir, Permissions::from_mode(0o555)) .map_err(PythonLayerError::MakeSitePackagesReadOnlyIo)?; log_info("Installation completed"); @@ -287,7 +270,6 @@ fn generate_layer_metadata( #[derive(Debug)] pub(crate) enum PythonLayerError { BootstrapPipCommand(CommandError), - CompileByteCodeCommand(CommandError), DownloadUnpackArchive(DownloadUnpackArchiveError), LocateBundledPipIo(io::Error), MakeSitePackagesReadOnlyIo(io::Error), From d0ed798d86cdc7178289f522160cc01711187677 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 17 Jan 2023 22:59:41 +0000 Subject: [PATCH 11/71] Refresh Cargo.lock --- Cargo.lock | 88 +++++++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67a8767..6dc23d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,9 +81,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bytes" @@ -93,9 +93,9 @@ checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "camino" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ad0e1e3e88dd237a156ab9f571021b8a158caa0ae44b1968a241efb5144c1e" +checksum = "c77df041dc383319cc661b428b6961a005db4d6808d5e12536931b1ca9556055" dependencies = [ "serde", ] @@ -475,9 +475,9 @@ checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libcnb" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525c69469fb63994037d42e13fe0218b95efeaa4f7c7da50ec20b8a0f4133d6e" +checksum = "a69d45189983fb0a9996ded95236bb8689437b0e1e636dddf2500e9ec27ab4c0" dependencies = [ "libcnb-data", "libcnb-proc-macros", @@ -488,9 +488,9 @@ dependencies = [ [[package]] name = "libcnb-data" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad960f5527b27ca85ec621876d7b4d03f17cc5c752727ad9db76727c8962afb0" +checksum = "d3a065640c66df2a6e54aedbb805d264c87020937323b90eea7397108b73d3aa" dependencies = [ "fancy-regex", "libcnb-proc-macros", @@ -501,9 +501,9 @@ dependencies = [ [[package]] name = "libcnb-package" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b19bfe059955e9c0bc79335e4f2a24d6d1fb1bc7af66d6273b5b717535a27446" +checksum = "d9ed34a92d997ad9b0666ddbcc3995191e7642ee50ffa760497d2fb3bff7c5b5" dependencies = [ "cargo_metadata", "libcnb-data", @@ -513,9 +513,9 @@ dependencies = [ [[package]] name = "libcnb-proc-macros" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0818a0b0a3ff34b0d585ab2180aec1ad701593d16ceb0be7f87aa6b57a37b6fa" +checksum = "25b3879fd4fc4338421de1ec797ab5ef0abe6d0e90f843dbf3b56c25bc703ebe" dependencies = [ "cargo_metadata", "fancy-regex", @@ -525,9 +525,9 @@ dependencies = [ [[package]] name = "libcnb-test" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26403ea43f48643acc19561491ca9e7b738eec0be2b02f96ea0c74518f9a63e" +checksum = "5f414f5b106078d0bbb67b9e3d3bf9e21012f3a318505649e8e99c9d36d200ea" dependencies = [ "bollard", "cargo_metadata", @@ -543,9 +543,9 @@ dependencies = [ [[package]] name = "libherokubuildpack" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808d58945c03f51376de7e359e979f6f259473ee27c03188dcedd7ef41224ba5" +checksum = "8085f21847f46079ce900bf2169e3e51ffa3685dc298aa71056a29d96d4413cb" dependencies = [ "termcolor", ] @@ -659,9 +659,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" dependencies = [ "unicode-ident", ] @@ -701,9 +701,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "regex-syntax", ] @@ -740,9 +740,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.7" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", @@ -909,9 +909,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -953,9 +953,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" dependencies = [ "autocfg", "bytes", @@ -1030,9 +1030,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "unicode-bidi" @@ -1063,9 +1063,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "ureq" -version = "2.6.0" +version = "2.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc0c46e911514c4edd735f38d2e493c182c1d4f7a1f89022e14ea3f9833be24b" +checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d" dependencies = [ "base64", "log", @@ -1251,45 +1251,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "xattr" From fb848becd0449f1ef5b1f34ee406ded4dde7b69b Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 17 Jan 2023 23:00:38 +0000 Subject: [PATCH 12/71] Update test after salesforce-functions v0.3.0 release --- tests/integration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration.rs b/tests/integration.rs index 0dcacbb..590b8f5 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -214,7 +214,7 @@ fn function_python_version_too_old() { assert_contains!( context.pack_stderr, indoc! {" - ERROR: Ignored the following versions that require a different python version: 0.1.0 Requires-Python >=3.10; 0.2.0 Requires-Python >=3.10 + ERROR: Ignored the following versions that require a different python version: 0.1.0 Requires-Python >=3.10; 0.2.0 Requires-Python >=3.10; 0.3.0 Requires-Python >=3.10 ERROR: Could not find a version that satisfies the requirement salesforce-functions (from versions: none) ERROR: No matching distribution found for salesforce-functions From a4abc3f9a6bb1e4d8c89aa9d1a5fb5a013bd0ee9 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 27 Jan 2023 14:01:51 +0000 Subject: [PATCH 13/71] Remove functions integration test for too-old Python Since it's testing an implementation detail of the functions runtime and/or Pip, which isn't helpful to test in this buildpack. --- tests/integration.rs | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/integration.rs b/tests/integration.rs index 590b8f5..dcd1007 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -201,34 +201,6 @@ fn function_python_3_10() { ); } -#[test] -#[ignore = "integration test"] -fn function_python_version_too_old() { - TestRunner::default().build( - BuildConfig::new( - "heroku/builder:22", - "test-fixtures/function_python_version_too_old", - ) - .expected_pack_result(PackResult::Failure), - |context| { - assert_contains!( - context.pack_stderr, - indoc! {" - ERROR: Ignored the following versions that require a different python version: 0.1.0 Requires-Python >=3.10; 0.2.0 Requires-Python >=3.10; 0.3.0 Requires-Python >=3.10 - ERROR: Could not find a version that satisfies the requirement salesforce-functions (from versions: none) - ERROR: No matching distribution found for salesforce-functions - - [Error: Unable to install dependencies using pip] - The 'pip install' command to install the application's dependencies from - 'requirements.txt' failed (exit status: 1). - - See the log output above for more information. - "} - ); - }, - ); -} - #[test] #[ignore = "integration test"] fn function_python_version_unavailable() { From 40aa0af8e502e207afb20fb6c297e7054665bbf7 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 27 Jan 2023 14:02:42 +0000 Subject: [PATCH 14/71] Remove caching support from the pip layer Since we cache the pip cache only instead. --- src/layers/pip_dependencies.rs | 67 +++------------------------------- src/main.rs | 1 - 2 files changed, 5 insertions(+), 63 deletions(-) diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index f098d6e..85bf3d4 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -1,14 +1,12 @@ -use crate::python_version::PythonVersion; use crate::utils::{self, CommandError}; use crate::{BuildpackError, PythonBuildpack}; use libcnb::build::BuildContext; -use libcnb::data::buildpack::StackId; use libcnb::data::layer_content_metadata::LayerTypes; +use libcnb::generic::GenericMetadata; use libcnb::layer::{Layer, LayerResult, LayerResultBuilder}; use libcnb::layer_env::{LayerEnv, ModificationBehavior, Scope}; use libcnb::{Buildpack, Env}; use libherokubuildpack::log::log_info; -use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use std::process::Command; use std::{fs, io}; @@ -16,31 +14,24 @@ use std::{fs, io}; pub(crate) struct PipDependenciesLayer<'a> { pub env: &'a Env, pub pip_cache_dir: PathBuf, - pub python_version: &'a PythonVersion, -} - -#[derive(Clone, Deserialize, PartialEq, Serialize)] -pub(crate) struct PipDependenciesLayerMetadata { - python_version: String, - stack: StackId, } impl Layer for PipDependenciesLayer<'_> { type Buildpack = PythonBuildpack; - type Metadata = PipDependenciesLayerMetadata; + type Metadata = GenericMetadata; fn types(&self) -> LayerTypes { LayerTypes { build: true, - // TODO: Re-enabling caching once remaining invalidation logic finished. cache: false, launch: true, } } + // TODO: Explain why we're not caching here. fn create( &self, - context: &BuildContext, + _context: &BuildContext, layer_path: &Path, ) -> Result, ::Error> { // TODO: Explain PYTHONUSERBASE and that it will contain bin/, lib/.../site-packages/ @@ -59,7 +50,6 @@ impl Layer for PipDependenciesLayer<'_> { log_info("Running pip install"); // TODO: Explain why we're using user install - // TODO: Refactor this out so it can be shared with `update()` // TODO: Mention that we're intentionally not using env_clear() otherwise // PATH won't be set, and Pip won't be able to find things like Git. utils::run_command( @@ -92,55 +82,10 @@ impl Layer for PipDependenciesLayer<'_> { log_info("Pip install completed"); - let layer_metadata = generate_layer_metadata(&context.stack_id, self.python_version); - LayerResultBuilder::new(layer_metadata) + LayerResultBuilder::new(GenericMetadata::default()) .env(layer_env) .build() } - - // TODO: Re-enabling caching once remaining invalidation logic finished. - // fn update( - // &self, - // _context: &BuildContext, - // _layer_data: &LayerData, - // ) -> Result, ::Error> { - // // TODO - // unimplemented!() - // } - // - // fn existing_layer_strategy( - // &self, - // context: &BuildContext, - // layer_data: &LayerData, - // ) -> Result::Error> { - // // TODO: Also invalidate based on requirements.txt contents - // // TODO: Decide whether sub-requirements files should also invalidate? If not, should we warn? - // // TODO: Also invalidate based on time since layer creation - // // TODO: Decide what should be logged - // // TODO: Re-test the performance of caching site-modules vs only caching Pip's cache. - // #[allow(unreachable_code)] - // if layer_data.content_metadata.metadata - // == generate_layer_metadata(&context.stack_id, self.python_version) - // { - // log_info("Re-using cached dependencies"); - // Ok(ExistingLayerStrategy::Update) - // } else { - // log_info("Discarding cached dependencies"); - // Ok(ExistingLayerStrategy::Recreate) - // } - // } -} - -fn generate_layer_metadata( - stack_id: &StackId, - python_version: &PythonVersion, -) -> PipDependenciesLayerMetadata { - // TODO: Add requirements.txt SHA256 or similar - // TODO: Add timestamp field or similar - PipDependenciesLayerMetadata { - python_version: python_version.to_string(), - stack: stack_id.clone(), - } } #[derive(Debug)] @@ -154,5 +99,3 @@ impl From for BuildpackError { Self::PipLayer(error) } } - -// TODO: Unit tests for cache invalidation handling? diff --git a/src/main.rs b/src/main.rs index 9308299..8b0884a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -101,7 +101,6 @@ impl Buildpack for PythonBuildpack { PipDependenciesLayer { env: &env, pip_cache_dir: pip_cache_layer.path, - python_version: &python_version, }, )?; pip_layer.env From 2abc573170fa7bde37821d63a0e90afb9d2e8698 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 27 Jan 2023 14:04:07 +0000 Subject: [PATCH 15/71] Add VSCode configs --- .vscode/extensions.json | 3 +++ .vscode/settings.json | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..4b9c1c6 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["rust-lang.rust-analyzer"], +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..108d59e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "rust-analyzer.check.command": "clippy", + "rust-analyzer.imports.granularity.enforce": true, + "rust-analyzer.imports.granularity.group": "module", + "rust-analyzer.imports.prefix": "crate", +} From ef88b8089736f6eecae5c28144bc32c420068764 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 27 Jan 2023 14:05:00 +0000 Subject: [PATCH 16/71] Bump minimum Rust version to 1.67 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 0e2a95e..c6a38ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "python-buildpack" version = "0.0.0" edition = "2021" -rust-version = "1.66" +rust-version = "1.67" publish = false [dependencies] From 1be9dc4e45b15b2b7528aff9ef76441e76a6bfa3 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 27 Jan 2023 14:05:10 +0000 Subject: [PATCH 17/71] Refresh Cargo.lock --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6dc23d0..ab7ed16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,9 +181,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "fancy-regex" @@ -995,9 +995,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] @@ -1036,9 +1036,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" @@ -1194,9 +1194,9 @@ dependencies = [ [[package]] name = "which" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", "libc", From 42343e3b517fe522d8fc6fff20a12de32e547622 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 27 Jan 2023 14:18:08 +0000 Subject: [PATCH 18/71] Switch Dependabot to monthly --- .github/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 98e44ee..c0e00fb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,8 +3,8 @@ updates: - package-ecosystem: "cargo" directory: "/" schedule: - interval: "weekly" + interval: "monthly" - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "monthly" From 3489dc54cc465659f47a4d7ac0dcb671317d82d4 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 27 Jan 2023 14:22:07 +0000 Subject: [PATCH 19/71] Switch GitHub Action runner image back to `ubuntu-latest` Since `ubuntu-latest` and `ubuntu-22.04` are now equivalent, since their rollout of the change in default image version has now completed. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c813a82..ba4fd0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ env: jobs: lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 @@ -30,7 +30,7 @@ jobs: run: cargo fmt -- --check unit-test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 @@ -42,7 +42,7 @@ jobs: run: cargo test --locked integration-test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 From eb98a862afd4c0eb83377603c307389e18194e4a Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 27 Jan 2023 16:30:09 +0000 Subject: [PATCH 20/71] Remove now-unused test fixture --- test-fixtures/function_python_version_too_old/main.py | 5 ----- test-fixtures/function_python_version_too_old/project.toml | 2 -- .../function_python_version_too_old/requirements.txt | 1 - test-fixtures/function_python_version_too_old/runtime.txt | 1 - 4 files changed, 9 deletions(-) delete mode 100644 test-fixtures/function_python_version_too_old/main.py delete mode 100644 test-fixtures/function_python_version_too_old/project.toml delete mode 100644 test-fixtures/function_python_version_too_old/requirements.txt delete mode 100644 test-fixtures/function_python_version_too_old/runtime.txt diff --git a/test-fixtures/function_python_version_too_old/main.py b/test-fixtures/function_python_version_too_old/main.py deleted file mode 100644 index 80920de..0000000 --- a/test-fixtures/function_python_version_too_old/main.py +++ /dev/null @@ -1,5 +0,0 @@ -from salesforce_functions import Context, InvocationEvent - - -async def function(_event: InvocationEvent[None], _context: Context) -> None: - return None diff --git a/test-fixtures/function_python_version_too_old/project.toml b/test-fixtures/function_python_version_too_old/project.toml deleted file mode 100644 index ef6d5f8..0000000 --- a/test-fixtures/function_python_version_too_old/project.toml +++ /dev/null @@ -1,2 +0,0 @@ -[com.salesforce] -type = "function" diff --git a/test-fixtures/function_python_version_too_old/requirements.txt b/test-fixtures/function_python_version_too_old/requirements.txt deleted file mode 100644 index ced5be3..0000000 --- a/test-fixtures/function_python_version_too_old/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -salesforce-functions diff --git a/test-fixtures/function_python_version_too_old/runtime.txt b/test-fixtures/function_python_version_too_old/runtime.txt deleted file mode 100644 index c9cbcea..0000000 --- a/test-fixtures/function_python_version_too_old/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.9.16 From 47d06d509fb112c008195f5f53fd6b24d34c70a5 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Sat, 28 Jan 2023 14:39:17 +0000 Subject: [PATCH 21/71] Pass detect for non-functions too --- src/main.rs | 11 ----------- tests/integration.rs | 13 ++++--------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8b0884a..7266558 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,17 +38,6 @@ impl Buildpack for PythonBuildpack { type Error = BuildpackError; fn detect(&self, context: DetectContext) -> libcnb::Result { - // For the functions alpha/beta, we need to release the CNB into the main builder image, - // however only want to make it available for functions for now, since the CNB is still - // experimental and not feature-complete for non-function use-cases. - // TODO: Remove this once the buildpack is ready for non-functions use. - if !functions::is_function_project(&context.app_dir) - .map_err(BuildpackError::ProjectDescriptor)? - { - log_info("A project.toml file containing a suitable Salesforce Function configuration was not found."); - return DetectResultBuilder::fail().build(); - } - // In the future we will add support for requiring this buildpack through the build plan, // but we first need a better understanding of real-world use-cases, so that we can work // out how best to support them without sacrificing existing error handling UX (such as diff --git a/tests/integration.rs b/tests/integration.rs index dcd1007..d6bac88 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -10,20 +10,15 @@ use std::time::Duration; const TEST_PORT: u16 = 12345; -// For now, these integration tests only cover functions, since: -// - that's what needs to ship first -// - the buildpack's detect by design rejects anything but a function, so for now -// all tests here need to actually be a function to get past detect - #[test] #[ignore = "integration test"] -fn detect_rejects_non_functions() { +fn detect_rejects_non_python_projects() { TestRunner::default().build( - BuildConfig::new("heroku/builder:22", "test-fixtures/default") + BuildConfig::new("heroku/builder:22", "test-fixtures/empty") .expected_pack_result(PackResult::Failure), |context| { - // We can't test the detect failure reason, since by default pack CLI only shows output for - // non-zero, non-100 exit codes, and `libcnb-test` support enabling pack build's verbose mode: + // We can't test the detect failure reason, since by default pack CLI only shows output for non-zero, + // non-100 exit codes, and `libcnb-test` does not support enabling pack build's verbose mode: // https://github.com/heroku/libcnb.rs/issues/383 assert_contains!( context.pack_stdout, From 76c282290507acefd47a93e4f5b69cc08103aa16 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 31 Jan 2023 18:34:38 +0000 Subject: [PATCH 22/71] Update to pip 23.0, setuptools 67.0.0, wheel 0.38.4 --- src/layers/python.rs | 6 +++--- tests/integration.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/layers/python.rs b/src/layers/python.rs index a3293bd..eb6eb79 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -15,9 +15,9 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{fs, io}; -const PIP_VERSION: &str = "22.3.1"; -const SETUPTOOLS_VERSION: &str = "65.6.3"; -const WHEEL_VERSION: &str = "0.38.3"; +const PIP_VERSION: &str = "23.0"; +const SETUPTOOLS_VERSION: &str = "67.0.0"; +const WHEEL_VERSION: &str = "0.38.4"; pub(crate) struct PythonLayer<'a> { pub env: &'a Env, diff --git a/tests/integration.rs b/tests/integration.rs index d6bac88..5b652ac 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -50,7 +50,7 @@ fn function_template() { Python installation successful [Installing Pip] - Installing pip 22.3.1, setuptools 65.6.3 and wheel 0.38.3 + Installing pip 23.0, setuptools 67.0.0 and wheel 0.38.4 Installation completed [Installing dependencies using Pip] @@ -122,7 +122,7 @@ fn function_repeat_build() { Re-using cached Python 3.11.1 [Installing Pip] - Re-using cached pip 22.3.1, setuptools 65.6.3 and wheel 0.38.3 + Re-using cached pip 23.0, setuptools 67.0.0 and wheel 0.38.4 [Installing dependencies using Pip] Re-using cached pip-cache From da42891e8f7c42f045eaf39298744cad3372b6e5 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 1 Feb 2023 07:14:07 +0000 Subject: [PATCH 23/71] Clean up functions handling --- src/functions.rs | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/functions.rs b/src/functions.rs index 0bf0c5a..ba4731d 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -8,9 +8,6 @@ use std::process::{Command, Output}; pub const FUNCTION_RUNTIME_PROGRAM_NAME: &str = "sf-functions-python"; -// TODO: Decide default number of workers. -const SERVE_SUBCOMMAND: &str = "serve --host 0.0.0.0 --port \"${PORT:-8080}\" --workers 4 ."; - /// Detect whether the specified project directory is that of a Salesforce Function. /// /// Returns `Ok(true)` if the specified project directory contains a `project.toml` file with a @@ -27,11 +24,6 @@ pub(crate) fn is_function_project(app_dir: &Path) -> Result Result<(), CheckFunctionError> { // Not using `utils::run_command` since we want to capture output and only // display it if the check command fails. @@ -63,7 +55,20 @@ pub(crate) fn launch_config() -> Launch { ProcessBuilder::new(process_type!("web"), "bash") .args([ "-c", - &format!("exec {FUNCTION_RUNTIME_PROGRAM_NAME} {SERVE_SUBCOMMAND}"), + &[ + "exec", + FUNCTION_RUNTIME_PROGRAM_NAME, + "serve", + "--host", + "0.0.0.0", + "--port", + "\"${PORT:-8080}\"", + // TODO: Determine optimal number of workers. + "--workers", + "4", + ".", + ] + .join(" "), ]) .default(true) .direct(true) @@ -99,7 +104,7 @@ mod tests { } #[test] - fn is_function_project_function_project_toml() { + fn is_function_project_valid_function_project_toml() { let app_dir = Path::new("test-fixtures/function_template"); assert!(is_function_project(app_dir).unwrap()); From 1ac2ec631fe9d99d6a7e889b000a68a69c3aa304 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 1 Feb 2023 14:47:09 +0000 Subject: [PATCH 24/71] More unit tests, rustdocs and comments --- src/errors.rs | 1 - src/functions.rs | 21 ++--- src/layers/pip_dependencies.rs | 1 + src/layers/python.rs | 1 + src/main.rs | 3 + src/package_manager.rs | 36 +++++++- src/python_version.rs | 88 ++++++++++--------- src/runtime_txt.rs | 50 ++++++++++- src/utils.rs | 74 +++++++++++++++- test-fixtures/function_python_3.10/main.py | 20 ----- .../function_python_3.10/project.toml | 3 - .../function_python_3.10/requirements.txt | 1 - .../function_python_version_invalid/main.py | 5 -- .../project.toml | 2 - .../requirements.txt | 1 - .../main.py | 5 -- .../project.toml | 2 - .../requirements.txt | 1 - .../runtime.txt | 0 .../requirements.txt | 0 .../runtime.txt | 0 .../requirements.txt | 0 .../runtime.txt | 0 tests/integration.rs | 69 +-------------- 24 files changed, 214 insertions(+), 170 deletions(-) delete mode 100644 test-fixtures/function_python_3.10/main.py delete mode 100644 test-fixtures/function_python_3.10/project.toml delete mode 100644 test-fixtures/function_python_3.10/requirements.txt delete mode 100644 test-fixtures/function_python_version_invalid/main.py delete mode 100644 test-fixtures/function_python_version_invalid/project.toml delete mode 100644 test-fixtures/function_python_version_invalid/requirements.txt delete mode 100644 test-fixtures/function_python_version_unavailable/main.py delete mode 100644 test-fixtures/function_python_version_unavailable/project.toml delete mode 100644 test-fixtures/function_python_version_unavailable/requirements.txt rename test-fixtures/{function_python_3.10 => runtime_txt_python_3.10}/runtime.txt (100%) create mode 100644 test-fixtures/runtime_txt_python_version_invalid/requirements.txt rename test-fixtures/{function_python_version_invalid => runtime_txt_python_version_invalid}/runtime.txt (100%) create mode 100644 test-fixtures/runtime_txt_python_version_unavailable/requirements.txt rename test-fixtures/{function_python_version_unavailable => runtime_txt_python_version_unavailable}/runtime.txt (100%) diff --git a/src/errors.rs b/src/errors.rs index 83089f9..5952fd4 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -63,7 +63,6 @@ fn on_project_descriptor_error(project_descriptor_error: ReadProjectDescriptorEr "reading the (optional) project.toml file", &io_error, ), - // TODO: Add more detail here, like example file contents for functions? ReadProjectDescriptorError::Parse(toml_error) => log_error( "Invalid project.toml", formatdoc! {" diff --git a/src/functions.rs b/src/functions.rs index ba4731d..8ccc14d 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -6,7 +6,8 @@ use std::io; use std::path::Path; use std::process::{Command, Output}; -pub const FUNCTION_RUNTIME_PROGRAM_NAME: &str = "sf-functions-python"; +/// The program/script name of the Python Functions runtime's CLI. +pub(crate) const FUNCTION_RUNTIME_PROGRAM_NAME: &str = "sf-functions-python"; /// Detect whether the specified project directory is that of a Salesforce Function. /// @@ -91,31 +92,25 @@ mod tests { #[test] fn is_function_project_no_project_toml() { - let app_dir = Path::new("test-fixtures/empty"); - - assert!(!is_function_project(app_dir).unwrap()); + assert!(!is_function_project(Path::new("test-fixtures/empty")).unwrap()); } #[test] fn is_function_project_non_salesforce_project_toml() { - let app_dir = Path::new("test-fixtures/project_toml_non_salesforce"); - - assert!(!is_function_project(app_dir).unwrap()); + assert!( + !is_function_project(Path::new("test-fixtures/project_toml_non_salesforce")).unwrap() + ); } #[test] fn is_function_project_valid_function_project_toml() { - let app_dir = Path::new("test-fixtures/function_template"); - - assert!(is_function_project(app_dir).unwrap()); + assert!(is_function_project(Path::new("test-fixtures/function_template")).unwrap()); } #[test] fn is_function_project_invalid_project_toml() { - let app_dir = Path::new("test-fixtures/project_toml_invalid"); - assert!(matches!( - is_function_project(app_dir).unwrap_err(), + is_function_project(Path::new("test-fixtures/project_toml_invalid")).unwrap_err(), ReadProjectDescriptorError::Parse(_) )); } diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 85bf3d4..20ae75f 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -88,6 +88,7 @@ impl Layer for PipDependenciesLayer<'_> { } } +/// Errors that can occur when installing the project's dependencies into a layer using Pip. #[derive(Debug)] pub(crate) enum PipDependenciesLayerError { CreateSrcDirIo(io::Error), diff --git a/src/layers/python.rs b/src/layers/python.rs index eb6eb79..d27dbef 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -267,6 +267,7 @@ fn generate_layer_metadata( } } +/// Errors that can occur when installing Python and required packaging tools into a layer. #[derive(Debug)] pub(crate) enum PythonLayerError { BootstrapPipCommand(CommandError), diff --git a/src/main.rs b/src/main.rs index 7266558..6ed06ce 100644 --- a/src/main.rs +++ b/src/main.rs @@ -67,6 +67,7 @@ impl Buildpack for PythonBuildpack { // env vars will still be excluded, due to the use of `clear-env` in `buildpack.toml`. let mut env = Env::from_current(); + // Create the layer containing the Python runtime and required packaging tools. let python_layer = context.handle_layer( layer_name!("python"), PythonLayer { @@ -76,6 +77,8 @@ impl Buildpack for PythonBuildpack { )?; env = python_layer.env.apply(Scope::Build, &env); + // Create the layers for the application dependencies and package manager cache. + // In the future support will be added for package managers other than pip. let dependencies_layer_env = match package_manager { PackageManager::Pip => { log_header("Installing dependencies using Pip"); diff --git a/src/package_manager.rs b/src/package_manager.rs index 50cf745..8d04596 100644 --- a/src/package_manager.rs +++ b/src/package_manager.rs @@ -1,14 +1,20 @@ use std::io; use std::path::Path; +/// A ordered mapping of project filenames to their associated package manager. +/// Earlier entries will take precedence if a project matches multiple package managers. +pub(crate) const PACKAGE_MANAGER_FILE_MAPPING: [(&str, PackageManager); 1] = + [("requirements.txt", PackageManager::Pip)]; + +/// Python package managers supported by the buildpack. +#[derive(Debug)] pub(crate) enum PackageManager { Pip, } -const PACKAGE_MANAGER_FILE_MAPPING: [(&str, PackageManager); 1] = - [("requirements.txt", PackageManager::Pip)]; - -// TODO: Unit test +/// Determine the Python package manager to use for a project, or return an error if no supported +/// package manager files are found. If a project contains the files for multiple package managers, +/// then the earliest entry in `PACKAGE_MANAGER_FILE_MAPPING` takes precedence. pub(crate) fn determine_package_manager( app_dir: &Path, ) -> Result { @@ -26,8 +32,30 @@ pub(crate) fn determine_package_manager( Err(DeterminePackageManagerError::NoneFound) } +/// Errors that can occur when determining which Python package manager to use for a project. #[derive(Debug)] pub(crate) enum DeterminePackageManagerError { Io(io::Error), NoneFound, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn determine_package_manager_requirements_txt() { + assert!(matches!( + determine_package_manager(Path::new("test-fixtures/default")).unwrap(), + PackageManager::Pip + )); + } + + #[test] + fn determine_package_manager_none() { + assert!(matches!( + determine_package_manager(Path::new("test-fixtures/empty")).unwrap_err(), + DeterminePackageManagerError::NoneFound + )); + } +} diff --git a/src/python_version.rs b/src/python_version.rs index ac91e62..81ae3d5 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -4,12 +4,14 @@ use libherokubuildpack::log::log_info; use std::fmt::{self, Display}; use std::path::Path; +/// The Python version that will be installed if the project does not specify an explicit version. pub(crate) const DEFAULT_PYTHON_VERSION: PythonVersion = PythonVersion { major: 3, minor: 11, patch: 1, }; +/// Representation of a specific Python `X.Y.Z` version. #[derive(Clone, Debug, PartialEq)] pub(crate) struct PythonVersion { pub major: u16, @@ -33,35 +35,9 @@ impl Display for PythonVersion { } } -// string -> requested python version -> exact python version -> python runtime (incl URL etc) - -// resolving python version: -// failure modes: Nonsensical, unknown to buildpack, known but not supported, known and used to be supported but no longer -// Does this occur inside each `get_version` / creation of `PythonVersion`? -// But then each error type needs 3-4 additional enum variants -// Depends on whether we want different error messages for each? -// Though could still vary error message by using `PythonVersion.source` etc - -// Questions: -// How should Python version detection precedence work? - -// TODO: Add tests for `get_version`? Or test caller? Or integration test? -// -// Possible tests: -// - some IO error -> Err(RuntimeTxtError::Io) -// - file present but invalid -> Err(RuntimeTxtError::Parse) -// - file present and valid -> Ok(Some(python_version)) -// - file not present -> Ok(None) - -// warnings: -// EOL major version, non-latest minor version, deprecated version specifier? -// output warnings as found during build, or at end of the build log? -// does EOL warnings use requested Python version or resolved version? I suppose resolved since needs EOL date etc, plus range version might still be outdated? - -// logging: -// Do we log for version specifier files not found? Or only when found? -// where do we log? In get_version, determine_python_version, or in the caller and have to store the version source in `PythonVersion`? - +/// Determine the Python version that should be installed for the project. +/// +/// If no known version specifier file is found a default Python version will be used. pub(crate) fn determine_python_version( app_dir: &Path, ) -> Result { @@ -84,21 +60,47 @@ pub(crate) fn determine_python_version( Ok(DEFAULT_PYTHON_VERSION) } -pub(crate) fn _determine_python_version2( - app_dir: &Path, -) -> Result { - runtime_txt::read_version(app_dir) - .map_err(PythonVersionError::RuntimeTxt) - .transpose() - .or_else(|| { - runtime_txt::read_version(app_dir) - .map_err(PythonVersionError::RuntimeTxt) - .transpose() - }) - .unwrap_or(Ok(DEFAULT_PYTHON_VERSION)) -} - +/// Errors that can occur when determining which Python package manager to use for a project. #[derive(Debug)] pub(crate) enum PythonVersionError { RuntimeTxt(ReadRuntimeTxtError), } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn determine_python_version_runtime_txt_valid() { + assert_eq!( + determine_python_version(Path::new("test-fixtures/runtime_txt_python_3.10")).unwrap(), + PythonVersion::new(3, 10, 9) + ); + assert_eq!( + determine_python_version(Path::new( + "test-fixtures/runtime_txt_python_version_unavailable" + )) + .unwrap(), + PythonVersion::new(999, 999, 999) + ); + } + + #[test] + fn determine_python_version_runtime_txt_error() { + assert!(matches!( + determine_python_version(Path::new( + "test-fixtures/runtime_txt_python_version_invalid" + )) + .unwrap_err(), + PythonVersionError::RuntimeTxt(ReadRuntimeTxtError::Parse(_)) + )); + } + + #[test] + fn determine_python_version_none_specified() { + assert_eq!( + determine_python_version(Path::new("test-fixtures/empty")).unwrap(), + DEFAULT_PYTHON_VERSION + ); + } +} diff --git a/src/runtime_txt.rs b/src/runtime_txt.rs index e19d403..b70c9a5 100644 --- a/src/runtime_txt.rs +++ b/src/runtime_txt.rs @@ -3,7 +3,11 @@ use crate::utils; use std::io; use std::path::Path; -/// TODO +/// Retrieve a parsed Python version from a `runtime.txt` file if it exists in the +/// specified project directory. +/// +/// Returns `Ok(None)` if the file does not exist, but returns the error for all other +/// forms of IO or parsing errors. pub(crate) fn read_version(app_dir: &Path) -> Result, ReadRuntimeTxtError> { let runtime_txt_path = app_dir.join("runtime.txt"); @@ -45,12 +49,14 @@ fn parse(contents: &str) -> Result { } } +/// Errors that can occur when reading and parsing a `runtime.txt` file. #[derive(Debug)] pub(crate) enum ReadRuntimeTxtError { Io(io::Error), Parse(ParseRuntimeTxtError), } +/// Errors that can occur when parsing the contents of a `runtime.txt` file. #[derive(Debug, PartialEq)] pub(crate) struct ParseRuntimeTxtError { pub cleaned_contents: String, @@ -185,4 +191,46 @@ mod tests { }) ); } + + #[test] + fn read_version_valid_runtime_txt() { + assert_eq!( + read_version(Path::new("test-fixtures/runtime_txt_python_3.10")).unwrap(), + Some(PythonVersion::new(3, 10, 9)) + ); + assert_eq!( + read_version(Path::new( + "test-fixtures/runtime_txt_python_version_unavailable" + )) + .unwrap(), + Some(PythonVersion::new(999, 999, 999)) + ); + } + + #[test] + fn read_version_runtime_txt_not_present() { + assert_eq!( + read_version(Path::new("test-fixtures/empty")).unwrap(), + None + ); + } + + #[test] + fn read_version_io_error() { + assert!(matches!( + read_version(Path::new("test-fixtures/empty/.gitkeep")).unwrap_err(), + ReadRuntimeTxtError::Io(_) + )); + } + + #[test] + fn read_version_parse_error() { + assert!(matches!( + read_version(Path::new( + "test-fixtures/runtime_txt_python_version_invalid" + )) + .unwrap_err(), + ReadRuntimeTxtError::Parse(_) + )); + } } diff --git a/src/utils.rs b/src/utils.rs index fc0836e..9e99ac7 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,7 +4,13 @@ use std::process::{Command, ExitStatus}; use std::{fs, io}; use tar::Archive; -// TODO: Unit test that all files from PACKAGE_MANAGER_FILES are in here. +/// Filenames that if found in a project mean it should be treated as a Python project, +/// and so pass this buildpack's detection phase. +/// +/// This list is deliberately larger than just the list of supported package manager files, +/// so that Python projects that are missing some of the required files still pass detection, +/// allowing us to show a more detailed error message during the build phase than is possible +/// during detect. const KNOWN_PYTHON_PROJECT_FILES: [&str; 9] = [ ".python-version", "main.py", @@ -17,7 +23,8 @@ const KNOWN_PYTHON_PROJECT_FILES: [&str; 9] = [ "setup.py", ]; -// TODO: Unit test +/// Returns whether the specified project directory is that of a Python project, and so +/// should pass buildpack detection. pub(crate) fn is_python_project(app_dir: &Path) -> io::Result { // Until `Iterator::try_find` is stabilised, this is cleaner as a for loop. for filename in KNOWN_PYTHON_PROJECT_FILES { @@ -29,7 +36,8 @@ pub(crate) fn is_python_project(app_dir: &Path) -> io::Result { Ok(false) } -// TODO: Unit test +/// Read the contents of the provided filepath if the file exists, gracefully handling +/// the file not being present, but still returning any other form of IO error. pub(crate) fn read_optional_file(path: &Path) -> io::Result> { fs::read_to_string(path) .map(Some) @@ -39,6 +47,7 @@ pub(crate) fn read_optional_file(path: &Path) -> io::Result> { }) } +/// Download a gzipped tar file and unpack it to the specified directory. pub(crate) fn download_and_unpack_gzipped_archive( uri: &str, destination: &Path, @@ -54,12 +63,15 @@ pub(crate) fn download_and_unpack_gzipped_archive( .map_err(DownloadUnpackArchiveError::Io) } +/// Errors that can occur when downloading and unpacking an archive using `download_and_unpack_gzipped_archive`. #[derive(Debug)] pub(crate) enum DownloadUnpackArchiveError { Io(io::Error), Request(ureq::Error), } +/// A helper for running an external process using [`Command`], that checks the exit +/// status of the process was non-zero. pub(crate) fn run_command(command: &mut Command) -> Result<(), CommandError> { command .status() @@ -73,8 +85,64 @@ pub(crate) fn run_command(command: &mut Command) -> Result<(), CommandError> { }) } +/// Errors that can occur when running an external process using `run_command`. #[derive(Debug)] pub(crate) enum CommandError { Io(io::Error), NonZeroExitStatus(ExitStatus), } + +#[cfg(test)] +mod tests { + use super::*; + use crate::package_manager::PACKAGE_MANAGER_FILE_MAPPING; + + #[test] + fn is_python_project_valid_project() { + assert!(is_python_project(Path::new("test-fixtures/default")).unwrap()); + } + + #[test] + fn is_python_project_empty() { + assert!(!is_python_project(Path::new("test-fixtures/empty")).unwrap()); + } + + #[test] + fn is_python_project_io_error() { + assert!(is_python_project(Path::new("test-fixtures/empty/.gitkeep")).is_err()); + } + + #[test] + fn read_optional_file_valid_file() { + assert_eq!( + read_optional_file(Path::new( + "test-fixtures/runtime_txt_python_3.10/runtime.txt" + )) + .unwrap(), + Some("python-3.10.9\n".to_string()) + ); + } + + #[test] + fn read_optional_file_missing_file() { + assert_eq!( + read_optional_file(Path::new( + "test-fixtures/non-existent-dir/non-existent-file" + )) + .unwrap(), + None + ); + } + + #[test] + fn read_optional_file_io_error() { + assert!(read_optional_file(Path::new("test-fixtures/")).is_err()); + } + + #[test] + fn known_python_project_files_contains_all_package_manager_files() { + assert!(PACKAGE_MANAGER_FILE_MAPPING + .iter() + .all(|(filename, _)| { KNOWN_PYTHON_PROJECT_FILES.contains(filename) })); + } +} diff --git a/test-fixtures/function_python_3.10/main.py b/test-fixtures/function_python_3.10/main.py deleted file mode 100644 index 4c05d20..0000000 --- a/test-fixtures/function_python_3.10/main.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Any - -from salesforce_functions import Context, InvocationEvent, get_logger - -# The type of the data payload sent with the invocation event. -# Change this to a more specific type matching the expected payload for -# improved IDE auto-completion and linting coverage. For example: -# `EventPayloadType = dict[str, Any]` -EventPayloadType = Any - -logger = get_logger() - - -async def function(event: InvocationEvent[EventPayloadType], context: Context): - """Describe the function here.""" - - result = await context.org.data_api.query("SELECT Id, Name FROM Account") - logger.info(f"Function successfully queried {result.total_size} account records!") - - return result.records diff --git a/test-fixtures/function_python_3.10/project.toml b/test-fixtures/function_python_3.10/project.toml deleted file mode 100644 index 332e751..0000000 --- a/test-fixtures/function_python_3.10/project.toml +++ /dev/null @@ -1,3 +0,0 @@ -[com.salesforce] -type = "function" -salesforce-api-version = "56.0" diff --git a/test-fixtures/function_python_3.10/requirements.txt b/test-fixtures/function_python_3.10/requirements.txt deleted file mode 100644 index ced5be3..0000000 --- a/test-fixtures/function_python_3.10/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -salesforce-functions diff --git a/test-fixtures/function_python_version_invalid/main.py b/test-fixtures/function_python_version_invalid/main.py deleted file mode 100644 index 80920de..0000000 --- a/test-fixtures/function_python_version_invalid/main.py +++ /dev/null @@ -1,5 +0,0 @@ -from salesforce_functions import Context, InvocationEvent - - -async def function(_event: InvocationEvent[None], _context: Context) -> None: - return None diff --git a/test-fixtures/function_python_version_invalid/project.toml b/test-fixtures/function_python_version_invalid/project.toml deleted file mode 100644 index ef6d5f8..0000000 --- a/test-fixtures/function_python_version_invalid/project.toml +++ /dev/null @@ -1,2 +0,0 @@ -[com.salesforce] -type = "function" diff --git a/test-fixtures/function_python_version_invalid/requirements.txt b/test-fixtures/function_python_version_invalid/requirements.txt deleted file mode 100644 index ced5be3..0000000 --- a/test-fixtures/function_python_version_invalid/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -salesforce-functions diff --git a/test-fixtures/function_python_version_unavailable/main.py b/test-fixtures/function_python_version_unavailable/main.py deleted file mode 100644 index 80920de..0000000 --- a/test-fixtures/function_python_version_unavailable/main.py +++ /dev/null @@ -1,5 +0,0 @@ -from salesforce_functions import Context, InvocationEvent - - -async def function(_event: InvocationEvent[None], _context: Context) -> None: - return None diff --git a/test-fixtures/function_python_version_unavailable/project.toml b/test-fixtures/function_python_version_unavailable/project.toml deleted file mode 100644 index ef6d5f8..0000000 --- a/test-fixtures/function_python_version_unavailable/project.toml +++ /dev/null @@ -1,2 +0,0 @@ -[com.salesforce] -type = "function" diff --git a/test-fixtures/function_python_version_unavailable/requirements.txt b/test-fixtures/function_python_version_unavailable/requirements.txt deleted file mode 100644 index ced5be3..0000000 --- a/test-fixtures/function_python_version_unavailable/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -salesforce-functions diff --git a/test-fixtures/function_python_3.10/runtime.txt b/test-fixtures/runtime_txt_python_3.10/runtime.txt similarity index 100% rename from test-fixtures/function_python_3.10/runtime.txt rename to test-fixtures/runtime_txt_python_3.10/runtime.txt diff --git a/test-fixtures/runtime_txt_python_version_invalid/requirements.txt b/test-fixtures/runtime_txt_python_version_invalid/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/test-fixtures/function_python_version_invalid/runtime.txt b/test-fixtures/runtime_txt_python_version_invalid/runtime.txt similarity index 100% rename from test-fixtures/function_python_version_invalid/runtime.txt rename to test-fixtures/runtime_txt_python_version_invalid/runtime.txt diff --git a/test-fixtures/runtime_txt_python_version_unavailable/requirements.txt b/test-fixtures/runtime_txt_python_version_unavailable/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/test-fixtures/function_python_version_unavailable/runtime.txt b/test-fixtures/runtime_txt_python_version_unavailable/runtime.txt similarity index 100% rename from test-fixtures/function_python_version_unavailable/runtime.txt rename to test-fixtures/runtime_txt_python_version_unavailable/runtime.txt diff --git a/tests/integration.rs b/tests/integration.rs index 5b652ac..92508fa 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -137,72 +137,11 @@ fn function_repeat_build() { #[test] #[ignore = "integration test"] -fn function_python_3_10() { - TestRunner::default().build( - BuildConfig::new("heroku/builder:22", "test-fixtures/function_python_3.10"), - |context| { - assert_contains!( - context.pack_stdout, - indoc! {" - [Determining Python version] - Using Python version 3.10.9 specified in runtime.txt - - [Installing Python] - Downloading Python 3.10.9 - Python installation successful - "} - ); - - assert_contains!( - context.pack_stdout, - indoc! {" - Pip install completed - - [Validating Salesforce Function] - Function passed validation. - "} - ); - - context.start_container( - ContainerConfig::new() - .env("PORT", TEST_PORT.to_string()) - .expose_port(TEST_PORT), - |container| { - let address_on_host = container.address_for_port(TEST_PORT).unwrap(); - let url = format!("http://{}:{}", address_on_host.ip(), address_on_host.port()); - - // Retries needed since the server takes a moment to start up. - let mut attempts_remaining = 5; - let response = loop { - let response = ureq::post(&url).set("x-health-check", "true").call(); - if response.is_ok() || attempts_remaining == 0 { - break response; - } - attempts_remaining -= 1; - thread::sleep(Duration::from_secs(1)); - }; - - let server_log_output = container.logs_now(); - assert_contains!( - server_log_output.stderr, - &format!("Uvicorn running on http://0.0.0.0:{TEST_PORT}") - ); - - let body = response.unwrap().into_string().unwrap(); - assert_eq!(body, r#""OK""#); - }, - ); - }, - ); -} - -#[test] -#[ignore = "integration test"] -fn function_python_version_unavailable() { +fn runtime_txt_python_version_unavailable() { TestRunner::default().build( BuildConfig::new( "heroku/builder:22", - "test-fixtures/function_python_version_unavailable", + "test-fixtures/runtime_txt_python_version_unavailable", ) .expected_pack_result(PackResult::Failure), |context| { @@ -225,11 +164,11 @@ fn function_python_version_unavailable() { #[test] #[ignore = "integration test"] -fn function_python_version_invalid() { +fn runtime_txt_python_version_invalid() { TestRunner::default().build( BuildConfig::new( "heroku/builder:22", - "test-fixtures/function_python_version_invalid", + "test-fixtures/runtime_txt_python_version_invalid", ) .expected_pack_result(PackResult::Failure), |context| { From 448921e0c35a5bdc3698db0f16b66e8b082c451a Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 1 Feb 2023 14:48:17 +0000 Subject: [PATCH 25/71] Switch buildpack ID back to `heroku/python` --- buildpack.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/buildpack.toml b/buildpack.toml index 6b5ecfa..110cbfb 100644 --- a/buildpack.toml +++ b/buildpack.toml @@ -1,9 +1,7 @@ api = "0.8" [buildpack] -# The buildpack ID here is temporary, for the Python functions alpha/beta. -# TODO: Change it back to `heroku/python` once the buildpack is ready for non-functions use. -id = "heroku/python-functions-experimental" +id = "heroku/python" version = "0.1.0" name = "Python" homepage = "https://github.com/heroku/buildpacks-python" From 92046bb122ecb6f3148177c292cb0bc15f45858f Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:49:27 +0000 Subject: [PATCH 26/71] Test `heroku/buildpacks:20` in CI too --- .github/workflows/ci.yml | 6 ++++++ tests/integration.rs | 35 ++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba4fd0d..c823524 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,12 @@ jobs: integration-test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + builder: ["heroku/buildpacks:20", "heroku/builder:22"] + env: + INTEGRATION_TEST_CNB_BUILDER: ${{ matrix.builder }} steps: - name: Checkout uses: actions/checkout@v3 diff --git a/tests/integration.rs b/tests/integration.rs index 92508fa..5e73188 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -3,18 +3,23 @@ #![warn(clippy::pedantic)] -use indoc::indoc; +use indoc::{formatdoc, indoc}; use libcnb_test::{assert_contains, BuildConfig, ContainerConfig, PackResult, TestRunner}; -use std::thread; use std::time::Duration; +use std::{env, thread}; +const DEFAULT_BUILDER: &str = "heroku/builder:22"; const TEST_PORT: u16 = 12345; +fn builder() -> String { + env::var("INTEGRATION_TEST_CNB_BUILDER").unwrap_or(DEFAULT_BUILDER.to_string()) +} + #[test] #[ignore = "integration test"] fn detect_rejects_non_python_projects() { TestRunner::default().build( - BuildConfig::new("heroku/builder:22", "test-fixtures/empty") + BuildConfig::new(builder(), "test-fixtures/empty") .expected_pack_result(PackResult::Failure), |context| { // We can't test the detect failure reason, since by default pack CLI only shows output for non-zero, @@ -32,7 +37,7 @@ fn detect_rejects_non_python_projects() { #[ignore = "integration test"] fn function_template() { TestRunner::default().build( - BuildConfig::new("heroku/builder:22", "test-fixtures/function_template"), + BuildConfig::new(builder(), "test-fixtures/function_template"), |context| { // Pip outputs git clone output to stderr for some reason, so stderr isn't empty. // TODO: Decide whether this is a bug in pip and/or if we should work around it. @@ -107,7 +112,7 @@ fn function_template() { #[ignore = "integration test"] fn function_repeat_build() { TestRunner::default().build( - BuildConfig::new("heroku/builder:22", "test-fixtures/function_template"), + BuildConfig::new(builder(), "test-fixtures/function_template"), |context| { let config = context.config.clone(); context.rebuild(config, |rebuild_context| { @@ -138,18 +143,26 @@ fn function_repeat_build() { #[test] #[ignore = "integration test"] fn runtime_txt_python_version_unavailable() { + let builder = builder(); + TestRunner::default().build( BuildConfig::new( - "heroku/builder:22", + &builder, "test-fixtures/runtime_txt_python_version_unavailable", ) .expected_pack_result(PackResult::Failure), |context| { + let expected_stack = match builder.as_str() { + "heroku/buildpacks:20" => "heroku-20", + "heroku/builder:22" => "heroku-22", + _ => unimplemented!("Unknown builder!"), + }; + assert_contains!( context.pack_stderr, - indoc! {" + &formatdoc! {" [Error: Requested Python version is not available] - The requested Python version (999.999.999) is not available for this stack (heroku-22). + The requested Python version (999.999.999) is not available for this stack ({expected_stack}). Please update the version in 'runtime.txt' to a supported Python version, or else remove the file to instead use the default version (currently Python 3.11.1). @@ -167,7 +180,7 @@ fn runtime_txt_python_version_unavailable() { fn runtime_txt_python_version_invalid() { TestRunner::default().build( BuildConfig::new( - "heroku/builder:22", + builder(), "test-fixtures/runtime_txt_python_version_invalid", ) .expected_pack_result(PackResult::Failure), @@ -203,7 +216,7 @@ fn runtime_txt_python_version_invalid() { fn function_missing_functions_package() { TestRunner::default().build( BuildConfig::new( - "heroku/builder:22", + builder(), "test-fixtures/function_missing_functions_package", ) .expected_pack_result(PackResult::Failure), @@ -231,7 +244,7 @@ fn function_missing_functions_package() { fn function_fails_self_check() { TestRunner::default().build( BuildConfig::new( - "heroku/builder:22", + builder(), "test-fixtures/function_fails_self_check", ) .expected_pack_result(PackResult::Failure), From 6c32581a32a0d88644e4e83a4a7439f8fe8df67d Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:54:23 +0000 Subject: [PATCH 27/71] Shorten CI job name So it fits in the sidebar --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c823524..00aca60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,9 +46,9 @@ jobs: strategy: fail-fast: false matrix: - builder: ["heroku/buildpacks:20", "heroku/builder:22"] + builder: ["builder:22", "buildpacks:20"] env: - INTEGRATION_TEST_CNB_BUILDER: ${{ matrix.builder }} + INTEGRATION_TEST_CNB_BUILDER: heroku/${{ matrix.builder }} steps: - name: Checkout uses: actions/checkout@v3 From d849faed57c511e0e6f70c3a611f97b9bd91a8ac Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Feb 2023 13:29:35 +0000 Subject: [PATCH 28/71] Update Cargo dependencies --- Cargo.lock | 140 ++++++++++++++++++++++++++------------ Cargo.toml | 4 +- src/project_descriptor.rs | 10 ++- 3 files changed, 102 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab7ed16..aad302b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,9 +87,9 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "camino" @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "cargo_metadata" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982a0cf6a99c350d7246035613882e376d58cebe571785abc5da4f648d53ac0a" +checksum = "08a1ec454bc3eead8719cb56e15dbbfecdbc14e4b3a3ae4936cc6e31f5fc0d07" dependencies = [ "camino", "cargo-platform", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -250,24 +250,24 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-macro" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", @@ -276,21 +276,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures-core", "futures-macro", @@ -376,9 +376,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" dependencies = [ "bytes", "futures-channel", @@ -439,9 +439,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.8" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2d6f23ffea9d7e76c53eee25dfb67bcd8fde7f1198b0855350698c9f07c780" +checksum = "6fe2b9d82064e8a0226fddb3547f37f28eaa46d0fc210e275d835f08cf3b76a7" [[package]] name = "instant" @@ -460,9 +460,9 @@ checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -483,7 +483,7 @@ dependencies = [ "libcnb-proc-macros", "serde", "thiserror", - "toml", + "toml 0.5.11", ] [[package]] @@ -496,7 +496,7 @@ dependencies = [ "libcnb-proc-macros", "serde", "thiserror", - "toml", + "toml 0.5.11", ] [[package]] @@ -507,7 +507,7 @@ checksum = "d9ed34a92d997ad9b0666ddbcc3995191e7642ee50ffa760497d2fb3bff7c5b5" dependencies = [ "cargo_metadata", "libcnb-data", - "toml", + "toml 0.5.11", "which", ] @@ -597,6 +597,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "nom8" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +dependencies = [ + "memchr", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -677,7 +686,7 @@ dependencies = [ "libherokubuildpack", "serde", "tar", - "toml", + "toml 0.7.1", "ureq", ] @@ -806,6 +815,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -947,15 +965,15 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.24.2" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", "bytes", @@ -1002,6 +1020,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "772c1426ab886e7362aedf4abc9c0d1348a979517efedfc25862944d10137af0" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90a238ee2e6ede22fb95350acc78e21dc40da00bb66c0334bde83de4ed89424e" +dependencies = [ + "indexmap", + "nom8", + "serde", + "serde_spanned", + "toml_datetime", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -1111,9 +1163,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1121,9 +1173,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -1136,9 +1188,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1146,9 +1198,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -1159,15 +1211,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index c6a38ad..9929756 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,12 +12,12 @@ publish = false # https://github.com/rust-lang/libz-sys/issues/93 # As such we have to use the next best alternate backend, which is `zlib`. flate2 = { version = "1", default-features = false, features = ["zlib"] } -indoc = "1" +indoc = "2" libcnb = "0.11" libherokubuildpack = { version = "0.11", default-features = false, features = ["log"] } serde = "1" tar = "0.4" -toml = "0.5" +toml = "0.7" ureq = { version = "2", default-features = false, features = ["tls"] } [dev-dependencies] diff --git a/src/project_descriptor.rs b/src/project_descriptor.rs index fa91dab..5a70b67 100644 --- a/src/project_descriptor.rs +++ b/src/project_descriptor.rs @@ -99,6 +99,7 @@ pub(crate) enum ReadProjectDescriptorError { #[cfg(test)] mod tests { use super::*; + use libcnb_test::assert_contains; #[test] fn deserialize_empty_descriptor() { @@ -180,10 +181,7 @@ mod tests { "#; let error = parse(toml_str).unwrap_err(); - assert_eq!( - error.to_string(), - "missing field `type` for key `com.salesforce` at line 2 column 13" - ); + assert_contains!(error.to_string(), "missing field `type`"); } #[test] @@ -194,9 +192,9 @@ mod tests { "#; let error = parse(toml_str).unwrap_err(); - assert_eq!( + assert_contains!( error.to_string(), - "unknown variant `some_unknown_type`, expected `function` for key `com.salesforce.type` at line 2 column 13" + "unknown variant `some_unknown_type`, expected `function`" ); } From f85f39d3dc58be5eacf372f80e9bdf895b906410 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Feb 2023 13:33:48 +0000 Subject: [PATCH 29/71] Update to setuptools 67.1.0 --- src/layers/python.rs | 2 +- tests/integration.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/layers/python.rs b/src/layers/python.rs index d27dbef..7a789d2 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -16,7 +16,7 @@ use std::process::Command; use std::{fs, io}; const PIP_VERSION: &str = "23.0"; -const SETUPTOOLS_VERSION: &str = "67.0.0"; +const SETUPTOOLS_VERSION: &str = "67.1.0"; const WHEEL_VERSION: &str = "0.38.4"; pub(crate) struct PythonLayer<'a> { diff --git a/tests/integration.rs b/tests/integration.rs index 5e73188..22151c5 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -55,7 +55,7 @@ fn function_template() { Python installation successful [Installing Pip] - Installing pip 23.0, setuptools 67.0.0 and wheel 0.38.4 + Installing pip 23.0, setuptools 67.1.0 and wheel 0.38.4 Installation completed [Installing dependencies using Pip] @@ -127,7 +127,7 @@ fn function_repeat_build() { Re-using cached Python 3.11.1 [Installing Pip] - Re-using cached pip 23.0, setuptools 67.0.0 and wheel 0.38.4 + Re-using cached pip 23.0, setuptools 67.1.0 and wheel 0.38.4 [Installing dependencies using Pip] Re-using cached pip-cache From bbe592dcead14804cac9e25c324f2bb54ef1bddb Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Feb 2023 13:50:48 +0000 Subject: [PATCH 30/71] Remove notes.md --- notes.md | 330 ------------------------------------------------------- 1 file changed, 330 deletions(-) delete mode 100644 notes.md diff --git a/notes.md b/notes.md deleted file mode 100644 index 236b1a1..0000000 --- a/notes.md +++ /dev/null @@ -1,330 +0,0 @@ -## Python package resolution -What are all of the ways packages end up on sys.path? And in what order? --> Script/current dir, PYTHONPATH, user site-packages (incl `.pth` and `usercustomize`), system site-packages (incl `.pth` and `sitecustomize`) -Can system site-packages location be overridden? --> Not really, since needs to be same as libs etc -Can user site-packages be overridden? --> Yes, using `PYTHONUSERBASE` -What deps do Pip, Poetry and pipenv have? Can the tools be installed outside of the env they are managing? --> Pip: None (it vendors). Managing deps outside of a venv is not supported (other than `--target` and perhaps `--prefix`). See: https://github.com/pypa/pip/issues/5472 --> Poetry: lots! However installing in a venv is both supported and recommended. --> Pipenv: lots! However installing in a venv is supported and kinda recommended. -Is pip needed when installing Poetry/pipenv? --> Poetry: Yes --> Pipenv: Yes -What deps will the Python invoker have? (ie can cause conflicts) Or fully vendored / in Rust? --> TBD -How do user installs work when there are conflicting dependencies? Can they be used inside a virtualenv? --> Seems to work well. And no, can't be used in a venv. See https://pip.pypa.io/en/stable/user_guide/#user-installs -What approaches do other CNBs use? --> GCP: Other buildpacks put their `requirements.txt` files into the build plan and then a single pip-install CNB installs it. Prior to that that they tried using `--prefix` and `--target` with `PYTHONPATH`. They cannot use `PYTHONUSERBASE` fully due to compatibility issues with their GAE image using system Python and having to use virtualenvs (which don't support user installs). --> Paketo: `PYTHONUSERBASE` to set install location during pip install of pip/deps, but then `PYTHONPATH` afterwards. They used to use `PYTHONUSERBASE` for both but changed in https://github.com/paketo-buildpacks/pip-install/pull/58 to "allow other buildpacks to use `PYTHONUSERBASE`" -- seems like they perhaps haven't realised about the `PYTHONPATH` shadowing stdlib issues? -What are the issues with using `--target` and `--prefix` that meant GCP stopped using them? --> https://github.com/GoogleCloudPlatform/buildpacks/commit/7768ebe4d5f300598b86328f607eeb70ab7b7131 --> https://github.com/GoogleCloudPlatform/buildpacks/commit/410b552aba55404bdb45acb638112feb271de01f --> https://github.com/GoogleCloudPlatform/buildpacks/commit/b93391cd653eef7336bc154466fa6d3de4ed337b --> https://github.com/pypa/pip/issues/8799 -So what are the alternatives for where to install packages? --> New venv (w/wo Pip / system site-packages) --> Arbitrary directory and point at it with `PYTHONPATH` --> Arbitrary directory used as user install location with `PYTHONUSERBASE` --> System site-packages in same layer as Python runtime --> Arbitrary directory and point at it with `.pth` file from user/system site-packages -Resources: -https://peps.python.org/pep-0370/ -https://docs.python.org/3.10/library/site.html -https://docs.python.org/3.10/install/index.html#alternate-installation -https://docs.python.org/3.10/using/cmdline.html#envvar-PYTHONNOUSERSITE -https://docs.python.org/3.10/using/cmdline.html#envvar-PYTHONPATH -https://docs.python.org/3.10/library/sys.html#sys.path -https://docs.python.org/3.10/library/sysconfig.html#installation-paths -https://docs.python.org/3.11/library/sys_path_init.html#sys-path-init - -## Installation locations -- Pip/setuptools/wheel: System site-packages in same layer as Python runtime -- Poetry/Pipenv (if applicable): Venv using `--symlinks --system-site-packages --without-pip` (using `--without-pip` saves ~8.5 MB and 1.6s on macOS). Must install using `python -m pip`. -- App dependencies: User site-packages -- Function invoker (if in Python): Arbitrary directory added to `PYTHONPATH` or make the user install - -## Installing dependencies with pip -- Do we support having no package manager being used? --> TBD -- Does a single layer handle all install types, or separate layer per package manager? --> Separate -- When to cache/invalidate site-packages? --> Invalidation needed to clean up removed packages (otherwise have to manually remove), and ensure unpinned deps are updated (if not using --upgrade) -- Should the pip cache also be cached? If so, when to invalidate that? --> Helps when cached site-packages invalidated, or if a previously used package added back -- Should we use `--upgrade`? --> Pros: Ensures unpinned deps stay up to date. Might mean we don't need to invalidate site-packages as often. --> Cons: Causes pip to still query PyPI even for `==` deps. --> Are people using `--upgrade` locally? -- What is the perf impact of caching site-packages vs pip cache? What about `--upgrade`? -- Options: `pip install --user --disable-pip-version-check --cache-dir --no-input` -- What about `requirements.txt` files with an include? -- Do we need to use `--exists-action`? -- No way to purge pip cache of items older than X (https://github.com/pypa/pip/issues/8355) - -curl -O https://raw.githubusercontent.com/mozilla/treeherder/master/requirements/common.txt -rm -rf venv /root/.cache/pip/ && python -m venv --symlink venv && time venv/bin/pip install --disable-pip-version-check -r common.txt -q --no-cache-dir - -## When does site-packages need invalidating? -- Python version changed (any, or just major?) --> Yes, perhaps any? -- Stack changed --> Yes -- Pip/setuptools/wheel version changed? --> Don't think so -- requirements.txt changes - -## Should we use `--upgrade` or `--upgrade --upgrade-strategy eager`? -- Pros: - - Means updated versions of unpinned packages (or unspecified transitive deps) are pulled in (without invalidating site-packages) - - Means pip logs show what changed (vs invalidating site-packages) -- Cons: - - Pip still queries PyPI for `==` pinned deps, slowing otherwise no-op runs. - - If an updated package drops a dep, then that dep isn't uninstalled (vs invalidating site-packages). - - Using `--upgrade --upgrade-strategy eager` results in errors for projects using hashes where a dependency has a transitive dep on setuptools (such as gunicorn) -- Other: - - Updates are pulled in immediately rather than after a delay - - Does `--upgrade` match what people are using locally? - - Does pip handle transitive dep updates any differently from empty site-packages? - -## Should we invalidate on root requirements.txt changes -- Yes! Have to otherwise package removals don't work. - -## What isn't handled when invalidating on root requirements.txt changes when not using `--upgrade`? -- Updated versions of unpinned packages (or unspecified transitive deps) are not pulled in -- Removals from transitive requirements.txt files (unless we scan for those too) -- Explicit package updates that drop a dep, in transitive requirements.txt files (unless we scan for those too) - -## What isn't handled when invalidating on root requirements.txt changes when using `--upgrade`? -- If an implicitly updated package drops a dep, then that dep isn't uninstalled (vs invalidating site-packages). -- Removals from transitive requirements.txt files (unless we scan for those too) -- Explicit package updates that drop a dep, in transitive requirements.txt files (unless we scan for those too) - -## How could we handle transitive requirements.txt files? -- Scan root requirements.txt for `-r ...` usages and check for changes to those too -- Output a warning if `-r ...` usages found and encourage users to stop using them or switch to eg Poetry -- Offer alternative locations to just the repo root, hoping people would use those instead of includes? (But doesn't cover all use-cases eg common deps) - -## Timings for treeherder's common.txt (Python 3.9, in venv, wheel installed, --disable-pip-version-check) -- Clean install, --no-cache-dir: 37.3s -- Clean install, cold cache: 37.8s -- Clean install, warm cache (all): 33.7s (however zstandard cached built wheel not used due to hashes) -- No-op repeat install, --no-cache, no upgrade: 0.61s -- No-op repeat install, warm cache, no upgrade: 0.61s -- No-op repeat install, --no-cache, --upgrade: 3.3s -- No-op repeat install, warm cache, --upgrade: 3.3s - -## Timings for treeherder's common.txt with hashes removed (Python 3.9, in venv, wheel installed, --disable-pip-version-check) -- Clean install, --no-cache-dir: 37.8s -- Clean install, cold cache: 37.8s -- Clean install, warm cache (all): 9.0s (without wheel installed this increases to 12.9s) -- Clean install, warm cache (3 MB wheel dir only): 12.8s -- Clean install, warm cache (72 MB http dir only): 33.9s - -## Timings for getting-started-guide's requirements.txt (Python 3.9, in venv, wheel installed, --disable-pip-version-check) -- Clean install, --no-cache-dir: 5.6s -- Clean install, cold cache: 5.7s -- Clean install, warm cache (all): 1.4s -- Clean install, warm cache (0.5 MB wheel dir only): 1.9s -- Clean install, warm cache (8.7 MB http dir only): 5.1s -- No-op repeat install, warm cache, no upgrade: 0.28s - -## Pip cache conclusions -- Wheel generation is where most of the time is spent (on a fast connection at least) -- If caching pip cache must have wheel installed or wheels won't be cached properly -- Could just cache wheels directory of pip cache since fraction of the size for most of the benefit. But wouldn't help slow connections. -- Invalidating site-packages increases install time from: 0.25s -> 1.4s (small project), 0.6s -> 9s (large project), 0.6s -> 34s (large project using hashes) -- Invalidating pip cache too increases install time from: 1.4s -> 5.7s (small project), 9s -> 38s (large project), 34s -> 38s (large project using hashes) -- Pip hashes really impact caching - should we output a warning? - -## Possible layer invalidation conditions -- Python version (either only when the major version changes, or also including minor version changes) -- Stack -- pip/setuptools/wheel version -- Poetry/pipenv version -- Input files from app (eg requirements.txt/Poetry.lock hash) -- Time since layer created -- Buildpack changes that aren't backwards compatible with old caches - -## Layer scenarios -- Initial install: `build()` -> `create()` -- Keeping cached layer: `build()` -> `existing_layer_strategy()` -- Recreating cached layer: `build()` -> `existing_layer_strategy()` -> `create()` -- Updating cached layer: `build()` -> `existing_layer_strategy()` -> `update()` - -## Logging -- What do users care about in the logs? - - If something went wrong, what it was, whether it was their fault or not, and how to resolve - - What is happening in general, so it doesn't seem like a black box - - How behaviour can be customised - - Why has behaviour changed since last build, particularly if something is now broken. -- When to use headings vs not? -- Should there always be a "doing thing" and "finished thing" message or just one or the other? -- How verbose should the logs be (particularly for output from subprocesses)? -- Should the verbosity be user controllable? Should we ask for a standard env var upstream? -- What should the logs show for using cache vs invalidating cache? - -## Errors -- Remove unwraps throughout and replace with new error enum variants -- How fine grained should the io::Error instances be? -- should layer errors be flattened into the top level buildpack error enum, or have their own error enums? -- Should the error `From` implementations live with the error enums (eg in the layer), or in errors.rs? -- What if anything should be covered by retries? Presumably only things involving network I/O? How well do pip's retries work? - -## Misc -- Utils for calling subprocesses -- Clear the env when calling subprocesses too (for most of them at least) -- What logic lives in the layer vs outside? -- Need to make Procfile mandatory given no default entrypoint. Although don't want to fail detect? -- Should set User Agent on outbound network requests -- Should we use https://docs.gunicorn.org/en/stable/settings.html#preload-app by default? - -## Unit tests -- What things do/don't need a unit test? -- Should the unit test cover lower down functions or their parents? - -## Integration tests -- Check Python static library works -- Check behaviour if buildpack run twice - -## Poetry -- Should it use a different layer name for the `site-packages` layer? - -## Improvements/decisions deferred to the future -- SHA256 checking of Python download. -- Decide whether to move pip/setuptools/wheel requirements to a requirements file so Dependabot can update them. - - However then means it's harder for us to list versions. - - Also, if integration tests include versions in log output and it's hardcoded, then Dependabot PRs will need manual updates anyway. -- Decide whether to use hashes for pip/setuptools/wheel requirements. - -## Python version support -- Do we support "3.*" / "*"", or just "3.x.*"? -- Do we support major version syntax in runtime.txt? -- Which of these other formats do we support? - - pyproject.toml's project.requires-python - - a new pyproject.toml table/property - - .python-version (with or w/o major version support?) - - tool.poetry.dependencies.python in pyproject.toml - - CNB project.toml file - -### pyproject.toml -[project] -requires-python = ">=3.8" -requires-python = "~=3.8" (means >=3.8, <4.0) -requires-python = "~=3.8.2" (means >=3.8.2, <3.9) -requires-python = "==3.8" (means ==3.8.0) -requires-python = "==3.8.*" -https://www.python.org/dev/peps/pep-0621/#requires-python -https://www.python.org/dev/peps/pep-0440/#version-specifiers -~=: Compatible release clause -==: Version matching clause -!=: Version exclusion clause -<=, >=: Inclusive ordered comparison clause -<, >: Exclusive ordered comparison clause -===: Arbitrary equality clause. - -### pyproject.toml -[tool.poetry.dependencies] -python = "^3.9" - -### .python-version -X.Y.Z -didn't used to support X.Y unless using a plugin, but now does: https://github.com/pyenv/pyenv#prefix-auto-resolution - -# pyc locations -- python stdlib -- pip/setuptools/wheel install in system site-packages -- app dependencies installed by pip in user site-packages -- poetry install in venv -- app dependencies installed by poetry in user site-packages -- app python files themselves in app dir - -# pyc alternatives -- timestamp (default) -- checked hash by disabling automatic compileall then running manually -- checked hash by setting SOURCE_DATE_EPOCH (only works via py_compile not by just running) -- unchecked hash by disabling automatic compileall then running manually -- delete the pyc files and let them be generated at build and/or app boot - -# pyc timings -- `python:3-slim`, native, `pip --version`, no pycs (creating timestamp): 0.628s -- `python:3-slim`, native, `pip --version`, no pycs (creating none): 0.571s -- `python:3-slim`, native, `pip --version`, existing timestamp: 0.151s -- `python:3-slim`, native, `pip --version`, existing checked: 0.161s -- `python:3-slim`, native, `pip --version`, existing unchecked: 0.152s -- `python:3-slim`, native, compileall pip dir, timestamp: 0.565s -- `python:3-slim`, native, compileall site-packages, checked: 0.637s -- `python:3-slim`, native, compileall site-packages, checked, workers=0: 0.199s -- `python:3-slim`, native, compileall python lib dir, timestamp: 1.277s -- `python:3-slim`, native, compileall python lib dir, checked: 1.275s -- `python:3-slim`, native, compileall python lib dir, checked, workers=0: 0.423s -- `python:3-slim`, qemu, `pip --version`, no pycs (creating timestamp): 5.475s -- `python:3-slim`, qemu, `pip --version`, no pycs (creating none): 5.357s -- `python:3-slim`, qemu, `pip --version`, existing timestamp: 1.360s -- `python:3-slim`, qemu, `pip --version`, existing checked: 1.386s -- `python:3-slim`, qemu, `pip --version`, existing unchecked: 1.356s -- `python:3-slim`, qemu, compileall pip dir, timestamp: 4.883s -- `python:3-slim`, qemu, compileall pip dir, checked: 4.869s -- `python:3-slim`, qemu, compileall python lib dir, timestamp: 11.682s -- `python:3-slim`, qemu, compileall python lib dir, checked: 11.708s -- `python:3-slim`, qemu, compileall python lib dir, checked, workers=0: 3.436s -- heroku gsg-ci, Perf-M, `pip --version`, existing timestamp: 0.202s -- heroku gsg-ci, Perf-M, `pip --version`, existing checked: 0.211s -- heroku gsg-ci, Perf-M, `pip --version`, existing unchecked: 0.202s -- heroku gsg-ci, Perf-M, `manage.py check`, existing timestamp: 0.283s -- heroku gsg-ci, Perf-M, `manage.py check`, existing checked: 0.299s -- heroku gsg-ci, Perf-M, `manage.py check`, existing unchecked: 0.282s - -Tested using: - -``` -find /app/.heroku/python/lib/python3.10/ -depth -type f -name "*.pyc" -delete -time python -m compileall -qq --invalidation-mode timestamp /app/.heroku/python/lib/python3.10/ -time python -m compileall -qq --invalidation-mode checked-hash /app/.heroku/python/lib/python3.10/ -time python -m compileall -qq --invalidation-mode unchecked-hash /app/.heroku/python/lib/python3.10/ -``` - -``` -find /usr/local -depth -type f -name "*.pyc" -delete -time python -m compileall -qq --invalidation-mode timestamp /usr/local/lib/python3.10/ -time python -m compileall -qq --invalidation-mode checked-hash /usr/local/lib/python3.10/ -time python -m compileall -qq --invalidation-mode unchecked-hash /usr/local/lib/python3.10/ -while true; do time pip --version; done -export SOURCE_DATE_EPOCH=1 -``` - -# Summary of runtime perf impact of checked vs unchecked pycs -- Native Docker, pip --version: +9ms on 152ms = +5.9% -- QEMU Docker, pip --version: +30ms on 1,356ms = +2.2% -- Heroku, pip --version: +9ms on 202ms = +4.5% -- Heroku, gsg manage.py check: +17ms on 282ms = +6.0% - -# pyc conclusion -- For Python runtime archive: delete all pycs, then regenerate using unchecked-hash -- For pip/setuptools/wheel: install using --no-compile, generate using unchecked-hash + concurrency -- For app dependencies installed using pip, either: - - Install using --no-compile, generate using unchecked-hash + concurrency - - Install using --no-compile, generate using checked-hash + concurrency - - Install normally, but ensure checked-hash by setting SOURCE_DATE_EPOCH -- For app dependencies installed using poetry (which doesn't support --no-compile), either: - - Install normally, but ensure checked-hash by setting SOURCE_DATE_EPOCH - - Install normally, then regenerate using unchecked-hash + concurrency - - Install normally, then regenerate using checked-hash + concurrency - -# bundled pip timings -- Bundled pip qemu: 5.2s for `--version` -- Bundled pip native: 0.6s for `--version` -- Unpacked pip qemu, without pycs: 3.3s for `--version` -- Unpacked pip native, without pycs: 0.4s for `--version` -- Unpacked pip qemu, with pycs: 1.4s for `--version` -- Unpacked pip native, with pycs: 0.2s for `--version` - -// before: -// time until pip install completed: 14.65s -// time until all completed (incl pycs): 16.65s -// after: -// time until pip install completed: 9.15s -// time until all completed (incl pycs): 11.15s From d5bb909c102abaa7a5ae99209eed6e6df4ff4de1 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Feb 2023 13:50:59 +0000 Subject: [PATCH 31/71] Update LICENSE year --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 4a8073f..8b62596 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2022 Salesforce, Inc. +Copyright (c) 2023 Salesforce, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without From 2f66e6846910e06d5a51f63c1928ace016155827 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Feb 2023 13:54:44 +0000 Subject: [PATCH 32/71] Move fixtures under tests/fixtures/ --- src/functions.rs | 8 ++++---- src/package_manager.rs | 4 ++-- src/project_descriptor.rs | 16 ++++++++-------- src/python_version.rs | 8 ++++---- src/runtime_txt.rs | 10 +++++----- src/utils.rs | 12 ++++++------ .../fixtures}/default/requirements.txt | 0 {test-fixtures => tests/fixtures}/empty/.gitkeep | 0 .../fixtures}/function_fails_self_check/main.py | 0 .../function_fails_self_check/project.toml | 0 .../function_fails_self_check/requirements.txt | 0 .../function_missing_functions_package/main.py | 0 .../project.toml | 0 .../requirements.txt | 0 .../fixtures}/function_template/README.md | 0 .../fixtures}/function_template/main.py | 0 .../fixtures}/function_template/payload.json | 0 .../fixtures}/function_template/project.toml | 0 .../fixtures}/function_template/requirements.txt | 0 .../fixtures}/project_toml_invalid/project.toml | 0 .../project_toml_non_salesforce/project.toml | 0 .../runtime_txt_python_3.10/runtime.txt | 0 .../requirements.txt | 0 .../runtime.txt | 0 .../requirements.txt | 0 .../runtime.txt | 0 tests/integration.rs | 14 +++++++------- 27 files changed, 36 insertions(+), 36 deletions(-) rename {test-fixtures => tests/fixtures}/default/requirements.txt (100%) rename {test-fixtures => tests/fixtures}/empty/.gitkeep (100%) rename {test-fixtures => tests/fixtures}/function_fails_self_check/main.py (100%) rename {test-fixtures => tests/fixtures}/function_fails_self_check/project.toml (100%) rename {test-fixtures => tests/fixtures}/function_fails_self_check/requirements.txt (100%) rename {test-fixtures => tests/fixtures}/function_missing_functions_package/main.py (100%) rename {test-fixtures => tests/fixtures}/function_missing_functions_package/project.toml (100%) rename {test-fixtures => tests/fixtures}/function_missing_functions_package/requirements.txt (100%) rename {test-fixtures => tests/fixtures}/function_template/README.md (100%) rename {test-fixtures => tests/fixtures}/function_template/main.py (100%) rename {test-fixtures => tests/fixtures}/function_template/payload.json (100%) rename {test-fixtures => tests/fixtures}/function_template/project.toml (100%) rename {test-fixtures => tests/fixtures}/function_template/requirements.txt (100%) rename {test-fixtures => tests/fixtures}/project_toml_invalid/project.toml (100%) rename {test-fixtures => tests/fixtures}/project_toml_non_salesforce/project.toml (100%) rename {test-fixtures => tests/fixtures}/runtime_txt_python_3.10/runtime.txt (100%) rename {test-fixtures => tests/fixtures}/runtime_txt_python_version_invalid/requirements.txt (100%) rename {test-fixtures => tests/fixtures}/runtime_txt_python_version_invalid/runtime.txt (100%) rename {test-fixtures => tests/fixtures}/runtime_txt_python_version_unavailable/requirements.txt (100%) rename {test-fixtures => tests/fixtures}/runtime_txt_python_version_unavailable/runtime.txt (100%) diff --git a/src/functions.rs b/src/functions.rs index 8ccc14d..888cadf 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -92,25 +92,25 @@ mod tests { #[test] fn is_function_project_no_project_toml() { - assert!(!is_function_project(Path::new("test-fixtures/empty")).unwrap()); + assert!(!is_function_project(Path::new("tests/fixtures/empty")).unwrap()); } #[test] fn is_function_project_non_salesforce_project_toml() { assert!( - !is_function_project(Path::new("test-fixtures/project_toml_non_salesforce")).unwrap() + !is_function_project(Path::new("tests/fixtures/project_toml_non_salesforce")).unwrap() ); } #[test] fn is_function_project_valid_function_project_toml() { - assert!(is_function_project(Path::new("test-fixtures/function_template")).unwrap()); + assert!(is_function_project(Path::new("tests/fixtures/function_template")).unwrap()); } #[test] fn is_function_project_invalid_project_toml() { assert!(matches!( - is_function_project(Path::new("test-fixtures/project_toml_invalid")).unwrap_err(), + is_function_project(Path::new("tests/fixtures/project_toml_invalid")).unwrap_err(), ReadProjectDescriptorError::Parse(_) )); } diff --git a/src/package_manager.rs b/src/package_manager.rs index 8d04596..b75d330 100644 --- a/src/package_manager.rs +++ b/src/package_manager.rs @@ -46,7 +46,7 @@ mod tests { #[test] fn determine_package_manager_requirements_txt() { assert!(matches!( - determine_package_manager(Path::new("test-fixtures/default")).unwrap(), + determine_package_manager(Path::new("tests/fixtures/default")).unwrap(), PackageManager::Pip )); } @@ -54,7 +54,7 @@ mod tests { #[test] fn determine_package_manager_none() { assert!(matches!( - determine_package_manager(Path::new("test-fixtures/empty")).unwrap_err(), + determine_package_manager(Path::new("tests/fixtures/empty")).unwrap_err(), DeterminePackageManagerError::NoneFound )); } diff --git a/src/project_descriptor.rs b/src/project_descriptor.rs index 5a70b67..26c04b1 100644 --- a/src/project_descriptor.rs +++ b/src/project_descriptor.rs @@ -200,14 +200,14 @@ mod tests { #[test] fn read_project_descriptor_no_project_toml_file() { - let app_dir = Path::new("test-fixtures/empty"); + let app_dir = Path::new("tests/fixtures/empty"); assert_eq!(read_project_descriptor(app_dir).unwrap(), None); } #[test] fn read_project_descriptor_non_salesforce() { - let app_dir = Path::new("test-fixtures/project_toml_non_salesforce"); + let app_dir = Path::new("tests/fixtures/project_toml_non_salesforce"); assert_eq!( read_project_descriptor(app_dir).unwrap(), @@ -219,7 +219,7 @@ mod tests { #[test] fn read_project_descriptor_function() { - let app_dir = Path::new("test-fixtures/function_template"); + let app_dir = Path::new("tests/fixtures/function_template"); assert_eq!( read_project_descriptor(app_dir).unwrap(), @@ -235,7 +235,7 @@ mod tests { #[test] fn read_project_descriptor_invalid_project_toml_file() { - let app_dir = Path::new("test-fixtures/project_toml_invalid"); + let app_dir = Path::new("tests/fixtures/project_toml_invalid"); assert!(matches!( read_project_descriptor(app_dir).unwrap_err(), @@ -245,21 +245,21 @@ mod tests { #[test] fn get_salesforce_project_type_missing() { - let app_dir = Path::new("test-fixtures/empty"); + let app_dir = Path::new("tests/fixtures/empty"); assert_eq!(read_salesforce_project_type(app_dir).unwrap(), None); } #[test] fn get_salesforce_project_type_non_salesforce() { - let app_dir = Path::new("test-fixtures/project_toml_non_salesforce"); + let app_dir = Path::new("tests/fixtures/project_toml_non_salesforce"); assert_eq!(read_salesforce_project_type(app_dir).unwrap(), None); } #[test] fn get_salesforce_project_type_function() { - let app_dir = Path::new("test-fixtures/function_template"); + let app_dir = Path::new("tests/fixtures/function_template"); assert_eq!( read_salesforce_project_type(app_dir).unwrap(), @@ -269,7 +269,7 @@ mod tests { #[test] fn get_salesforce_project_type_invalid_project_toml_file() { - let app_dir = Path::new("test-fixtures/project_toml_invalid"); + let app_dir = Path::new("tests/fixtures/project_toml_invalid"); assert!(matches!( read_salesforce_project_type(app_dir).unwrap_err(), diff --git a/src/python_version.rs b/src/python_version.rs index 81ae3d5..1f1b95a 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -73,12 +73,12 @@ mod tests { #[test] fn determine_python_version_runtime_txt_valid() { assert_eq!( - determine_python_version(Path::new("test-fixtures/runtime_txt_python_3.10")).unwrap(), + determine_python_version(Path::new("tests/fixtures/runtime_txt_python_3.10")).unwrap(), PythonVersion::new(3, 10, 9) ); assert_eq!( determine_python_version(Path::new( - "test-fixtures/runtime_txt_python_version_unavailable" + "tests/fixtures/runtime_txt_python_version_unavailable" )) .unwrap(), PythonVersion::new(999, 999, 999) @@ -89,7 +89,7 @@ mod tests { fn determine_python_version_runtime_txt_error() { assert!(matches!( determine_python_version(Path::new( - "test-fixtures/runtime_txt_python_version_invalid" + "tests/fixtures/runtime_txt_python_version_invalid" )) .unwrap_err(), PythonVersionError::RuntimeTxt(ReadRuntimeTxtError::Parse(_)) @@ -99,7 +99,7 @@ mod tests { #[test] fn determine_python_version_none_specified() { assert_eq!( - determine_python_version(Path::new("test-fixtures/empty")).unwrap(), + determine_python_version(Path::new("tests/fixtures/empty")).unwrap(), DEFAULT_PYTHON_VERSION ); } diff --git a/src/runtime_txt.rs b/src/runtime_txt.rs index b70c9a5..c423905 100644 --- a/src/runtime_txt.rs +++ b/src/runtime_txt.rs @@ -195,12 +195,12 @@ mod tests { #[test] fn read_version_valid_runtime_txt() { assert_eq!( - read_version(Path::new("test-fixtures/runtime_txt_python_3.10")).unwrap(), + read_version(Path::new("tests/fixtures/runtime_txt_python_3.10")).unwrap(), Some(PythonVersion::new(3, 10, 9)) ); assert_eq!( read_version(Path::new( - "test-fixtures/runtime_txt_python_version_unavailable" + "tests/fixtures/runtime_txt_python_version_unavailable" )) .unwrap(), Some(PythonVersion::new(999, 999, 999)) @@ -210,7 +210,7 @@ mod tests { #[test] fn read_version_runtime_txt_not_present() { assert_eq!( - read_version(Path::new("test-fixtures/empty")).unwrap(), + read_version(Path::new("tests/fixtures/empty")).unwrap(), None ); } @@ -218,7 +218,7 @@ mod tests { #[test] fn read_version_io_error() { assert!(matches!( - read_version(Path::new("test-fixtures/empty/.gitkeep")).unwrap_err(), + read_version(Path::new("tests/fixtures/empty/.gitkeep")).unwrap_err(), ReadRuntimeTxtError::Io(_) )); } @@ -227,7 +227,7 @@ mod tests { fn read_version_parse_error() { assert!(matches!( read_version(Path::new( - "test-fixtures/runtime_txt_python_version_invalid" + "tests/fixtures/runtime_txt_python_version_invalid" )) .unwrap_err(), ReadRuntimeTxtError::Parse(_) diff --git a/src/utils.rs b/src/utils.rs index 9e99ac7..5ae9882 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -99,24 +99,24 @@ mod tests { #[test] fn is_python_project_valid_project() { - assert!(is_python_project(Path::new("test-fixtures/default")).unwrap()); + assert!(is_python_project(Path::new("tests/fixtures/default")).unwrap()); } #[test] fn is_python_project_empty() { - assert!(!is_python_project(Path::new("test-fixtures/empty")).unwrap()); + assert!(!is_python_project(Path::new("tests/fixtures/empty")).unwrap()); } #[test] fn is_python_project_io_error() { - assert!(is_python_project(Path::new("test-fixtures/empty/.gitkeep")).is_err()); + assert!(is_python_project(Path::new("tests/fixtures/empty/.gitkeep")).is_err()); } #[test] fn read_optional_file_valid_file() { assert_eq!( read_optional_file(Path::new( - "test-fixtures/runtime_txt_python_3.10/runtime.txt" + "tests/fixtures/runtime_txt_python_3.10/runtime.txt" )) .unwrap(), Some("python-3.10.9\n".to_string()) @@ -127,7 +127,7 @@ mod tests { fn read_optional_file_missing_file() { assert_eq!( read_optional_file(Path::new( - "test-fixtures/non-existent-dir/non-existent-file" + "tests/fixtures/non-existent-dir/non-existent-file" )) .unwrap(), None @@ -136,7 +136,7 @@ mod tests { #[test] fn read_optional_file_io_error() { - assert!(read_optional_file(Path::new("test-fixtures/")).is_err()); + assert!(read_optional_file(Path::new("tests/fixtures/")).is_err()); } #[test] diff --git a/test-fixtures/default/requirements.txt b/tests/fixtures/default/requirements.txt similarity index 100% rename from test-fixtures/default/requirements.txt rename to tests/fixtures/default/requirements.txt diff --git a/test-fixtures/empty/.gitkeep b/tests/fixtures/empty/.gitkeep similarity index 100% rename from test-fixtures/empty/.gitkeep rename to tests/fixtures/empty/.gitkeep diff --git a/test-fixtures/function_fails_self_check/main.py b/tests/fixtures/function_fails_self_check/main.py similarity index 100% rename from test-fixtures/function_fails_self_check/main.py rename to tests/fixtures/function_fails_self_check/main.py diff --git a/test-fixtures/function_fails_self_check/project.toml b/tests/fixtures/function_fails_self_check/project.toml similarity index 100% rename from test-fixtures/function_fails_self_check/project.toml rename to tests/fixtures/function_fails_self_check/project.toml diff --git a/test-fixtures/function_fails_self_check/requirements.txt b/tests/fixtures/function_fails_self_check/requirements.txt similarity index 100% rename from test-fixtures/function_fails_self_check/requirements.txt rename to tests/fixtures/function_fails_self_check/requirements.txt diff --git a/test-fixtures/function_missing_functions_package/main.py b/tests/fixtures/function_missing_functions_package/main.py similarity index 100% rename from test-fixtures/function_missing_functions_package/main.py rename to tests/fixtures/function_missing_functions_package/main.py diff --git a/test-fixtures/function_missing_functions_package/project.toml b/tests/fixtures/function_missing_functions_package/project.toml similarity index 100% rename from test-fixtures/function_missing_functions_package/project.toml rename to tests/fixtures/function_missing_functions_package/project.toml diff --git a/test-fixtures/function_missing_functions_package/requirements.txt b/tests/fixtures/function_missing_functions_package/requirements.txt similarity index 100% rename from test-fixtures/function_missing_functions_package/requirements.txt rename to tests/fixtures/function_missing_functions_package/requirements.txt diff --git a/test-fixtures/function_template/README.md b/tests/fixtures/function_template/README.md similarity index 100% rename from test-fixtures/function_template/README.md rename to tests/fixtures/function_template/README.md diff --git a/test-fixtures/function_template/main.py b/tests/fixtures/function_template/main.py similarity index 100% rename from test-fixtures/function_template/main.py rename to tests/fixtures/function_template/main.py diff --git a/test-fixtures/function_template/payload.json b/tests/fixtures/function_template/payload.json similarity index 100% rename from test-fixtures/function_template/payload.json rename to tests/fixtures/function_template/payload.json diff --git a/test-fixtures/function_template/project.toml b/tests/fixtures/function_template/project.toml similarity index 100% rename from test-fixtures/function_template/project.toml rename to tests/fixtures/function_template/project.toml diff --git a/test-fixtures/function_template/requirements.txt b/tests/fixtures/function_template/requirements.txt similarity index 100% rename from test-fixtures/function_template/requirements.txt rename to tests/fixtures/function_template/requirements.txt diff --git a/test-fixtures/project_toml_invalid/project.toml b/tests/fixtures/project_toml_invalid/project.toml similarity index 100% rename from test-fixtures/project_toml_invalid/project.toml rename to tests/fixtures/project_toml_invalid/project.toml diff --git a/test-fixtures/project_toml_non_salesforce/project.toml b/tests/fixtures/project_toml_non_salesforce/project.toml similarity index 100% rename from test-fixtures/project_toml_non_salesforce/project.toml rename to tests/fixtures/project_toml_non_salesforce/project.toml diff --git a/test-fixtures/runtime_txt_python_3.10/runtime.txt b/tests/fixtures/runtime_txt_python_3.10/runtime.txt similarity index 100% rename from test-fixtures/runtime_txt_python_3.10/runtime.txt rename to tests/fixtures/runtime_txt_python_3.10/runtime.txt diff --git a/test-fixtures/runtime_txt_python_version_invalid/requirements.txt b/tests/fixtures/runtime_txt_python_version_invalid/requirements.txt similarity index 100% rename from test-fixtures/runtime_txt_python_version_invalid/requirements.txt rename to tests/fixtures/runtime_txt_python_version_invalid/requirements.txt diff --git a/test-fixtures/runtime_txt_python_version_invalid/runtime.txt b/tests/fixtures/runtime_txt_python_version_invalid/runtime.txt similarity index 100% rename from test-fixtures/runtime_txt_python_version_invalid/runtime.txt rename to tests/fixtures/runtime_txt_python_version_invalid/runtime.txt diff --git a/test-fixtures/runtime_txt_python_version_unavailable/requirements.txt b/tests/fixtures/runtime_txt_python_version_unavailable/requirements.txt similarity index 100% rename from test-fixtures/runtime_txt_python_version_unavailable/requirements.txt rename to tests/fixtures/runtime_txt_python_version_unavailable/requirements.txt diff --git a/test-fixtures/runtime_txt_python_version_unavailable/runtime.txt b/tests/fixtures/runtime_txt_python_version_unavailable/runtime.txt similarity index 100% rename from test-fixtures/runtime_txt_python_version_unavailable/runtime.txt rename to tests/fixtures/runtime_txt_python_version_unavailable/runtime.txt diff --git a/tests/integration.rs b/tests/integration.rs index 22151c5..ea9eae6 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -19,7 +19,7 @@ fn builder() -> String { #[ignore = "integration test"] fn detect_rejects_non_python_projects() { TestRunner::default().build( - BuildConfig::new(builder(), "test-fixtures/empty") + BuildConfig::new(builder(), "tests/fixtures/empty") .expected_pack_result(PackResult::Failure), |context| { // We can't test the detect failure reason, since by default pack CLI only shows output for non-zero, @@ -37,7 +37,7 @@ fn detect_rejects_non_python_projects() { #[ignore = "integration test"] fn function_template() { TestRunner::default().build( - BuildConfig::new(builder(), "test-fixtures/function_template"), + BuildConfig::new(builder(), "tests/fixtures/function_template"), |context| { // Pip outputs git clone output to stderr for some reason, so stderr isn't empty. // TODO: Decide whether this is a bug in pip and/or if we should work around it. @@ -112,7 +112,7 @@ fn function_template() { #[ignore = "integration test"] fn function_repeat_build() { TestRunner::default().build( - BuildConfig::new(builder(), "test-fixtures/function_template"), + BuildConfig::new(builder(), "tests/fixtures/function_template"), |context| { let config = context.config.clone(); context.rebuild(config, |rebuild_context| { @@ -148,7 +148,7 @@ fn runtime_txt_python_version_unavailable() { TestRunner::default().build( BuildConfig::new( &builder, - "test-fixtures/runtime_txt_python_version_unavailable", + "tests/fixtures/runtime_txt_python_version_unavailable", ) .expected_pack_result(PackResult::Failure), |context| { @@ -181,7 +181,7 @@ fn runtime_txt_python_version_invalid() { TestRunner::default().build( BuildConfig::new( builder(), - "test-fixtures/runtime_txt_python_version_invalid", + "tests/fixtures/runtime_txt_python_version_invalid", ) .expected_pack_result(PackResult::Failure), |context| { @@ -217,7 +217,7 @@ fn function_missing_functions_package() { TestRunner::default().build( BuildConfig::new( builder(), - "test-fixtures/function_missing_functions_package", + "tests/fixtures/function_missing_functions_package", ) .expected_pack_result(PackResult::Failure), |context| { @@ -245,7 +245,7 @@ fn function_fails_self_check() { TestRunner::default().build( BuildConfig::new( builder(), - "test-fixtures/function_fails_self_check", + "tests/fixtures/function_fails_self_check", ) .expected_pack_result(PackResult::Failure), |context| { From 738762561db39268554a7415a27814900d102757 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Feb 2023 14:04:41 +0000 Subject: [PATCH 33/71] Switch check-changelog to `ubuntu-latest` --- .github/workflows/check_changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_changelog.yml b/.github/workflows/check_changelog.yml index c590644..b3ab6a1 100644 --- a/.github/workflows/check_changelog.yml +++ b/.github/workflows/check_changelog.yml @@ -9,7 +9,7 @@ permissions: jobs: check-changelog: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest if: | !contains(github.event.pull_request.labels.*.name, 'skip changelog') && !contains(github.event.pull_request.labels.*.name, 'dependencies') From 8ef2a8414a6aad4d3c6170f6904a53a722c14391 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Feb 2023 14:15:55 +0000 Subject: [PATCH 34/71] Fix functions integration test after 0.5.0 release --- tests/integration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration.rs b/tests/integration.rs index ea9eae6..de3ff20 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -257,7 +257,7 @@ fn function_fails_self_check() { there is a problem with the Python Salesforce Function in this project. Details: - Function failed validation: 'invalid' is not a valid Salesforce REST API version. Update 'salesforce-api-version' in project.toml to a version of form 'X.Y'. + Function failed validation: 'invalid' isn't a valid Salesforce REST API version. Update the 'salesforce-api-version' key in project.toml to a version that uses the form 'X.Y', such as '56.0'. "} ); }, From 70373e5fda8c5241ad98aab57b8213e799d9e959 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Feb 2023 14:17:56 +0000 Subject: [PATCH 35/71] Add CHANGELOG.md --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..53a218b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- Initial implementation. From 1d2db4687f592d2ccf98514b60d4cee73d0fb1b5 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Feb 2023 14:26:14 +0000 Subject: [PATCH 36/71] Only run CI on PRs not branches To avoid duplicate runs. --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00aca60..cfaa73c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,9 +3,8 @@ name: CI on: push: # Avoid duplicate builds on PRs. - # TODO: Uncomment once this is merged to `main`. - # branches: - # - main + branches: + - main pull_request: permissions: From a53fd3e0423a5883124738473b8806cbf7ead810 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:15:11 +0000 Subject: [PATCH 37/71] Switch from env var to `--disable-pip-version-check` --- src/layers/pip_dependencies.rs | 3 +++ src/layers/python.rs | 9 +-------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 20ae75f..13a93ee 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -58,6 +58,9 @@ impl Layer for PipDependenciesLayer<'_> { "install", "--cache-dir", &self.pip_cache_dir.to_string_lossy(), + // We use a curated Pip version, so skip the update check to speed up Pip invocations, + // reduce build log spam and prevent users from thinking they need to manually upgrade. + "--disable-pip-version-check", "--no-input", // Prevent warning about the `bin/` directory not being on `PATH`, since it // will be added automatically by libcnb/lifecycle later. diff --git a/src/layers/python.rs b/src/layers/python.rs index 7a789d2..dda7444 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -112,14 +112,6 @@ impl Layer for PythonLayer<'_> { "PKG_CONFIG_PATH", ":", ) - // We use a curated Pip version, so skip the update check to speed up Pip invocations, - // reduce build log spam and prevent users from thinking they need to manually upgrade. - .chainable_insert( - Scope::All, - ModificationBehavior::Override, - "PIP_DISABLE_PIP_VERSION_CHECK", - "1", - ) // Disable Python's output buffering to ensure logs aren't dropped if an app crashes. .chainable_insert( Scope::All, @@ -154,6 +146,7 @@ impl Layer for PythonLayer<'_> { .args([ &bundled_pip_module.to_string_lossy(), "install", + "--disable-pip-version-check", "--no-cache-dir", "--no-input", "--quiet", From c9bae312f1042b29b0797208c870fb9b1bd301b9 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 7 Feb 2023 17:59:28 +0000 Subject: [PATCH 38/71] Python/pip layer refactoring + unit tests --- src/layers/pip_dependencies.rs | 7 +- src/layers/python.rs | 197 ++++++++++++++++++++++----------- src/main.rs | 6 +- 3 files changed, 139 insertions(+), 71 deletions(-) diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 13a93ee..691fe59 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -11,8 +11,11 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{fs, io}; +/// Layer containing the application's Python dependencies, installed using Pip. pub(crate) struct PipDependenciesLayer<'a> { - pub env: &'a Env, + /// Environment variables inherited from earlier buildpack steps. + pub base_env: &'a Env, + /// The path to the Pip cache directory, which is stored in another layer since it isn't needed at runtime. pub pip_cache_dir: PathBuf, } @@ -42,7 +45,7 @@ impl Layer for PipDependenciesLayer<'_> { "PYTHONUSERBASE", layer_path, ); - let env = layer_env.apply(Scope::Build, self.env); + let env = layer_env.apply(Scope::Build, self.base_env); let src_dir = layer_path.join("src"); fs::create_dir(&src_dir).map_err(PipDependenciesLayerError::CreateSrcDirIo)?; diff --git a/src/layers/python.rs b/src/layers/python.rs index dda7444..88c0edf 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -19,8 +19,11 @@ const PIP_VERSION: &str = "23.0"; const SETUPTOOLS_VERSION: &str = "67.1.0"; const WHEEL_VERSION: &str = "0.38.4"; +/// Layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`. pub(crate) struct PythonLayer<'a> { - pub env: &'a Env, + /// Environment variables inherited from earlier buildpack steps. + pub base_env: &'a Env, + /// The Python version that will be installed. pub python_version: &'a PythonVersion, } @@ -45,7 +48,6 @@ impl Layer for PythonLayer<'_> { } } - #[allow(clippy::too_many_lines)] fn create( &self, context: &BuildContext, @@ -75,51 +77,8 @@ impl Layer for PythonLayer<'_> { })?; log_info("Python installation successful"); - // Remember to force invalidation of the cached layer if this list ever changes. - let layer_env = LayerEnv::new() - // We have to set `CPATH` explicitly, since the automatic path set by lifecycle/libcnb is - // `/include/` whereas Python's header files are at `/include/pythonX.Y/` - // (and compilers don't recursively search). - .chainable_insert( - Scope::All, - ModificationBehavior::Prepend, - "CPATH", - layer_path.join(format!( - "include/python{}.{}", - self.python_version.major, self.python_version.minor - )), - ) - .chainable_insert(Scope::All, ModificationBehavior::Delimiter, "CPATH", ":") - // Ensure Python uses a Unicode locate, to prevent the issues described in: - // https://github.com/docker-library/python/pull/570 - .chainable_insert( - Scope::All, - ModificationBehavior::Override, - "LANG", - "C.UTF-8", - ) - // We have to set `PKG_CONFIG_PATH` explicitly, since the automatic path set by lifecycle/libcnb - // is `/pkgconfig/`, whereas Python's pkgconfig files are at `/lib/pkgconfig/`. - .chainable_insert( - Scope::All, - ModificationBehavior::Prepend, - "PKG_CONFIG_PATH", - layer_path.join("lib/pkgconfig"), - ) - .chainable_insert( - Scope::All, - ModificationBehavior::Delimiter, - "PKG_CONFIG_PATH", - ":", - ) - // Disable Python's output buffering to ensure logs aren't dropped if an app crashes. - .chainable_insert( - Scope::All, - ModificationBehavior::Override, - "PYTHONUNBUFFERED", - "1", - ); - let mut env = layer_env.apply(Scope::Build, self.env); + let layer_env = generate_layer_env(layer_path, self.python_version); + let mut env = layer_env.apply(Scope::Build, self.base_env); // The Python binaries are built using `--shared`, and since they're being installed at a // different location from their original `--prefix`, they need `LD_LIBRARY_PATH` to be set @@ -138,15 +97,18 @@ impl Layer for PythonLayer<'_> { )); let site_packages_dir = python_stdlib_dir.join("site-packages"); - // TODO: Explain what's happening here - let bundled_pip_module = - bundled_pip_module(&python_stdlib_dir).map_err(PythonLayerError::LocateBundledPipIo)?; + // Python bundles Pip within its standard library, which we can use to install our chosen + // pip version from PyPI, saving us from having to download the usual pip bootstrap script. + let bundled_pip_module_path = bundled_pip_module_path(&python_stdlib_dir) + .map_err(PythonLayerError::LocateBundledPipIo)?; + utils::run_command( Command::new(python_binary) .args([ - &bundled_pip_module.to_string_lossy(), + &bundled_pip_module_path.to_string_lossy(), "install", "--disable-pip-version-check", + // There is no point using Pip's cache here, since the layer itself will be cached. "--no-cache-dir", "--no-input", "--quiet", @@ -168,7 +130,7 @@ impl Layer for PythonLayer<'_> { // site-packages, it's possible other buildpacks or custom scripts may forget to do so. // By making the system site-packages directory read-only, Pip will automatically use // user installs in such cases: - // https://github.com/pypa/pip/blob/22.3.1/src/pip/_internal/commands/install.py#L706-L764 + // https://github.com/pypa/pip/blob/23.0/src/pip/_internal/commands/install.py#L715-L773 fs::set_permissions(site_packages_dir, Permissions::from_mode(0o555)) .map_err(PythonLayerError::MakeSitePackagesReadOnlyIo)?; @@ -222,28 +184,80 @@ impl Layer for PythonLayer<'_> { } } -// TODO: Explain what's happening here -// The bundled version of Pip (and thus the wheel filename) varies across Python versions, -// so we have to search the bundled wheels directory for the appropriate file. -// TODO: This returns a module path rather than a wheel path - change? -fn bundled_pip_module(python_stdlib_dir: &Path) -> io::Result { +/// Environment variables that will be set by this layer. +fn generate_layer_env(layer_path: &Path, python_version: &PythonVersion) -> LayerEnv { + // Several of the env vars below are technically build-time only vars, however, we use + // `Scope::All` instead of `Scope::Build` to reduce confusion if pip install commands + // are used at runtime when debugging. + // + // Remember to force invalidation of the cached layer if these env vars ever change. + LayerEnv::new() + // We have to set `CPATH` explicitly, since the automatic path set by lifecycle/libcnb is + // `/include/` whereas Python's header files are at `/include/pythonX.Y/` + // (and compilers don't recursively search). + .chainable_insert( + Scope::All, + ModificationBehavior::Prepend, + "CPATH", + layer_path.join(format!( + "include/python{}.{}", + python_version.major, python_version.minor + )), + ) + .chainable_insert(Scope::All, ModificationBehavior::Delimiter, "CPATH", ":") + // Ensure Python uses a Unicode locate, to prevent the issues described in: + // https://github.com/docker-library/python/pull/570 + .chainable_insert( + Scope::All, + ModificationBehavior::Override, + "LANG", + "C.UTF-8", + ) + // We have to set `PKG_CONFIG_PATH` explicitly, since the automatic path set by lifecycle/libcnb + // is `/pkgconfig/`, whereas Python's pkgconfig files are at `/lib/pkgconfig/`. + .chainable_insert( + Scope::All, + ModificationBehavior::Prepend, + "PKG_CONFIG_PATH", + layer_path.join("lib/pkgconfig"), + ) + .chainable_insert( + Scope::All, + ModificationBehavior::Delimiter, + "PKG_CONFIG_PATH", + ":", + ) + // Disable Python's output buffering to ensure logs aren't dropped if an app crashes. + .chainable_insert( + Scope::All, + ModificationBehavior::Override, + "PYTHONUNBUFFERED", + "1", + ) +} + +/// The path to the Pip module bundled in Python's standard library. +fn bundled_pip_module_path(python_stdlib_dir: &Path) -> io::Result { let bundled_wheels_dir = python_stdlib_dir.join("ensurepip/_bundled"); - let pip_wheel_filename_prefix = "pip-"; + // The wheel filename includes the Pip version (for example `pip-XX.Y-py3-none-any.whl`), + // which varies from one Python release to the next (including between patch releases). + // As such, we have to find the wheel based on the known filename prefix of `pip-`. for entry in fs::read_dir(bundled_wheels_dir)? { let entry = entry?; - if entry - .file_name() - .to_string_lossy() - .starts_with(pip_wheel_filename_prefix) - { - return Ok(entry.path().join("pip")); + if entry.file_name().to_string_lossy().starts_with("pip-") { + let pip_wheel_path = entry.path(); + // The Pip module exists inside the pip wheel (which is a zip file), however, + // Python can load it directly by appending the module name to the zip filename, + // as though it were a path. For example: `pip-XX.Y-py3-none-any.whl/pip` + let pip_module_path = pip_wheel_path.join("pip"); + return Ok(pip_module_path); } } Err(io::Error::new( io::ErrorKind::NotFound, - format!("No files found matching the filename prefix of '{pip_wheel_filename_prefix}'"), + "No files found matching the pip wheel filename prefix", )) } @@ -279,4 +293,55 @@ impl From for BuildpackError { } } -// TODO: Unit tests for cache invalidation handling? +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_layer_env() { + let mut base_env = Env::new(); + base_env.insert("CPATH", "/base"); + base_env.insert("LANG", "this-should-be-overridden"); + base_env.insert("PKG_CONFIG_PATH", "/base"); + base_env.insert("PYTHONUNBUFFERED", "this-should-be-overridden"); + + let layer_env = generate_layer_env( + Path::new("/layers/python"), + &PythonVersion { + major: 3, + minor: 11, + patch: 1, + }, + ); + + // Remember to force invalidation of the cached layer if these env vars ever change. + assert_eq!( + environment_as_sorted_vector(&layer_env.apply(Scope::Build, &base_env)), + vec![ + ("CPATH", "/layers/python/include/python3.11:/base"), + ("LANG", "C.UTF-8"), + ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig:/base"), + ("PYTHONUNBUFFERED", "1"), + ] + ); + assert_eq!( + environment_as_sorted_vector(&layer_env.apply(Scope::Launch, &base_env)), + vec![ + ("CPATH", "/layers/python/include/python3.11:/base"), + ("LANG", "C.UTF-8"), + ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig:/base"), + ("PYTHONUNBUFFERED", "1"), + ] + ); + } + + fn environment_as_sorted_vector(environment: &Env) -> Vec<(&str, &str)> { + let mut result: Vec<(&str, &str)> = environment + .iter() + .map(|(k, v)| (k.to_str().unwrap(), v.to_str().unwrap())) + .collect(); + + result.sort_by_key(|kv| kv.0); + result + } +} diff --git a/src/main.rs b/src/main.rs index 6ed06ce..17b3f0d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -67,11 +67,11 @@ impl Buildpack for PythonBuildpack { // env vars will still be excluded, due to the use of `clear-env` in `buildpack.toml`. let mut env = Env::from_current(); - // Create the layer containing the Python runtime and required packaging tools. + // Create the layer containing the Python runtime and the packages `pip`, `setuptools` and `wheel`. let python_layer = context.handle_layer( layer_name!("python"), PythonLayer { - env: &env, + base_env: &env, python_version: &python_version, }, )?; @@ -91,7 +91,7 @@ impl Buildpack for PythonBuildpack { let pip_layer = context.handle_layer( layer_name!("dependencies"), PipDependenciesLayer { - env: &env, + base_env: &env, pip_cache_dir: pip_cache_layer.path, }, )?; From 631b212787b544b73720b9c12773aaeec6237dee Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 7 Feb 2023 22:18:58 +0000 Subject: [PATCH 39/71] Refresh Cargo.lock --- Cargo.lock | 293 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 211 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aad302b..1f1f553 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -20,6 +29,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bit-set" version = "0.5.3" @@ -43,11 +58,11 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bollard" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82e7850583ead5f8bbef247e2a3c37a19bd576e8420cd262a6711921827e1e5" +checksum = "af254ed2da4936ef73309e9597180558821cb16ae9bba4cb24ce6b612d8d80ed" dependencies = [ - "base64", + "base64 0.21.0", "bollard-stubs", "bytes", "futures-core", @@ -61,6 +76,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "serde_repr", "serde_urlencoded", "thiserror", "tokio", @@ -71,9 +87,9 @@ dependencies = [ [[package]] name = "bollard-stubs" -version = "1.42.0-rc.3" +version = "1.42.0-rc.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed59b5c00048f48d7af971b71f800fdf23e858844a6f9e4d32ca72e9399e7864" +checksum = "602bda35f33aeb571cef387dcd4042c643a8bf689d8aaac2cc47ea24cb7bc7e0" dependencies = [ "serde", "serde_with", @@ -135,6 +151,35 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +dependencies = [ + "iana-time-zone", + "num-integer", + "num-traits", + "serde", + "winapi", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "crc32fast" version = "1.3.2" @@ -145,36 +190,45 @@ dependencies = [ ] [[package]] -name = "darling" -version = "0.13.4" +name = "cxx" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +checksum = "bc831ee6a32dd495436e317595e639a587aa9907bef96fe6e6abc290ab6204e9" dependencies = [ - "darling_core", - "darling_macro", + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", ] [[package]] -name = "darling_core" -version = "0.13.4" +name = "cxx-build" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +checksum = "94331d54f1b1a8895cd81049f7eaaaef9d05a7dcb4d1fd08bf3ff0806246789d" dependencies = [ - "fnv", - "ident_case", + "cc", + "codespan-reporting", + "once_cell", "proc-macro2", "quote", - "strsim", + "scratch", "syn", ] [[package]] -name = "darling_macro" -version = "0.13.4" +name = "cxxbridge-flags" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +checksum = "48dcd35ba14ca9b40d6e4b4b39961f23d835dbb8eed74565ded361d93e1feb8a" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bbeb29798b407ccd82a3324ade1a7286e0d29851475990b612670f6f5124d2" dependencies = [ - "darling_core", + "proc-macro2", "quote", "syn", ] @@ -187,9 +241,9 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "fancy-regex" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0678ab2d46fa5195aaf59ad034c083d351377d4af57f3e073c074d0da3e3c766" +checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" dependencies = [ "bit-set", "regex", @@ -244,9 +298,9 @@ dependencies = [ [[package]] name = "fs_extra" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures-channel" @@ -412,10 +466,28 @@ dependencies = [ ] [[package]] -name = "ident_case" -version = "1.0.1" +name = "iana-time-zone" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] [[package]] name = "idna" @@ -435,6 +507,7 @@ checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", + "serde", ] [[package]] @@ -475,47 +548,47 @@ checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libcnb" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69d45189983fb0a9996ded95236bb8689437b0e1e636dddf2500e9ec27ab4c0" +checksum = "4e4fd7573558173267930e31446da65a0275770bde88847cad4b4cf9a6ff8375" dependencies = [ "libcnb-data", "libcnb-proc-macros", "serde", "thiserror", - "toml 0.5.11", + "toml", ] [[package]] name = "libcnb-data" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a065640c66df2a6e54aedbb805d264c87020937323b90eea7397108b73d3aa" +checksum = "8c0112478d479c8900929894426818bea8e769ce923536a58baac719d3ca4dcb" dependencies = [ "fancy-regex", "libcnb-proc-macros", "serde", "thiserror", - "toml 0.5.11", + "toml", ] [[package]] name = "libcnb-package" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ed34a92d997ad9b0666ddbcc3995191e7642ee50ffa760497d2fb3bff7c5b5" +checksum = "aacd18d358a1078cf48f518ef8398c504f8d4fc691ba2e8773bafa1a71d66b59" dependencies = [ "cargo_metadata", "libcnb-data", - "toml 0.5.11", + "toml", "which", ] [[package]] name = "libcnb-proc-macros" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b3879fd4fc4338421de1ec797ab5ef0abe6d0e90f843dbf3b56c25bc703ebe" +checksum = "5930cea22615255081c0c44b902e6e8b37a824ebe1374a7c7d52724d5b7d6e4e" dependencies = [ "cargo_metadata", "fancy-regex", @@ -525,9 +598,9 @@ dependencies = [ [[package]] name = "libcnb-test" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f414f5b106078d0bbb67b9e3d3bf9e21012f3a318505649e8e99c9d36d200ea" +checksum = "f86e8c1847c8ba3c37e30841ee241887203110f4373731e7967706ab77c42b7d" dependencies = [ "bollard", "cargo_metadata", @@ -543,9 +616,9 @@ dependencies = [ [[package]] name = "libherokubuildpack" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8085f21847f46079ce900bf2169e3e51ffa3685dc298aa71056a29d96d4413cb" +checksum = "878674906e0140191f89047ef1e8c142cb31becce91b4e64b1b6419fe03da7c1" dependencies = [ "termcolor", ] @@ -561,6 +634,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + [[package]] name = "log" version = "0.4.17" @@ -606,6 +688,25 @@ dependencies = [ "memchr", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -668,9 +769,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "proc-macro2" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] @@ -686,7 +787,7 @@ dependencies = [ "libherokubuildpack", "serde", "tar", - "toml 0.7.1", + "toml", "ureq", ] @@ -765,6 +866,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +[[package]] +name = "scratch" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" + [[package]] name = "sct" version = "0.7.0" @@ -806,15 +913,26 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a" dependencies = [ "itoa", "ryu", "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a5ec9fa74a20ebbe5d9ac23dac1fc96ba0ecfe9f50f2843b52e537b10fbcb4e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_spanned" version = "0.6.1" @@ -838,24 +956,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "1.14.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +checksum = "30d904179146de381af4c93d3af6ca4984b3152db687dacb9c3c35e86f39809c" dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap", "serde", - "serde_with_macros", -] - -[[package]] -name = "serde_with_macros" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", + "serde_json", + "time", ] [[package]] @@ -883,12 +994,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "syn" version = "1.0.107" @@ -954,6 +1059,33 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +dependencies = [ + "time-core", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1013,18 +1145,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.11" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "toml" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772c1426ab886e7362aedf4abc9c0d1348a979517efedfc25862944d10137af0" +checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6" dependencies = [ "serde", "serde_spanned", @@ -1043,9 +1166,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.1" +version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90a238ee2e6ede22fb95350acc78e21dc40da00bb66c0334bde83de4ed89424e" +checksum = "5e6a7712b49e1775fb9a7b998de6635b299237f48b404dde71704f2e0e7f37e5" dependencies = [ "indexmap", "nom8", @@ -1107,6 +1230,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "untrusted" version = "0.7.1" @@ -1119,7 +1248,7 @@ version = "2.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d" dependencies = [ - "base64", + "base64 0.13.1", "log", "once_cell", "rustls", From 391519e620686945e960b872e28271dd8b73d4c6 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 7 Feb 2023 22:24:52 +0000 Subject: [PATCH 40/71] Cleanup errors.rs --- src/errors.rs | 85 ++++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 5952fd4..dbc801b 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -3,7 +3,7 @@ use crate::layers::pip_dependencies::PipDependenciesLayerError; use crate::layers::python::PythonLayerError; use crate::package_manager::DeterminePackageManagerError; use crate::project_descriptor::ReadProjectDescriptorError; -use crate::python_version::{PythonVersionError, DEFAULT_PYTHON_VERSION}; +use crate::python_version::{PythonVersion, PythonVersionError, DEFAULT_PYTHON_VERSION}; use crate::runtime_txt::{ParseRuntimeTxtError, ReadRuntimeTxtError}; use crate::utils::{CommandError, DownloadUnpackArchiveError}; use crate::BuildpackError; @@ -40,8 +40,8 @@ pub(crate) fn on_error(error: libcnb::Error) { }; } -fn on_buildpack_error(buildpack_error: BuildpackError) { - match buildpack_error { +fn on_buildpack_error(error: BuildpackError) { + match error { BuildpackError::CheckFunction(error) => on_check_function_error(error), BuildpackError::DetectIo(io_error) => log_io_error( "Unable to complete buildpack detection", @@ -56,8 +56,8 @@ fn on_buildpack_error(buildpack_error: BuildpackError) { }; } -fn on_project_descriptor_error(project_descriptor_error: ReadProjectDescriptorError) { - match project_descriptor_error { +fn on_project_descriptor_error(error: ReadProjectDescriptorError) { + match error { ReadProjectDescriptorError::Io(io_error) => log_io_error( "Unable to read project.toml", "reading the (optional) project.toml file", @@ -74,10 +74,8 @@ fn on_project_descriptor_error(project_descriptor_error: ReadProjectDescriptorEr }; } -fn on_determine_package_manager_error( - determine_package_manager_error: DeterminePackageManagerError, -) { - match determine_package_manager_error { +fn on_determine_package_manager_error(error: DeterminePackageManagerError) { + match error { DeterminePackageManagerError::Io(io_error) => log_io_error( "Unable to determine the package manager", "determining which Python package manager to use for this project", @@ -100,8 +98,8 @@ fn on_determine_package_manager_error( }; } -fn on_python_version_error(python_version_error: PythonVersionError) { - match python_version_error { +fn on_python_version_error(error: PythonVersionError) { + match error { PythonVersionError::RuntimeTxt(error) => match error { ReadRuntimeTxtError::Io(io_error) => log_io_error( "Unable to read runtime.txt", @@ -109,37 +107,40 @@ fn on_python_version_error(python_version_error: PythonVersionError) { &io_error, ), // TODO: Write the supported Python versions inline, instead of linking out to Dev Center. - ReadRuntimeTxtError::Parse(ParseRuntimeTxtError { cleaned_contents }) => log_error( - "Invalid Python version in runtime.txt", - formatdoc! {" - The Python version specified in 'runtime.txt' is not in the correct format. - - The following file contents were found: - {cleaned_contents} - - However, the file contents must begin with a 'python-' prefix, followed by the - version specified as '..'. Comments are not supported. - - For example, to request Python {DEFAULT_PYTHON_VERSION}, the correct version format is: - python-{major}.{minor}.{patch} - - Please update 'runtime.txt' to use the correct version format, or else remove - the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). - - For a list of the supported Python versions, see: - https://devcenter.heroku.com/articles/python-support#supported-runtimes - ", - major = DEFAULT_PYTHON_VERSION.major, - minor = DEFAULT_PYTHON_VERSION.minor, - patch = DEFAULT_PYTHON_VERSION.patch - }, - ), + ReadRuntimeTxtError::Parse(ParseRuntimeTxtError { cleaned_contents }) => { + let PythonVersion { + major, + minor, + patch, + } = DEFAULT_PYTHON_VERSION; + log_error( + "Invalid Python version in runtime.txt", + formatdoc! {" + The Python version specified in 'runtime.txt' is not in the correct format. + + The following file contents were found: + {cleaned_contents} + + However, the file contents must begin with a 'python-' prefix, followed by the + version specified as '..'. Comments are not supported. + + For example, to request Python {DEFAULT_PYTHON_VERSION}, the correct version format is: + python-{major}.{minor}.{patch} + + Please update 'runtime.txt' to use the correct version format, or else remove + the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). + + For a list of the supported Python versions, see: + https://devcenter.heroku.com/articles/python-support#supported-runtimes + "}, + ); + } }, }; } -fn on_python_layer_error(python_layer_error: PythonLayerError) { - match python_layer_error { +fn on_python_layer_error(error: PythonLayerError) { + match error { PythonLayerError::BootstrapPipCommand(error) => match error { CommandError::Io(io_error) => log_io_error( "Unable to bootstrap pip", @@ -210,8 +211,8 @@ fn on_python_layer_error(python_layer_error: PythonLayerError) { }; } -fn on_pip_dependencies_layer_error(pip_dependencies_layer_error: PipDependenciesLayerError) { - match pip_dependencies_layer_error { +fn on_pip_dependencies_layer_error(error: PipDependenciesLayerError) { + match error { PipDependenciesLayerError::CreateSrcDirIo(io_error) => log_io_error( "Unable to create 'src' directory required for pip install", "creating the 'src' directory in the pip layer, prior to running pip install", @@ -238,8 +239,8 @@ fn on_pip_dependencies_layer_error(pip_dependencies_layer_error: PipDependencies }; } -fn on_check_function_error(check_function_error: CheckFunctionError) { - match check_function_error { +fn on_check_function_error(error: CheckFunctionError) { + match error { CheckFunctionError::Io(io_error) => log_io_error( "Unable to run the Salesforce Functions self-check command", &format!("running the '{FUNCTION_RUNTIME_PROGRAM_NAME} check' command"), From bb01b50966400d31c6d39f981d8c435462ff60d1 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 8 Feb 2023 16:38:08 +0000 Subject: [PATCH 41/71] Add `.env_clear()` to all `Command` usages This is mostly for completeness, since we do want to use the current process's env vars, however we've already inherited them in `main.rs` via `Env::from_current()`, and so the env we're passing around is the exact env we want to use. By adding `.env_clear()` we prevent subtle bugs if we ever unset any of the inherited envs in the passed around `env`. --- src/functions.rs | 1 + src/layers/pip_dependencies.rs | 3 +-- src/layers/python.rs | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/functions.rs b/src/functions.rs index 888cadf..3c84d51 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -30,6 +30,7 @@ pub(crate) fn check_function(env: &Env) -> Result<(), CheckFunctionError> { // display it if the check command fails. Command::new(FUNCTION_RUNTIME_PROGRAM_NAME) .args(["check", "."]) + .env_clear() .envs(env) .output() .map_err(|io_error| match io_error.kind() { diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 691fe59..c5bb65c 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -53,8 +53,6 @@ impl Layer for PipDependenciesLayer<'_> { log_info("Running pip install"); // TODO: Explain why we're using user install - // TODO: Mention that we're intentionally not using env_clear() otherwise - // PATH won't be set, and Pip won't be able to find things like Git. utils::run_command( Command::new("pip") .args([ @@ -78,6 +76,7 @@ impl Layer for PipDependenciesLayer<'_> { "--src", &src_dir.to_string_lossy(), ]) + .env_clear() .envs(&env) // TODO: Explain why we're setting this // Using 1980-01-01T00:00:01Z to avoid: diff --git a/src/layers/python.rs b/src/layers/python.rs index 88c0edf..8dc4c40 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -116,6 +116,7 @@ impl Layer for PythonLayer<'_> { format!("setuptools=={SETUPTOOLS_VERSION}").as_str(), format!("wheel=={WHEEL_VERSION}").as_str(), ]) + .env_clear() .envs(&env) // TODO: Explain why we're setting this // Using 1980-01-01T00:00:01Z to avoid: From 49fdb6063bc41a3d22a992a7b843c9d06a88c051 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 8 Feb 2023 16:41:47 +0000 Subject: [PATCH 42/71] Refactor pip dependencies layer env handling + add a test --- src/layers/pip_dependencies.rs | 59 ++++++++++++++++++++++++++++------ src/layers/python.rs | 16 ++------- src/utils.rs | 13 ++++++++ 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index c5bb65c..4a8756c 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -37,14 +37,7 @@ impl Layer for PipDependenciesLayer<'_> { _context: &BuildContext, layer_path: &Path, ) -> Result, ::Error> { - // TODO: Explain PYTHONUSERBASE and that it will contain bin/, lib/.../site-packages/ - // etc and so does not need to be nested due to the env/ directory. - let layer_env = LayerEnv::new().chainable_insert( - Scope::All, - ModificationBehavior::Override, - "PYTHONUSERBASE", - layer_path, - ); + let layer_env = generate_layer_env(layer_path); let env = layer_env.apply(Scope::Build, self.base_env); let src_dir = layer_path.join("src"); @@ -52,7 +45,6 @@ impl Layer for PipDependenciesLayer<'_> { log_info("Running pip install"); - // TODO: Explain why we're using user install utils::run_command( Command::new("pip") .args([ @@ -68,6 +60,10 @@ impl Layer for PipDependenciesLayer<'_> { "--no-warn-script-location", "--progress", "off", + // Install dependencies into the user `site-packages` directory (set by `PYTHONUSERBASE`), + // rather than the system `site-packages` directory, since the latter is inside the + // Python runtime layer, and we want to keep the application dependencies in a separate + // layer to the runtime. "--user", "--requirement", "requirements.txt", @@ -93,6 +89,29 @@ impl Layer for PipDependenciesLayer<'_> { } } +/// Environment variables that will be set by this layer. +fn generate_layer_env(layer_path: &Path) -> LayerEnv { + LayerEnv::new() + // `PYTHONUSERBASE` overrides the default user base directory, which is used by Python to + // compute the path of the user `site-packages` directory: + // https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUSERBASE + // + // Setting this: + // - Makes `pip install --user` install the dependencies into the current layer rather + // than the user's home directory (which would be discarded at the end of the build). + // - Allows Python to find the installed packages at import time. + // + // It's fine for this directory to be set to the root of the layer, since all of the files + // created by Pip will be nested inside subdirectories (such as `bin/` or `lib/`), and so + // won't conflict with the CNB layer metadata related files generated by libcnb.rs. + .chainable_insert( + Scope::All, + ModificationBehavior::Override, + "PYTHONUSERBASE", + layer_path, + ) +} + /// Errors that can occur when installing the project's dependencies into a layer using Pip. #[derive(Debug)] pub(crate) enum PipDependenciesLayerError { @@ -105,3 +124,25 @@ impl From for BuildpackError { Self::PipLayer(error) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pip_dependencies_layer_env() { + let mut base_env = Env::new(); + base_env.insert("PYTHONUSERBASE", "this-should-be-overridden"); + + let layer_env = generate_layer_env(Path::new("/layers/dependencies")); + + assert_eq!( + utils::environment_as_sorted_vector(&layer_env.apply(Scope::Build, &base_env)), + vec![("PYTHONUSERBASE", "/layers/dependencies")] + ); + assert_eq!( + utils::environment_as_sorted_vector(&layer_env.apply(Scope::Launch, &base_env)), + vec![("PYTHONUSERBASE", "/layers/dependencies")] + ); + } +} diff --git a/src/layers/python.rs b/src/layers/python.rs index 8dc4c40..254966f 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -299,7 +299,7 @@ mod tests { use super::*; #[test] - fn test_generate_layer_env() { + fn python_layer_env() { let mut base_env = Env::new(); base_env.insert("CPATH", "/base"); base_env.insert("LANG", "this-should-be-overridden"); @@ -317,7 +317,7 @@ mod tests { // Remember to force invalidation of the cached layer if these env vars ever change. assert_eq!( - environment_as_sorted_vector(&layer_env.apply(Scope::Build, &base_env)), + utils::environment_as_sorted_vector(&layer_env.apply(Scope::Build, &base_env)), vec![ ("CPATH", "/layers/python/include/python3.11:/base"), ("LANG", "C.UTF-8"), @@ -326,7 +326,7 @@ mod tests { ] ); assert_eq!( - environment_as_sorted_vector(&layer_env.apply(Scope::Launch, &base_env)), + utils::environment_as_sorted_vector(&layer_env.apply(Scope::Launch, &base_env)), vec![ ("CPATH", "/layers/python/include/python3.11:/base"), ("LANG", "C.UTF-8"), @@ -335,14 +335,4 @@ mod tests { ] ); } - - fn environment_as_sorted_vector(environment: &Env) -> Vec<(&str, &str)> { - let mut result: Vec<(&str, &str)> = environment - .iter() - .map(|(k, v)| (k.to_str().unwrap(), v.to_str().unwrap())) - .collect(); - - result.sort_by_key(|kv| kv.0); - result - } } diff --git a/src/utils.rs b/src/utils.rs index 5ae9882..3408a52 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -92,6 +92,19 @@ pub(crate) enum CommandError { NonZeroExitStatus(ExitStatus), } +/// Convert a [`libcnb::Env`] to a sorted vector of key-value string slice tuples, for easier +/// testing of the environment variables set in the buildpack layers. +#[cfg(test)] +pub(crate) fn environment_as_sorted_vector(environment: &libcnb::Env) -> Vec<(&str, &str)> { + let mut result: Vec<(&str, &str)> = environment + .iter() + .map(|(k, v)| (k.to_str().unwrap(), v.to_str().unwrap())) + .collect(); + + result.sort_by_key(|kv| kv.0); + result +} + #[cfg(test)] mod tests { use super::*; From 2b6bf4203dfcb97a9de4368bb06682b49dea4ed2 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 9 Feb 2023 09:50:29 +0000 Subject: [PATCH 43/71] s/env/command_env/ --- src/functions.rs | 4 ++-- src/layers/pip_dependencies.rs | 10 +++++----- src/layers/python.rs | 8 ++++---- src/main.rs | 14 +++++++------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/functions.rs b/src/functions.rs index 3c84d51..605c78d 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -25,13 +25,13 @@ pub(crate) fn is_function_project(app_dir: &Path) -> Result Result<(), CheckFunctionError> { +pub(crate) fn check_function(command_env: &Env) -> Result<(), CheckFunctionError> { // Not using `utils::run_command` since we want to capture output and only // display it if the check command fails. Command::new(FUNCTION_RUNTIME_PROGRAM_NAME) .args(["check", "."]) .env_clear() - .envs(env) + .envs(command_env) .output() .map_err(|io_error| match io_error.kind() { io::ErrorKind::NotFound => CheckFunctionError::ProgramNotFound, diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 4a8756c..24f896d 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -14,7 +14,7 @@ use std::{fs, io}; /// Layer containing the application's Python dependencies, installed using Pip. pub(crate) struct PipDependenciesLayer<'a> { /// Environment variables inherited from earlier buildpack steps. - pub base_env: &'a Env, + pub command_env: &'a Env, /// The path to the Pip cache directory, which is stored in another layer since it isn't needed at runtime. pub pip_cache_dir: PathBuf, } @@ -38,7 +38,7 @@ impl Layer for PipDependenciesLayer<'_> { layer_path: &Path, ) -> Result, ::Error> { let layer_env = generate_layer_env(layer_path); - let env = layer_env.apply(Scope::Build, self.base_env); + let command_env = layer_env.apply(Scope::Build, self.command_env); let src_dir = layer_path.join("src"); fs::create_dir(&src_dir).map_err(PipDependenciesLayerError::CreateSrcDirIo)?; @@ -67,13 +67,13 @@ impl Layer for PipDependenciesLayer<'_> { "--user", "--requirement", "requirements.txt", - // Make pip clone any VCS repositories installed in editable mode into a directory in this layer, - // rather than the default of the current working directory (the app dir). + // Make pip clone any VCS repositories installed in editable mode into a directory in this + // layer, rather than the default of the current working directory (the app dir). "--src", &src_dir.to_string_lossy(), ]) .env_clear() - .envs(&env) + .envs(&command_env) // TODO: Explain why we're setting this // Using 1980-01-01T00:00:01Z to avoid: // ValueError: ZIP does not support timestamps before 1980 diff --git a/src/layers/python.rs b/src/layers/python.rs index 254966f..38e9c06 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -22,7 +22,7 @@ const WHEEL_VERSION: &str = "0.38.4"; /// Layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`. pub(crate) struct PythonLayer<'a> { /// Environment variables inherited from earlier buildpack steps. - pub base_env: &'a Env, + pub command_env: &'a Env, /// The Python version that will be installed. pub python_version: &'a PythonVersion, } @@ -78,14 +78,14 @@ impl Layer for PythonLayer<'_> { log_info("Python installation successful"); let layer_env = generate_layer_env(layer_path, self.python_version); - let mut env = layer_env.apply(Scope::Build, self.base_env); + let mut command_env = layer_env.apply(Scope::Build, self.command_env); // The Python binaries are built using `--shared`, and since they're being installed at a // different location from their original `--prefix`, they need `LD_LIBRARY_PATH` to be set // in order to find `libpython3`. Whilst `LD_LIBRARY_PATH` will be automatically set later by // lifecycle/libcnb, it's not set by libcnb until this `Layer` has ended, and so we have to // explicitly set it for the Python invocations within this layer. - env.insert("LD_LIBRARY_PATH", layer_path.join("lib")); + command_env.insert("LD_LIBRARY_PATH", layer_path.join("lib")); log_header("Installing Pip"); log_info(format!("Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION}")); @@ -117,7 +117,7 @@ impl Layer for PythonLayer<'_> { format!("wheel=={WHEEL_VERSION}").as_str(), ]) .env_clear() - .envs(&env) + .envs(&command_env) // TODO: Explain why we're setting this // Using 1980-01-01T00:00:01Z to avoid: // ValueError: ZIP does not support timestamps before 1980 diff --git a/src/main.rs b/src/main.rs index 17b3f0d..d57f574 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,17 +65,17 @@ impl Buildpack for PythonBuildpack { // We inherit the current process's env vars, since we want `PATH` and `HOME` to be set // so that later commands can find tools like Git in the stack image. Any user-provided // env vars will still be excluded, due to the use of `clear-env` in `buildpack.toml`. - let mut env = Env::from_current(); + let mut command_env = Env::from_current(); - // Create the layer containing the Python runtime and the packages `pip`, `setuptools` and `wheel`. + // Create the layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`. let python_layer = context.handle_layer( layer_name!("python"), PythonLayer { - base_env: &env, + command_env: &command_env, python_version: &python_version, }, )?; - env = python_layer.env.apply(Scope::Build, &env); + command_env = python_layer.env.apply(Scope::Build, &command_env); // Create the layers for the application dependencies and package manager cache. // In the future support will be added for package managers other than pip. @@ -91,18 +91,18 @@ impl Buildpack for PythonBuildpack { let pip_layer = context.handle_layer( layer_name!("dependencies"), PipDependenciesLayer { - base_env: &env, + command_env: &command_env, pip_cache_dir: pip_cache_layer.path, }, )?; pip_layer.env } }; - env = dependencies_layer_env.apply(Scope::Build, &env); + command_env = dependencies_layer_env.apply(Scope::Build, &command_env); if is_function { log_header("Validating Salesforce Function"); - functions::check_function(&env).map_err(BuildpackError::CheckFunction)?; + functions::check_function(&command_env).map_err(BuildpackError::CheckFunction)?; log_info("Function passed validation."); BuildResultBuilder::new() From 4ad4245bae866ef3ca12519b3409defda8dac361 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:47:45 +0000 Subject: [PATCH 44/71] More rustdocs/comments --- src/layers/pip_cache.rs | 1 + src/layers/pip_dependencies.rs | 30 +++++++++++++++++++++++++++--- src/main.rs | 7 +++++++ src/python_version.rs | 3 ++- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/layers/pip_cache.rs b/src/layers/pip_cache.rs index 55993a5..86d27d2 100644 --- a/src/layers/pip_cache.rs +++ b/src/layers/pip_cache.rs @@ -9,6 +9,7 @@ use libherokubuildpack::log::log_info; use serde::{Deserialize, Serialize}; use std::path::Path; +/// Layer containing Pip's cache of HTTP requests/downloads and built package wheels. pub(crate) struct PipCacheLayer<'a> { pub python_version: &'a PythonVersion, } diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 24f896d..9c59643 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -24,6 +24,23 @@ impl Layer for PipDependenciesLayer<'_> { type Metadata = GenericMetadata; fn types(&self) -> LayerTypes { + // This layer is not cached, since: + // - Pip is a package installer rather than a project/environment manager, and so does + // not deterministically manage installed Python packages. For example, if a package + // entry in a requirements file is later removed, Pip will not uninstall the package. + // In addition, there is no official lockfile support (only partial support via + // third-party requirements file tools), so changes in transitive dependencies add yet + // more opportunity for non-determinism between each install. + // - The Pip HTTP/wheel cache is itself cached in a separate layer, which covers the most + // time consuming part of performing a pip install: downloading the dependencies and then + // generating wheels (for any packages that use compiled components but don't distribute + // pre-built wheels matching the current Python version). + // - The only case where the Pip wheel cache doesn't help, is for projects that use + // hash-checking mode and so are affected by this Pip issue: + // https://github.com/pypa/pip/issues/5037 + // ...however, the limitation should really be fixed upstream, and this mode is rarely + // used in practice, and only by more advanced projects that would actually probably be + // better off using Poetry instead of Pip (once the buildpack supports Poetry). LayerTypes { build: true, cache: false, @@ -31,7 +48,6 @@ impl Layer for PipDependenciesLayer<'_> { } } - // TODO: Explain why we're not caching here. fn create( &self, _context: &BuildContext, @@ -40,6 +56,14 @@ impl Layer for PipDependenciesLayer<'_> { let layer_env = generate_layer_env(layer_path); let command_env = layer_env.apply(Scope::Build, self.command_env); + // When Pip installs dependencies from a VCS URL it has to clone the repository in order + // to install it. In standard installation mode the clone is made to a temporary directory + // and then deleted, however, when packages are installed in editable mode Pip must keep + // the repository around, since the directory is added to the Python path directly (via + // the `.pth` file created in `site-packages`). By default Pip will store the repository + // in the current working directory (the app dir), however, we would prefer it to be stored + // in the dependencies layer instead for consistency. (Plus if this layer were ever cached, + // storing the repository in the app dir would break on repeat-builds). let src_dir = layer_path.join("src"); fs::create_dir(&src_dir).map_err(PipDependenciesLayerError::CreateSrcDirIo)?; @@ -67,8 +91,8 @@ impl Layer for PipDependenciesLayer<'_> { "--user", "--requirement", "requirements.txt", - // Make pip clone any VCS repositories installed in editable mode into a directory in this - // layer, rather than the default of the current working directory (the app dir). + // Clone any VCS repositories installed in editable mode into the directory created + // above, rather than the default of the current working directory (the app dir). "--src", &src_dir.to_string_lossy(), ]) diff --git a/src/main.rs b/src/main.rs index d57f574..c34f787 100644 --- a/src/main.rs +++ b/src/main.rs @@ -120,12 +120,19 @@ impl Buildpack for PythonBuildpack { #[derive(Debug)] pub(crate) enum BuildpackError { + /// Errors running the `sf-functions-python check` command. CheckFunction(CheckFunctionError), + /// IO errors when performing buildpack detection. DetectIo(io::Error), + /// Errors determining which Python package manager to use for a project. DeterminePackageManager(DeterminePackageManagerError), + /// Errors installing the project's dependencies into a layer using Pip. PipLayer(PipDependenciesLayerError), + /// Errors reading and parsing a `project.toml` file. ProjectDescriptor(ReadProjectDescriptorError), + /// Errors installing Python and required packaging tools into a layer. PythonLayer(PythonLayerError), + /// Errors determining which Python version to use for a project. PythonVersion(PythonVersionError), } diff --git a/src/python_version.rs b/src/python_version.rs index 1f1b95a..7f09549 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -60,9 +60,10 @@ pub(crate) fn determine_python_version( Ok(DEFAULT_PYTHON_VERSION) } -/// Errors that can occur when determining which Python package manager to use for a project. +/// Errors that can occur when determining which Python version to use for a project. #[derive(Debug)] pub(crate) enum PythonVersionError { + /// Errors reading and parsing a `runtime.txt` file. RuntimeTxt(ReadRuntimeTxtError), } From 09083daa2b5b7424404fcb2067eb9dde6abb0450 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 9 Feb 2023 12:07:57 +0000 Subject: [PATCH 45/71] s/functions/salesforce_functions/ --- src/errors.rs | 12 ++++++------ src/main.rs | 13 +++++++------ src/{functions.rs => salesforce_functions.rs} | 10 +++++----- 3 files changed, 18 insertions(+), 17 deletions(-) rename src/{functions.rs => salesforce_functions.rs} (93%) diff --git a/src/errors.rs b/src/errors.rs index dbc801b..1451ae4 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,10 +1,10 @@ -use crate::functions::{CheckFunctionError, FUNCTION_RUNTIME_PROGRAM_NAME}; use crate::layers::pip_dependencies::PipDependenciesLayerError; use crate::layers::python::PythonLayerError; use crate::package_manager::DeterminePackageManagerError; use crate::project_descriptor::ReadProjectDescriptorError; use crate::python_version::{PythonVersion, PythonVersionError, DEFAULT_PYTHON_VERSION}; use crate::runtime_txt::{ParseRuntimeTxtError, ReadRuntimeTxtError}; +use crate::salesforce_functions::{CheckSalesforceFunctionError, FUNCTION_RUNTIME_PROGRAM_NAME}; use crate::utils::{CommandError, DownloadUnpackArchiveError}; use crate::BuildpackError; use indoc::{formatdoc, indoc}; @@ -42,7 +42,7 @@ pub(crate) fn on_error(error: libcnb::Error) { fn on_buildpack_error(error: BuildpackError) { match error { - BuildpackError::CheckFunction(error) => on_check_function_error(error), + BuildpackError::CheckSalesforceFunction(error) => on_check_salesforce_function_error(error), BuildpackError::DetectIo(io_error) => log_io_error( "Unable to complete buildpack detection", "determining if the Python buildpack should be run for this application", @@ -239,14 +239,14 @@ fn on_pip_dependencies_layer_error(error: PipDependenciesLayerError) { }; } -fn on_check_function_error(error: CheckFunctionError) { +fn on_check_salesforce_function_error(error: CheckSalesforceFunctionError) { match error { - CheckFunctionError::Io(io_error) => log_io_error( + CheckSalesforceFunctionError::Io(io_error) => log_io_error( "Unable to run the Salesforce Functions self-check command", &format!("running the '{FUNCTION_RUNTIME_PROGRAM_NAME} check' command"), &io_error, ), - CheckFunctionError::NonZeroExitStatus(output) => log_error( + CheckSalesforceFunctionError::NonZeroExitStatus(output) => log_error( "The Salesforce Functions self-check failed", formatdoc! {" The '{FUNCTION_RUNTIME_PROGRAM_NAME} check' command failed ({exit_status}), indicating @@ -259,7 +259,7 @@ fn on_check_function_error(error: CheckFunctionError) { stderr = String::from_utf8_lossy(&output.stderr), }, ), - CheckFunctionError::ProgramNotFound => log_error( + CheckSalesforceFunctionError::ProgramNotFound => log_error( "The Salesforce Functions package is not installed", formatdoc! {" The '{FUNCTION_RUNTIME_PROGRAM_NAME}' program that is required for Python Salesforce diff --git a/src/main.rs b/src/main.rs index c34f787..853072f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,21 +6,21 @@ #![allow(clippy::result_large_err)] mod errors; -mod functions; mod layers; mod package_manager; mod project_descriptor; mod python_version; mod runtime_txt; +mod salesforce_functions; mod utils; -use crate::functions::CheckFunctionError; use crate::layers::pip_cache::PipCacheLayer; use crate::layers::pip_dependencies::{PipDependenciesLayer, PipDependenciesLayerError}; use crate::layers::python::{PythonLayer, PythonLayerError}; use crate::package_manager::{DeterminePackageManagerError, PackageManager}; use crate::project_descriptor::ReadProjectDescriptorError; use crate::python_version::PythonVersionError; +use crate::salesforce_functions::CheckSalesforceFunctionError; use libcnb::build::{BuildContext, BuildResult, BuildResultBuilder}; use libcnb::data::layer_name; use libcnb::detect::{DetectContext, DetectResult, DetectResultBuilder}; @@ -53,7 +53,7 @@ impl Buildpack for PythonBuildpack { fn build(&self, context: BuildContext) -> libcnb::Result { // We perform all project analysis up front, so the build can fail early if the config is invalid. // TODO: Add a "Build config" header and list all config in one place? - let is_function = functions::is_function_project(&context.app_dir) + let is_function = salesforce_functions::is_function_project(&context.app_dir) .map_err(BuildpackError::ProjectDescriptor)?; let package_manager = package_manager::determine_package_manager(&context.app_dir) .map_err(BuildpackError::DeterminePackageManager)?; @@ -102,11 +102,12 @@ impl Buildpack for PythonBuildpack { if is_function { log_header("Validating Salesforce Function"); - functions::check_function(&command_env).map_err(BuildpackError::CheckFunction)?; + salesforce_functions::check_function(&command_env) + .map_err(BuildpackError::CheckSalesforceFunction)?; log_info("Function passed validation."); BuildResultBuilder::new() - .launch(functions::launch_config()) + .launch(salesforce_functions::launch_config()) .build() } else { BuildResultBuilder::new().build() @@ -121,7 +122,7 @@ impl Buildpack for PythonBuildpack { #[derive(Debug)] pub(crate) enum BuildpackError { /// Errors running the `sf-functions-python check` command. - CheckFunction(CheckFunctionError), + CheckSalesforceFunction(CheckSalesforceFunctionError), /// IO errors when performing buildpack detection. DetectIo(io::Error), /// Errors determining which Python package manager to use for a project. diff --git a/src/functions.rs b/src/salesforce_functions.rs similarity index 93% rename from src/functions.rs rename to src/salesforce_functions.rs index 605c78d..71bad71 100644 --- a/src/functions.rs +++ b/src/salesforce_functions.rs @@ -25,7 +25,7 @@ pub(crate) fn is_function_project(app_dir: &Path) -> Result Result<(), CheckFunctionError> { +pub(crate) fn check_function(command_env: &Env) -> Result<(), CheckSalesforceFunctionError> { // Not using `utils::run_command` since we want to capture output and only // display it if the check command fails. Command::new(FUNCTION_RUNTIME_PROGRAM_NAME) @@ -34,14 +34,14 @@ pub(crate) fn check_function(command_env: &Env) -> Result<(), CheckFunctionError .envs(command_env) .output() .map_err(|io_error| match io_error.kind() { - io::ErrorKind::NotFound => CheckFunctionError::ProgramNotFound, - _ => CheckFunctionError::Io(io_error), + io::ErrorKind::NotFound => CheckSalesforceFunctionError::ProgramNotFound, + _ => CheckSalesforceFunctionError::Io(io_error), }) .and_then(|output| { if output.status.success() { Ok(()) } else { - Err(CheckFunctionError::NonZeroExitStatus(output)) + Err(CheckSalesforceFunctionError::NonZeroExitStatus(output)) } }) } @@ -81,7 +81,7 @@ pub(crate) fn launch_config() -> Launch { /// Errors that can occur when running the `sf-functions-python check` command. #[derive(Debug)] -pub(crate) enum CheckFunctionError { +pub(crate) enum CheckSalesforceFunctionError { Io(io::Error), NonZeroExitStatus(Output), ProgramNotFound, From ea35f113c4ae964418653caab66bfb3d8b44c5c7 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 9 Feb 2023 12:16:27 +0000 Subject: [PATCH 46/71] Improve naming of error enums and their variants --- src/errors.rs | 20 ++++++++++---------- src/layers/pip_dependencies.rs | 2 +- src/layers/python.rs | 8 ++++---- src/main.rs | 6 +++--- src/project_descriptor.rs | 14 +++++++------- src/python_version.rs | 6 +++--- src/runtime_txt.rs | 12 ++++++------ src/salesforce_functions.rs | 6 +++--- 8 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 1451ae4..9c4e52a 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,9 +1,9 @@ use crate::layers::pip_dependencies::PipDependenciesLayerError; use crate::layers::python::PythonLayerError; use crate::package_manager::DeterminePackageManagerError; -use crate::project_descriptor::ReadProjectDescriptorError; +use crate::project_descriptor::ProjectDescriptorError; use crate::python_version::{PythonVersion, PythonVersionError, DEFAULT_PYTHON_VERSION}; -use crate::runtime_txt::{ParseRuntimeTxtError, ReadRuntimeTxtError}; +use crate::runtime_txt::{ParseRuntimeTxtError, RuntimeTxtError}; use crate::salesforce_functions::{CheckSalesforceFunctionError, FUNCTION_RUNTIME_PROGRAM_NAME}; use crate::utils::{CommandError, DownloadUnpackArchiveError}; use crate::BuildpackError; @@ -49,21 +49,21 @@ fn on_buildpack_error(error: BuildpackError) { &io_error, ), BuildpackError::DeterminePackageManager(error) => on_determine_package_manager_error(error), - BuildpackError::PipLayer(error) => on_pip_dependencies_layer_error(error), + BuildpackError::PipDependenciesLayer(error) => on_pip_dependencies_layer_error(error), BuildpackError::ProjectDescriptor(error) => on_project_descriptor_error(error), BuildpackError::PythonLayer(error) => on_python_layer_error(error), BuildpackError::PythonVersion(error) => on_python_version_error(error), }; } -fn on_project_descriptor_error(error: ReadProjectDescriptorError) { +fn on_project_descriptor_error(error: ProjectDescriptorError) { match error { - ReadProjectDescriptorError::Io(io_error) => log_io_error( + ProjectDescriptorError::Io(io_error) => log_io_error( "Unable to read project.toml", "reading the (optional) project.toml file", &io_error, ), - ReadProjectDescriptorError::Parse(toml_error) => log_error( + ProjectDescriptorError::Parse(toml_error) => log_error( "Invalid project.toml", formatdoc! {" A parsing/validation error error occurred whilst loading the project.toml file. @@ -101,13 +101,13 @@ fn on_determine_package_manager_error(error: DeterminePackageManagerError) { fn on_python_version_error(error: PythonVersionError) { match error { PythonVersionError::RuntimeTxt(error) => match error { - ReadRuntimeTxtError::Io(io_error) => log_io_error( + RuntimeTxtError::Io(io_error) => log_io_error( "Unable to read runtime.txt", "reading the (optional) runtime.txt file", &io_error, ), // TODO: Write the supported Python versions inline, instead of linking out to Dev Center. - ReadRuntimeTxtError::Parse(ParseRuntimeTxtError { cleaned_contents }) => { + RuntimeTxtError::Parse(ParseRuntimeTxtError { cleaned_contents }) => { let PythonVersion { major, minor, @@ -163,7 +163,7 @@ fn on_python_layer_error(error: PythonLayerError) { "}, ), }, - PythonLayerError::DownloadUnpackArchive(error) => match error { + PythonLayerError::DownloadUnpackPythonArchive(error) => match error { DownloadUnpackArchiveError::Io(io_error) => log_io_error( "Unable to unpack the Python archive", "unpacking the downloaded Python runtime archive and writing it to disk", @@ -193,7 +193,7 @@ fn on_python_layer_error(error: PythonLayerError) { ), // This error will change once the Python version is validated against a manifest. // TODO: Write the supported Python versions inline, instead of linking out to Dev Center. - PythonLayerError::PythonVersionNotFound { + PythonLayerError::PythonArchiveNotFound { python_version, stack, } => log_error( diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 9c59643..1fc5854 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -145,7 +145,7 @@ pub(crate) enum PipDependenciesLayerError { impl From for BuildpackError { fn from(error: PipDependenciesLayerError) -> Self { - Self::PipLayer(error) + Self::PipDependenciesLayer(error) } } diff --git a/src/layers/python.rs b/src/layers/python.rs index 38e9c06..6ac01ca 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -67,12 +67,12 @@ impl Layer for PythonLayer<'_> { // TODO: Remove this once the Python version is validated against a manifest (at which // point 404s can be treated as an internal error, instead of user error) DownloadUnpackArchiveError::Request(ureq::Error::Status(404, _)) => { - PythonLayerError::PythonVersionNotFound { + PythonLayerError::PythonArchiveNotFound { stack: context.stack_id.clone(), python_version: self.python_version.clone(), } } - other_error => PythonLayerError::DownloadUnpackArchive(other_error), + other_error => PythonLayerError::DownloadUnpackPythonArchive(other_error), } })?; log_info("Python installation successful"); @@ -279,10 +279,10 @@ fn generate_layer_metadata( #[derive(Debug)] pub(crate) enum PythonLayerError { BootstrapPipCommand(CommandError), - DownloadUnpackArchive(DownloadUnpackArchiveError), + DownloadUnpackPythonArchive(DownloadUnpackArchiveError), LocateBundledPipIo(io::Error), MakeSitePackagesReadOnlyIo(io::Error), - PythonVersionNotFound { + PythonArchiveNotFound { python_version: PythonVersion, stack: StackId, }, diff --git a/src/main.rs b/src/main.rs index 853072f..832b37b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,7 @@ use crate::layers::pip_cache::PipCacheLayer; use crate::layers::pip_dependencies::{PipDependenciesLayer, PipDependenciesLayerError}; use crate::layers::python::{PythonLayer, PythonLayerError}; use crate::package_manager::{DeterminePackageManagerError, PackageManager}; -use crate::project_descriptor::ReadProjectDescriptorError; +use crate::project_descriptor::ProjectDescriptorError; use crate::python_version::PythonVersionError; use crate::salesforce_functions::CheckSalesforceFunctionError; use libcnb::build::{BuildContext, BuildResult, BuildResultBuilder}; @@ -128,9 +128,9 @@ pub(crate) enum BuildpackError { /// Errors determining which Python package manager to use for a project. DeterminePackageManager(DeterminePackageManagerError), /// Errors installing the project's dependencies into a layer using Pip. - PipLayer(PipDependenciesLayerError), + PipDependenciesLayer(PipDependenciesLayerError), /// Errors reading and parsing a `project.toml` file. - ProjectDescriptor(ReadProjectDescriptorError), + ProjectDescriptor(ProjectDescriptorError), /// Errors installing Python and required packaging tools into a layer. PythonLayer(PythonLayerError), /// Errors determining which Python version to use for a project. diff --git a/src/project_descriptor.rs b/src/project_descriptor.rs index 26c04b1..77628b4 100644 --- a/src/project_descriptor.rs +++ b/src/project_descriptor.rs @@ -12,7 +12,7 @@ use std::path::Path; /// or the TOML document does not adhere to the schema. pub(crate) fn read_salesforce_project_type( app_dir: &Path, -) -> Result, ReadProjectDescriptorError> { +) -> Result, ProjectDescriptorError> { read_project_descriptor(app_dir).map(|descriptor| { descriptor .unwrap_or_default() @@ -31,12 +31,12 @@ pub(crate) fn read_salesforce_project_type( /// or the TOML document does not adhere to the schema. fn read_project_descriptor( app_dir: &Path, -) -> Result, ReadProjectDescriptorError> { +) -> Result, ProjectDescriptorError> { let project_descriptor_path = app_dir.join("project.toml"); utils::read_optional_file(&project_descriptor_path) - .map_err(ReadProjectDescriptorError::Io)? - .map(|contents| parse(&contents).map_err(ReadProjectDescriptorError::Parse)) + .map_err(ProjectDescriptorError::Io)? + .map(|contents| parse(&contents).map_err(ProjectDescriptorError::Parse)) .transpose() } @@ -91,7 +91,7 @@ pub(crate) enum SalesforceProjectType { /// Errors that can occur when reading and parsing a `project.toml` file. #[derive(Debug)] -pub(crate) enum ReadProjectDescriptorError { +pub(crate) enum ProjectDescriptorError { Io(io::Error), Parse(toml::de::Error), } @@ -239,7 +239,7 @@ mod tests { assert!(matches!( read_project_descriptor(app_dir).unwrap_err(), - ReadProjectDescriptorError::Parse(_) + ProjectDescriptorError::Parse(_) )); } @@ -273,7 +273,7 @@ mod tests { assert!(matches!( read_salesforce_project_type(app_dir).unwrap_err(), - ReadProjectDescriptorError::Parse(_) + ProjectDescriptorError::Parse(_) )); } } diff --git a/src/python_version.rs b/src/python_version.rs index 7f09549..6f7ed87 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -1,4 +1,4 @@ -use crate::runtime_txt::{self, ReadRuntimeTxtError}; +use crate::runtime_txt::{self, RuntimeTxtError}; use indoc::formatdoc; use libherokubuildpack::log::log_info; use std::fmt::{self, Display}; @@ -64,7 +64,7 @@ pub(crate) fn determine_python_version( #[derive(Debug)] pub(crate) enum PythonVersionError { /// Errors reading and parsing a `runtime.txt` file. - RuntimeTxt(ReadRuntimeTxtError), + RuntimeTxt(RuntimeTxtError), } #[cfg(test)] @@ -93,7 +93,7 @@ mod tests { "tests/fixtures/runtime_txt_python_version_invalid" )) .unwrap_err(), - PythonVersionError::RuntimeTxt(ReadRuntimeTxtError::Parse(_)) + PythonVersionError::RuntimeTxt(RuntimeTxtError::Parse(_)) )); } diff --git a/src/runtime_txt.rs b/src/runtime_txt.rs index c423905..bfe9ccb 100644 --- a/src/runtime_txt.rs +++ b/src/runtime_txt.rs @@ -8,12 +8,12 @@ use std::path::Path; /// /// Returns `Ok(None)` if the file does not exist, but returns the error for all other /// forms of IO or parsing errors. -pub(crate) fn read_version(app_dir: &Path) -> Result, ReadRuntimeTxtError> { +pub(crate) fn read_version(app_dir: &Path) -> Result, RuntimeTxtError> { let runtime_txt_path = app_dir.join("runtime.txt"); utils::read_optional_file(&runtime_txt_path) - .map_err(ReadRuntimeTxtError::Io)? - .map(|contents| parse(&contents).map_err(ReadRuntimeTxtError::Parse)) + .map_err(RuntimeTxtError::Io)? + .map(|contents| parse(&contents).map_err(RuntimeTxtError::Parse)) .transpose() } @@ -51,7 +51,7 @@ fn parse(contents: &str) -> Result { /// Errors that can occur when reading and parsing a `runtime.txt` file. #[derive(Debug)] -pub(crate) enum ReadRuntimeTxtError { +pub(crate) enum RuntimeTxtError { Io(io::Error), Parse(ParseRuntimeTxtError), } @@ -219,7 +219,7 @@ mod tests { fn read_version_io_error() { assert!(matches!( read_version(Path::new("tests/fixtures/empty/.gitkeep")).unwrap_err(), - ReadRuntimeTxtError::Io(_) + RuntimeTxtError::Io(_) )); } @@ -230,7 +230,7 @@ mod tests { "tests/fixtures/runtime_txt_python_version_invalid" )) .unwrap_err(), - ReadRuntimeTxtError::Parse(_) + RuntimeTxtError::Parse(_) )); } } diff --git a/src/salesforce_functions.rs b/src/salesforce_functions.rs index 71bad71..99493fa 100644 --- a/src/salesforce_functions.rs +++ b/src/salesforce_functions.rs @@ -1,4 +1,4 @@ -use crate::project_descriptor::{self, ReadProjectDescriptorError, SalesforceProjectType}; +use crate::project_descriptor::{self, ProjectDescriptorError, SalesforceProjectType}; use libcnb::data::launch::{Launch, LaunchBuilder, ProcessBuilder}; use libcnb::data::process_type; use libcnb::Env; @@ -19,7 +19,7 @@ pub(crate) const FUNCTION_RUNTIME_PROGRAM_NAME: &str = "sf-functions-python"; /// /// However, an error will be returned if any other IO error occurred, if the `project.toml` file /// is not valid TOML, or the TOML document does not adhere to the schema. -pub(crate) fn is_function_project(app_dir: &Path) -> Result { +pub(crate) fn is_function_project(app_dir: &Path) -> Result { project_descriptor::read_salesforce_project_type(app_dir) .map(|project_type| project_type == Some(SalesforceProjectType::Function)) } @@ -112,7 +112,7 @@ mod tests { fn is_function_project_invalid_project_toml() { assert!(matches!( is_function_project(Path::new("tests/fixtures/project_toml_invalid")).unwrap_err(), - ReadProjectDescriptorError::Parse(_) + ProjectDescriptorError::Parse(_) )); } } From 401984a5750ebe2ac37bc9b0a1ad052cb689f8ec Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 9 Feb 2023 15:59:41 +0000 Subject: [PATCH 47/71] Clean up SOURCE_DATE_EPOCH usages --- src/layers/pip_dependencies.rs | 26 +++++---- src/layers/python.rs | 97 ++++++++++++++++++++++++++++++++-- 2 files changed, 108 insertions(+), 15 deletions(-) diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 1fc5854..78ba0c9 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -39,8 +39,10 @@ impl Layer for PipDependenciesLayer<'_> { // hash-checking mode and so are affected by this Pip issue: // https://github.com/pypa/pip/issues/5037 // ...however, the limitation should really be fixed upstream, and this mode is rarely - // used in practice, and only by more advanced projects that would actually probably be - // better off using Poetry instead of Pip (once the buildpack supports Poetry). + // used in practice. + // + // Longer term, the best option for projects that want no-op deterministic installs will + // be to use Poetry instead of Pip (once the buildpack supports Poetry). LayerTypes { build: true, cache: false, @@ -85,9 +87,17 @@ impl Layer for PipDependenciesLayer<'_> { "--progress", "off", // Install dependencies into the user `site-packages` directory (set by `PYTHONUSERBASE`), - // rather than the system `site-packages` directory, since the latter is inside the - // Python runtime layer, and we want to keep the application dependencies in a separate - // layer to the runtime. + // rather than the system `site-packages` directory (since we want to keep dependencies in + // a separate layer to the Python runtime). + // + // Another option is to install into an arbitrary directory using Pip's `--target` option + // combined with adding that directory to `PYTHONPATH`, however: + // - Using `--target` causes a number of issues with Pip, eg: + // https://github.com/pypa/pip/issues/8799 + // - Directories added to `PYTHONPATH` take precedence over the Python stdlib (unlike + // the system or user site-packages directories), and so can result in hard to debug + // stdlib shadowing problems that users won't encounter locally (for example if one + // of the app's transitive dependencies is an outdated stdlib backport package). "--user", "--requirement", "requirements.txt", @@ -97,11 +107,7 @@ impl Layer for PipDependenciesLayer<'_> { &src_dir.to_string_lossy(), ]) .env_clear() - .envs(&command_env) - // TODO: Explain why we're setting this - // Using 1980-01-01T00:00:01Z to avoid: - // ValueError: ZIP does not support timestamps before 1980 - .env("SOURCE_DATE_EPOCH", "315532800"), + .envs(&command_env), ) .map_err(PipDependenciesLayerError::PipInstallCommand)?; diff --git a/src/layers/python.rs b/src/layers/python.rs index 6ac01ca..ac39c94 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -117,11 +117,7 @@ impl Layer for PythonLayer<'_> { format!("wheel=={WHEEL_VERSION}").as_str(), ]) .env_clear() - .envs(&command_env) - // TODO: Explain why we're setting this - // Using 1980-01-01T00:00:01Z to avoid: - // ValueError: ZIP does not support timestamps before 1980 - .env("SOURCE_DATE_EPOCH", "315532800"), + .envs(&command_env), ) .map_err(PythonLayerError::BootstrapPipCommand)?; @@ -235,6 +231,61 @@ fn generate_layer_env(layer_path: &Path, python_version: &PythonVersion) -> Laye "PYTHONUNBUFFERED", "1", ) + // By default, when Python creates cached bytecode files (`.pyc` files) it embeds the + // `.py` source file's last-modified time in the `.pyc` file, so it can later be used + // to determine whether the cached bytecode file needs regenerating. + // + // This causes the `.pyc` file contents (and thus layer SHA256) to be non-deterministic in + // cases where the `.py` file's last-modified time can vary (such as files installed by Pip, + // since it doesn't preserve the last modified time of the original downloaded package). + // + // In addition, as part of generating the OCI image, lifecycle resets the timestamps on all + // files to a fixed value in order to improve the determinism of builds: + // https://buildpacks.io/docs/features/reproducibility/#consequences-and-caveats + // + // At runtime, this then means the timestamps embedded in the `.pyc` files no longer match + // the timestamps of the original `.py` files, causing Python to have to regenerate the + // bytecode, and so losing any benefit of having kept the `.pyc` files in the image. + // + // One option to solve all of the above, would be to delete the `.pyc` files from the image + // at the end of the buildpack's build phase, however: + // - This means they need to be regenerated at app start boot, slowing boot times. + // (For a simple Django project on a Perf-M, boot time increases from ~0.5s to ~1.5s.) + // - If any other later buildpack runs any of the Python files added by this buildpack, then + // the timestamp based `.pyc` files will be created again, re-introducing non-determinism. + // + // Instead, we use the hash-based cache files mode added in Python 3.7+, which embeds a hash + // of the original `.py` file in the `.pyc` file instead of the timestamp: + // https://docs.python.org/3.11/reference/import.html#pyc-invalidation + // https://peps.python.org/pep-0552/ + // + // This mode can be enabled by passing `--invalidation-mode checked-hash` to `compileall`, + // or via the `SOURCE_DATE_EPOCH` env var: + // https://docs.python.org/3.11/library/compileall.html#cmdoption-compileall-invalidation-mode + // + // Note: Both the CLI args and the env var only apply to usages of `compileall` or `py_compile`, + // and not `.pyc` generation as part of Python importing a file during normal operation. + // + // We use the env var, since: + // - Pip calls `compileall` itself after installing packages, and doesn't allow us to + // customise the options passed to it, which would mean we'd have to pass `--no-compile` + // to Pip followed by running `compileall` manually ourselves, meaning more complexity + // every time we (or a later buildpack) use `pip install`. + // - When we add support for Poetry, we'll have to use an env var regardless, since Poetry + // doesn't allow customising the options passed to its internal Pip invocations, so we'd + // have no way of passing `--no-compile` to Pip. + .chainable_insert( + Scope::Build, + ModificationBehavior::Default, + "SOURCE_DATE_EPOCH", + // Whilst `compileall` doesn't use the value of `SOURCE_DATE_EPOCH` (only whether it is + // set or not), the value ends up being used when wheel archives are generated during + // the pip install. As such, we cannot use a zero value since the ZIP file format doesn't + // support dates before 1980. Instead, we use a value equivalent to `1980-01-01T00:00:01Z`, + // for parity with that used by lifecycle: + // https://github.com/buildpacks/lifecycle/blob/v0.15.3/archive/writer.go#L12 + "315532801", + ) } /// The path to the Pip module bundled in Python's standard library. @@ -300,11 +351,45 @@ mod tests { #[test] fn python_layer_env() { + let layer_env = generate_layer_env( + Path::new("/layers/python"), + &PythonVersion { + major: 3, + minor: 9, + patch: 0, + }, + ); + + // Remember to force invalidation of the cached layer if these env vars ever change. + assert_eq!( + utils::environment_as_sorted_vector(&layer_env.apply_to_empty(Scope::Build)), + vec![ + ("CPATH", "/layers/python/include/python3.9"), + ("LANG", "C.UTF-8"), + ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig"), + ("PYTHONUNBUFFERED", "1"), + ("SOURCE_DATE_EPOCH", "315532801"), + ] + ); + assert_eq!( + utils::environment_as_sorted_vector(&layer_env.apply_to_empty(Scope::Launch)), + vec![ + ("CPATH", "/layers/python/include/python3.9"), + ("LANG", "C.UTF-8"), + ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig"), + ("PYTHONUNBUFFERED", "1"), + ] + ); + } + + #[test] + fn python_layer_env_with_existing_env() { let mut base_env = Env::new(); base_env.insert("CPATH", "/base"); base_env.insert("LANG", "this-should-be-overridden"); base_env.insert("PKG_CONFIG_PATH", "/base"); base_env.insert("PYTHONUNBUFFERED", "this-should-be-overridden"); + base_env.insert("SOURCE_DATE_EPOCH", "this-should-be-preserved"); let layer_env = generate_layer_env( Path::new("/layers/python"), @@ -323,6 +408,7 @@ mod tests { ("LANG", "C.UTF-8"), ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig:/base"), ("PYTHONUNBUFFERED", "1"), + ("SOURCE_DATE_EPOCH", "this-should-be-preserved"), ] ); assert_eq!( @@ -332,6 +418,7 @@ mod tests { ("LANG", "C.UTF-8"), ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig:/base"), ("PYTHONUNBUFFERED", "1"), + ("SOURCE_DATE_EPOCH", "this-should-be-preserved"), ] ); } From 7e5873b0b7abd63ed72441aa05d660fcb62f5de0 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:20:44 +0000 Subject: [PATCH 48/71] Bump default Python to the newly released 3.11.2 --- src/python_version.rs | 2 +- tests/integration.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/python_version.rs b/src/python_version.rs index 6f7ed87..9997e66 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -8,7 +8,7 @@ use std::path::Path; pub(crate) const DEFAULT_PYTHON_VERSION: PythonVersion = PythonVersion { major: 3, minor: 11, - patch: 1, + patch: 2, }; /// Representation of a specific Python `X.Y.Z` version. diff --git a/tests/integration.rs b/tests/integration.rs index de3ff20..d2aadd1 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -47,11 +47,11 @@ fn function_template() { context.pack_stdout, indoc! {" [Determining Python version] - No Python version specified, using the current default of 3.11.1. + No Python version specified, using the current default of 3.11.2. To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes [Installing Python] - Downloading Python 3.11.1 + Downloading Python 3.11.2 Python installation successful [Installing Pip] @@ -120,11 +120,11 @@ fn function_repeat_build() { rebuild_context.pack_stdout, indoc! {" [Determining Python version] - No Python version specified, using the current default of 3.11.1. + No Python version specified, using the current default of 3.11.2. To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes [Installing Python] - Re-using cached Python 3.11.1 + Re-using cached Python 3.11.2 [Installing Pip] Re-using cached pip 23.0, setuptools 67.1.0 and wheel 0.38.4 @@ -165,7 +165,7 @@ fn runtime_txt_python_version_unavailable() { The requested Python version (999.999.999) is not available for this stack ({expected_stack}). Please update the version in 'runtime.txt' to a supported Python version, or else - remove the file to instead use the default version (currently Python 3.11.1). + remove the file to instead use the default version (currently Python 3.11.2). For a list of the supported Python versions, see: https://devcenter.heroku.com/articles/python-support#supported-runtimes @@ -197,11 +197,11 @@ fn runtime_txt_python_version_invalid() { However, the file contents must begin with a 'python-' prefix, followed by the version specified as '..'. Comments are not supported. - For example, to request Python 3.11.1, the correct version format is: - python-3.11.1 + For example, to request Python 3.11.2, the correct version format is: + python-3.11.2 Please update 'runtime.txt' to use the correct version format, or else remove - the file to instead use the default version (currently Python 3.11.1). + the file to instead use the default version (currently Python 3.11.2). For a list of the supported Python versions, see: https://devcenter.heroku.com/articles/python-support#supported-runtimes From f8ccc4211d184b2c308de4849c1a24413e5e6ca6 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 16 Feb 2023 16:47:40 +0000 Subject: [PATCH 49/71] Switch back to using `PIP_DISABLE_PIP_VERSION_CHECK` So that it applies to pip invocations in later buildpacks and when debugging at runtime. --- src/layers/pip_dependencies.rs | 3 --- src/layers/python.rs | 16 +++++++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 78ba0c9..9dc153f 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -77,9 +77,6 @@ impl Layer for PipDependenciesLayer<'_> { "install", "--cache-dir", &self.pip_cache_dir.to_string_lossy(), - // We use a curated Pip version, so skip the update check to speed up Pip invocations, - // reduce build log spam and prevent users from thinking they need to manually upgrade. - "--disable-pip-version-check", "--no-input", // Prevent warning about the `bin/` directory not being on `PATH`, since it // will be added automatically by libcnb/lifecycle later. diff --git a/src/layers/python.rs b/src/layers/python.rs index ac39c94..0fed499 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -107,7 +107,6 @@ impl Layer for PythonLayer<'_> { .args([ &bundled_pip_module_path.to_string_lossy(), "install", - "--disable-pip-version-check", // There is no point using Pip's cache here, since the layer itself will be cached. "--no-cache-dir", "--no-input", @@ -210,6 +209,16 @@ fn generate_layer_env(layer_path: &Path, python_version: &PythonVersion) -> Laye "LANG", "C.UTF-8", ) + // We use a curated Pip version, so disable the update check to speed up Pip invocations, + // reduce build log spam and prevent users from thinking they need to manually upgrade. + // This uses an env var (rather than the `--disable-pip-version-check` arg) so that it also + // takes effect for any pip invocations in later buildpacks or when debugging at runtime. + .chainable_insert( + Scope::All, + ModificationBehavior::Override, + "PIP_DISABLE_PIP_VERSION_CHECK", + "1", + ) // We have to set `PKG_CONFIG_PATH` explicitly, since the automatic path set by lifecycle/libcnb // is `/pkgconfig/`, whereas Python's pkgconfig files are at `/lib/pkgconfig/`. .chainable_insert( @@ -366,6 +375,7 @@ mod tests { vec![ ("CPATH", "/layers/python/include/python3.9"), ("LANG", "C.UTF-8"), + ("PIP_DISABLE_PIP_VERSION_CHECK", "1"), ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig"), ("PYTHONUNBUFFERED", "1"), ("SOURCE_DATE_EPOCH", "315532801"), @@ -376,6 +386,7 @@ mod tests { vec![ ("CPATH", "/layers/python/include/python3.9"), ("LANG", "C.UTF-8"), + ("PIP_DISABLE_PIP_VERSION_CHECK", "1"), ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig"), ("PYTHONUNBUFFERED", "1"), ] @@ -387,6 +398,7 @@ mod tests { let mut base_env = Env::new(); base_env.insert("CPATH", "/base"); base_env.insert("LANG", "this-should-be-overridden"); + base_env.insert("PIP_DISABLE_PIP_VERSION_CHECK", "this-should-be-overridden"); base_env.insert("PKG_CONFIG_PATH", "/base"); base_env.insert("PYTHONUNBUFFERED", "this-should-be-overridden"); base_env.insert("SOURCE_DATE_EPOCH", "this-should-be-preserved"); @@ -406,6 +418,7 @@ mod tests { vec![ ("CPATH", "/layers/python/include/python3.11:/base"), ("LANG", "C.UTF-8"), + ("PIP_DISABLE_PIP_VERSION_CHECK", "1"), ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig:/base"), ("PYTHONUNBUFFERED", "1"), ("SOURCE_DATE_EPOCH", "this-should-be-preserved"), @@ -416,6 +429,7 @@ mod tests { vec![ ("CPATH", "/layers/python/include/python3.11:/base"), ("LANG", "C.UTF-8"), + ("PIP_DISABLE_PIP_VERSION_CHECK", "1"), ("PKG_CONFIG_PATH", "/layers/python/lib/pkgconfig:/base"), ("PYTHONUNBUFFERED", "1"), ("SOURCE_DATE_EPOCH", "this-should-be-preserved"), From 1924ef086df9162b5c75f45517f64ed0756fc6e3 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 16 Feb 2023 21:31:48 +0000 Subject: [PATCH 50/71] Add integration tests --- src/layers/python.rs | 12 +- src/package_manager.rs | 3 +- src/project_descriptor.rs | 4 +- src/python_version.rs | 18 +- src/runtime_txt.rs | 14 +- src/salesforce_functions.rs | 4 +- src/utils.rs | 9 +- .../requirements.txt | 9 + .../pip_invalid_requirement/requirements.txt | 1 + .../requirements.txt | 0 .../pyproject.toml} | 0 tests/fixtures/python_3.10/requirements.txt | 2 + tests/fixtures/python_3.10/runtime.txt | 1 + tests/fixtures/python_3.11/requirements.txt | 2 + tests/fixtures/python_3.11/runtime.txt | 1 + tests/fixtures/python_3.7/requirements.txt | 2 + tests/fixtures/python_3.7/runtime.txt | 1 + tests/fixtures/python_3.8/requirements.txt | 2 + tests/fixtures/python_3.8/runtime.txt | 1 + tests/fixtures/python_3.9/requirements.txt | 2 + tests/fixtures/python_3.9/runtime.txt | 1 + .../requirements.txt | 2 + .../requirements.txt | 0 .../runtime.txt | 0 .../requirements.txt | 0 .../runtime.txt | 0 .../runtime_txt_python_3.10/runtime.txt | 1 - .../main.py | 0 .../project.toml | 0 .../requirements.txt | 0 .../main.py | 0 .../project.toml | 0 .../requirements.txt | 0 .../README.md | 0 .../main.py | 0 .../payload.json | 0 .../project.toml | 0 .../requirements.txt | 0 tests/integration.rs | 644 ++++++++++++++---- 39 files changed, 560 insertions(+), 176 deletions(-) create mode 100644 tests/fixtures/pip_editable_git_compiled/requirements.txt create mode 100644 tests/fixtures/pip_invalid_requirement/requirements.txt rename tests/fixtures/{default => project_toml_invalid}/requirements.txt (100%) rename tests/fixtures/{runtime_txt_python_version_invalid/requirements.txt => pyproject_toml_only/pyproject.toml} (100%) create mode 100644 tests/fixtures/python_3.10/requirements.txt create mode 100644 tests/fixtures/python_3.10/runtime.txt create mode 100644 tests/fixtures/python_3.11/requirements.txt create mode 100644 tests/fixtures/python_3.11/runtime.txt create mode 100644 tests/fixtures/python_3.7/requirements.txt create mode 100644 tests/fixtures/python_3.7/runtime.txt create mode 100644 tests/fixtures/python_3.8/requirements.txt create mode 100644 tests/fixtures/python_3.8/runtime.txt create mode 100644 tests/fixtures/python_3.9/requirements.txt create mode 100644 tests/fixtures/python_3.9/runtime.txt create mode 100644 tests/fixtures/python_version_unspecified/requirements.txt rename tests/fixtures/{runtime_txt_python_version_unavailable => runtime_txt_invalid_version}/requirements.txt (100%) rename tests/fixtures/{runtime_txt_python_version_invalid => runtime_txt_invalid_version}/runtime.txt (100%) create mode 100644 tests/fixtures/runtime_txt_non_existent_version/requirements.txt rename tests/fixtures/{runtime_txt_python_version_unavailable => runtime_txt_non_existent_version}/runtime.txt (100%) delete mode 100644 tests/fixtures/runtime_txt_python_3.10/runtime.txt rename tests/fixtures/{function_fails_self_check => salesforce_function_fails_self_check}/main.py (100%) rename tests/fixtures/{function_fails_self_check => salesforce_function_fails_self_check}/project.toml (100%) rename tests/fixtures/{function_fails_self_check => salesforce_function_fails_self_check}/requirements.txt (100%) rename tests/fixtures/{function_missing_functions_package => salesforce_function_missing_package}/main.py (100%) rename tests/fixtures/{function_missing_functions_package => salesforce_function_missing_package}/project.toml (100%) rename tests/fixtures/{function_missing_functions_package => salesforce_function_missing_package}/requirements.txt (100%) rename tests/fixtures/{function_template => salesforce_function_template}/README.md (100%) rename tests/fixtures/{function_template => salesforce_function_template}/main.py (100%) rename tests/fixtures/{function_template => salesforce_function_template}/payload.json (100%) rename tests/fixtures/{function_template => salesforce_function_template}/project.toml (100%) rename tests/fixtures/{function_template => salesforce_function_template}/requirements.txt (100%) diff --git a/src/layers/python.rs b/src/layers/python.rs index 0fed499..32f0d71 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -15,8 +15,8 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{fs, io}; -const PIP_VERSION: &str = "23.0"; -const SETUPTOOLS_VERSION: &str = "67.1.0"; +const PIP_VERSION: &str = "23.0.1"; +const SETUPTOOLS_VERSION: &str = "67.3.2"; const WHEEL_VERSION: &str = "0.38.4"; /// Layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`. @@ -188,9 +188,11 @@ fn generate_layer_env(layer_path: &Path, python_version: &PythonVersion) -> Laye // // Remember to force invalidation of the cached layer if these env vars ever change. LayerEnv::new() - // We have to set `CPATH` explicitly, since the automatic path set by lifecycle/libcnb is - // `/include/` whereas Python's header files are at `/include/pythonX.Y/` - // (and compilers don't recursively search). + // We have to set `CPATH` explicitly, since: + // - The automatic path set by lifecycle/libcnb is `/include/` whereas Python's + // headers are at `/include/pythonX.Y/` (compilers don't recursively search). + // - Older setuptools cannot find this directory without `CPATH` being set: + // https://github.com/pypa/setuptools/issues/3657 .chainable_insert( Scope::All, ModificationBehavior::Prepend, diff --git a/src/package_manager.rs b/src/package_manager.rs index b75d330..357a186 100644 --- a/src/package_manager.rs +++ b/src/package_manager.rs @@ -46,7 +46,8 @@ mod tests { #[test] fn determine_package_manager_requirements_txt() { assert!(matches!( - determine_package_manager(Path::new("tests/fixtures/default")).unwrap(), + determine_package_manager(Path::new("tests/fixtures/pip_editable_git_compiled")) + .unwrap(), PackageManager::Pip )); } diff --git a/src/project_descriptor.rs b/src/project_descriptor.rs index 77628b4..f5d3f1e 100644 --- a/src/project_descriptor.rs +++ b/src/project_descriptor.rs @@ -219,7 +219,7 @@ mod tests { #[test] fn read_project_descriptor_function() { - let app_dir = Path::new("tests/fixtures/function_template"); + let app_dir = Path::new("tests/fixtures/salesforce_function_template"); assert_eq!( read_project_descriptor(app_dir).unwrap(), @@ -259,7 +259,7 @@ mod tests { #[test] fn get_salesforce_project_type_function() { - let app_dir = Path::new("tests/fixtures/function_template"); + let app_dir = Path::new("tests/fixtures/salesforce_function_template"); assert_eq!( read_salesforce_project_type(app_dir).unwrap(), diff --git a/src/python_version.rs b/src/python_version.rs index 9997e66..8052e22 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -55,7 +55,7 @@ pub(crate) fn determine_python_version( // TODO: Write this content inline, instead of linking out to Dev Center. // Also adjust wording to mention pinning as a use-case, not just using a different version. log_info(formatdoc! {" - No Python version specified, using the current default of {DEFAULT_PYTHON_VERSION}. + No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes"}); Ok(DEFAULT_PYTHON_VERSION) } @@ -74,14 +74,12 @@ mod tests { #[test] fn determine_python_version_runtime_txt_valid() { assert_eq!( - determine_python_version(Path::new("tests/fixtures/runtime_txt_python_3.10")).unwrap(), - PythonVersion::new(3, 10, 9) + determine_python_version(Path::new("tests/fixtures/python_3.9")).unwrap(), + PythonVersion::new(3, 9, 16) ); assert_eq!( - determine_python_version(Path::new( - "tests/fixtures/runtime_txt_python_version_unavailable" - )) - .unwrap(), + determine_python_version(Path::new("tests/fixtures/runtime_txt_non_existent_version")) + .unwrap(), PythonVersion::new(999, 999, 999) ); } @@ -89,10 +87,8 @@ mod tests { #[test] fn determine_python_version_runtime_txt_error() { assert!(matches!( - determine_python_version(Path::new( - "tests/fixtures/runtime_txt_python_version_invalid" - )) - .unwrap_err(), + determine_python_version(Path::new("tests/fixtures/runtime_txt_invalid_version")) + .unwrap_err(), PythonVersionError::RuntimeTxt(RuntimeTxtError::Parse(_)) )); } diff --git a/src/runtime_txt.rs b/src/runtime_txt.rs index bfe9ccb..00ce4b4 100644 --- a/src/runtime_txt.rs +++ b/src/runtime_txt.rs @@ -195,14 +195,11 @@ mod tests { #[test] fn read_version_valid_runtime_txt() { assert_eq!( - read_version(Path::new("tests/fixtures/runtime_txt_python_3.10")).unwrap(), - Some(PythonVersion::new(3, 10, 9)) + read_version(Path::new("tests/fixtures/python_3.9")).unwrap(), + Some(PythonVersion::new(3, 9, 16)) ); assert_eq!( - read_version(Path::new( - "tests/fixtures/runtime_txt_python_version_unavailable" - )) - .unwrap(), + read_version(Path::new("tests/fixtures/runtime_txt_non_existent_version")).unwrap(), Some(PythonVersion::new(999, 999, 999)) ); } @@ -226,10 +223,7 @@ mod tests { #[test] fn read_version_parse_error() { assert!(matches!( - read_version(Path::new( - "tests/fixtures/runtime_txt_python_version_invalid" - )) - .unwrap_err(), + read_version(Path::new("tests/fixtures/runtime_txt_invalid_version")).unwrap_err(), RuntimeTxtError::Parse(_) )); } diff --git a/src/salesforce_functions.rs b/src/salesforce_functions.rs index 99493fa..4f4dbd2 100644 --- a/src/salesforce_functions.rs +++ b/src/salesforce_functions.rs @@ -105,7 +105,9 @@ mod tests { #[test] fn is_function_project_valid_function_project_toml() { - assert!(is_function_project(Path::new("tests/fixtures/function_template")).unwrap()); + assert!( + is_function_project(Path::new("tests/fixtures/salesforce_function_template")).unwrap() + ); } #[test] diff --git a/src/utils.rs b/src/utils.rs index 3408a52..b2b0a54 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -112,7 +112,7 @@ mod tests { #[test] fn is_python_project_valid_project() { - assert!(is_python_project(Path::new("tests/fixtures/default")).unwrap()); + assert!(is_python_project(Path::new("tests/fixtures/pyproject_toml_only")).unwrap()); } #[test] @@ -128,11 +128,8 @@ mod tests { #[test] fn read_optional_file_valid_file() { assert_eq!( - read_optional_file(Path::new( - "tests/fixtures/runtime_txt_python_3.10/runtime.txt" - )) - .unwrap(), - Some("python-3.10.9\n".to_string()) + read_optional_file(Path::new("tests/fixtures/python_3.9/runtime.txt")).unwrap(), + Some("python-3.9.16\n".to_string()) ); } diff --git a/tests/fixtures/pip_editable_git_compiled/requirements.txt b/tests/fixtures/pip_editable_git_compiled/requirements.txt new file mode 100644 index 0000000..191a23a --- /dev/null +++ b/tests/fixtures/pip_editable_git_compiled/requirements.txt @@ -0,0 +1,9 @@ +# This requirement uses a VCS URL and `-e` in order to test that: +# - Git from the stack image can be found (ie: the system PATH has been correctly propagated to pip). +# - The editable mode repository clone is saved into the dependencies layer (via the `--src` option). +# +# The psycopg2 package is used instead of a pure Python package, in order to test that: +# - The Python headers can be found in the `include/pythonX.Y/` directory of the Python layer. +# - Headers/libraries from the stack image can be found (in this case, for libpq-dev). + +-e git+https://github.com/psycopg/psycopg2@2_9_5#egg=psycopg2 diff --git a/tests/fixtures/pip_invalid_requirement/requirements.txt b/tests/fixtures/pip_invalid_requirement/requirements.txt new file mode 100644 index 0000000..db42b7e --- /dev/null +++ b/tests/fixtures/pip_invalid_requirement/requirements.txt @@ -0,0 +1 @@ +an-invalid-requirement! diff --git a/tests/fixtures/default/requirements.txt b/tests/fixtures/project_toml_invalid/requirements.txt similarity index 100% rename from tests/fixtures/default/requirements.txt rename to tests/fixtures/project_toml_invalid/requirements.txt diff --git a/tests/fixtures/runtime_txt_python_version_invalid/requirements.txt b/tests/fixtures/pyproject_toml_only/pyproject.toml similarity index 100% rename from tests/fixtures/runtime_txt_python_version_invalid/requirements.txt rename to tests/fixtures/pyproject_toml_only/pyproject.toml diff --git a/tests/fixtures/python_3.10/requirements.txt b/tests/fixtures/python_3.10/requirements.txt new file mode 100644 index 0000000..fd0f81c --- /dev/null +++ b/tests/fixtures/python_3.10/requirements.txt @@ -0,0 +1,2 @@ +# This package has been picked since it has no dependencies and is small/fast to install. +typing-extensions==4.4.0 diff --git a/tests/fixtures/python_3.10/runtime.txt b/tests/fixtures/python_3.10/runtime.txt new file mode 100644 index 0000000..9769138 --- /dev/null +++ b/tests/fixtures/python_3.10/runtime.txt @@ -0,0 +1 @@ +python-3.10.10 diff --git a/tests/fixtures/python_3.11/requirements.txt b/tests/fixtures/python_3.11/requirements.txt new file mode 100644 index 0000000..fd0f81c --- /dev/null +++ b/tests/fixtures/python_3.11/requirements.txt @@ -0,0 +1,2 @@ +# This package has been picked since it has no dependencies and is small/fast to install. +typing-extensions==4.4.0 diff --git a/tests/fixtures/python_3.11/runtime.txt b/tests/fixtures/python_3.11/runtime.txt new file mode 100644 index 0000000..04d03e3 --- /dev/null +++ b/tests/fixtures/python_3.11/runtime.txt @@ -0,0 +1 @@ +python-3.11.2 diff --git a/tests/fixtures/python_3.7/requirements.txt b/tests/fixtures/python_3.7/requirements.txt new file mode 100644 index 0000000..fd0f81c --- /dev/null +++ b/tests/fixtures/python_3.7/requirements.txt @@ -0,0 +1,2 @@ +# This package has been picked since it has no dependencies and is small/fast to install. +typing-extensions==4.4.0 diff --git a/tests/fixtures/python_3.7/runtime.txt b/tests/fixtures/python_3.7/runtime.txt new file mode 100644 index 0000000..91b17a1 --- /dev/null +++ b/tests/fixtures/python_3.7/runtime.txt @@ -0,0 +1 @@ +python-3.7.16 diff --git a/tests/fixtures/python_3.8/requirements.txt b/tests/fixtures/python_3.8/requirements.txt new file mode 100644 index 0000000..fd0f81c --- /dev/null +++ b/tests/fixtures/python_3.8/requirements.txt @@ -0,0 +1,2 @@ +# This package has been picked since it has no dependencies and is small/fast to install. +typing-extensions==4.4.0 diff --git a/tests/fixtures/python_3.8/runtime.txt b/tests/fixtures/python_3.8/runtime.txt new file mode 100644 index 0000000..9e9414f --- /dev/null +++ b/tests/fixtures/python_3.8/runtime.txt @@ -0,0 +1 @@ +python-3.8.16 diff --git a/tests/fixtures/python_3.9/requirements.txt b/tests/fixtures/python_3.9/requirements.txt new file mode 100644 index 0000000..fd0f81c --- /dev/null +++ b/tests/fixtures/python_3.9/requirements.txt @@ -0,0 +1,2 @@ +# This package has been picked since it has no dependencies and is small/fast to install. +typing-extensions==4.4.0 diff --git a/tests/fixtures/python_3.9/runtime.txt b/tests/fixtures/python_3.9/runtime.txt new file mode 100644 index 0000000..c9cbcea --- /dev/null +++ b/tests/fixtures/python_3.9/runtime.txt @@ -0,0 +1 @@ +python-3.9.16 diff --git a/tests/fixtures/python_version_unspecified/requirements.txt b/tests/fixtures/python_version_unspecified/requirements.txt new file mode 100644 index 0000000..fd0f81c --- /dev/null +++ b/tests/fixtures/python_version_unspecified/requirements.txt @@ -0,0 +1,2 @@ +# This package has been picked since it has no dependencies and is small/fast to install. +typing-extensions==4.4.0 diff --git a/tests/fixtures/runtime_txt_python_version_unavailable/requirements.txt b/tests/fixtures/runtime_txt_invalid_version/requirements.txt similarity index 100% rename from tests/fixtures/runtime_txt_python_version_unavailable/requirements.txt rename to tests/fixtures/runtime_txt_invalid_version/requirements.txt diff --git a/tests/fixtures/runtime_txt_python_version_invalid/runtime.txt b/tests/fixtures/runtime_txt_invalid_version/runtime.txt similarity index 100% rename from tests/fixtures/runtime_txt_python_version_invalid/runtime.txt rename to tests/fixtures/runtime_txt_invalid_version/runtime.txt diff --git a/tests/fixtures/runtime_txt_non_existent_version/requirements.txt b/tests/fixtures/runtime_txt_non_existent_version/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/runtime_txt_python_version_unavailable/runtime.txt b/tests/fixtures/runtime_txt_non_existent_version/runtime.txt similarity index 100% rename from tests/fixtures/runtime_txt_python_version_unavailable/runtime.txt rename to tests/fixtures/runtime_txt_non_existent_version/runtime.txt diff --git a/tests/fixtures/runtime_txt_python_3.10/runtime.txt b/tests/fixtures/runtime_txt_python_3.10/runtime.txt deleted file mode 100644 index 19c64f2..0000000 --- a/tests/fixtures/runtime_txt_python_3.10/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.10.9 diff --git a/tests/fixtures/function_fails_self_check/main.py b/tests/fixtures/salesforce_function_fails_self_check/main.py similarity index 100% rename from tests/fixtures/function_fails_self_check/main.py rename to tests/fixtures/salesforce_function_fails_self_check/main.py diff --git a/tests/fixtures/function_fails_self_check/project.toml b/tests/fixtures/salesforce_function_fails_self_check/project.toml similarity index 100% rename from tests/fixtures/function_fails_self_check/project.toml rename to tests/fixtures/salesforce_function_fails_self_check/project.toml diff --git a/tests/fixtures/function_fails_self_check/requirements.txt b/tests/fixtures/salesforce_function_fails_self_check/requirements.txt similarity index 100% rename from tests/fixtures/function_fails_self_check/requirements.txt rename to tests/fixtures/salesforce_function_fails_self_check/requirements.txt diff --git a/tests/fixtures/function_missing_functions_package/main.py b/tests/fixtures/salesforce_function_missing_package/main.py similarity index 100% rename from tests/fixtures/function_missing_functions_package/main.py rename to tests/fixtures/salesforce_function_missing_package/main.py diff --git a/tests/fixtures/function_missing_functions_package/project.toml b/tests/fixtures/salesforce_function_missing_package/project.toml similarity index 100% rename from tests/fixtures/function_missing_functions_package/project.toml rename to tests/fixtures/salesforce_function_missing_package/project.toml diff --git a/tests/fixtures/function_missing_functions_package/requirements.txt b/tests/fixtures/salesforce_function_missing_package/requirements.txt similarity index 100% rename from tests/fixtures/function_missing_functions_package/requirements.txt rename to tests/fixtures/salesforce_function_missing_package/requirements.txt diff --git a/tests/fixtures/function_template/README.md b/tests/fixtures/salesforce_function_template/README.md similarity index 100% rename from tests/fixtures/function_template/README.md rename to tests/fixtures/salesforce_function_template/README.md diff --git a/tests/fixtures/function_template/main.py b/tests/fixtures/salesforce_function_template/main.py similarity index 100% rename from tests/fixtures/function_template/main.py rename to tests/fixtures/salesforce_function_template/main.py diff --git a/tests/fixtures/function_template/payload.json b/tests/fixtures/salesforce_function_template/payload.json similarity index 100% rename from tests/fixtures/function_template/payload.json rename to tests/fixtures/salesforce_function_template/payload.json diff --git a/tests/fixtures/function_template/project.toml b/tests/fixtures/salesforce_function_template/project.toml similarity index 100% rename from tests/fixtures/function_template/project.toml rename to tests/fixtures/salesforce_function_template/project.toml diff --git a/tests/fixtures/function_template/requirements.txt b/tests/fixtures/salesforce_function_template/requirements.txt similarity index 100% rename from tests/fixtures/function_template/requirements.txt rename to tests/fixtures/salesforce_function_template/requirements.txt diff --git a/tests/integration.rs b/tests/integration.rs index d2aadd1..6a037e1 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -4,9 +4,27 @@ #![warn(clippy::pedantic)] use indoc::{formatdoc, indoc}; -use libcnb_test::{assert_contains, BuildConfig, ContainerConfig, PackResult, TestRunner}; +use libcnb::data::buildpack::{BuildpackVersion, SingleBuildpackDescriptor}; +use libcnb_test::{ + assert_contains, assert_empty, BuildConfig, ContainerConfig, PackResult, TestRunner, +}; use std::time::Duration; -use std::{env, thread}; +use std::{env, fs, thread}; + +// At the moment these can't be imported from the buildpack, since integration +// tests cannot access any interfaces for binary-only crates. +// TODO: Explore moving integration tests into the crate, per: +// https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html +const LATEST_PYTHON_3_7: &str = "3.7.16"; +const LATEST_PYTHON_3_8: &str = "3.8.16"; +const LATEST_PYTHON_3_9: &str = "3.9.16"; +const LATEST_PYTHON_3_10: &str = "3.10.10"; +const LATEST_PYTHON_3_11: &str = "3.11.2"; +const DEFAULT_PYTHON_VERSION: &str = LATEST_PYTHON_3_11; + +const PIP_VERSION: &str = "23.0.1"; +const SETUPTOOLS_VERSION: &str = "67.3.2"; +const WHEEL_VERSION: &str = "0.38.4"; const DEFAULT_BUILDER: &str = "heroku/builder:22"; const TEST_PORT: u16 = 12345; @@ -15,142 +33,289 @@ fn builder() -> String { env::var("INTEGRATION_TEST_CNB_BUILDER").unwrap_or(DEFAULT_BUILDER.to_string()) } +fn buildpack_version() -> BuildpackVersion { + let buildpack_toml = fs::read_to_string("buildpack.toml").unwrap(); + let buildpack_descriptor = + toml::from_str::>>(&buildpack_toml).unwrap(); + buildpack_descriptor.buildpack.version +} + +// Detect + #[test] #[ignore = "integration test"] fn detect_rejects_non_python_projects() { + let buildpack_version = buildpack_version(); + TestRunner::default().build( BuildConfig::new(builder(), "tests/fixtures/empty") .expected_pack_result(PackResult::Failure), |context| { - // We can't test the detect failure reason, since by default pack CLI only shows output for non-zero, - // non-100 exit codes, and `libcnb-test` does not support enabling pack build's verbose mode: - // https://github.com/heroku/libcnb.rs/issues/383 assert_contains!( context.pack_stdout, - "ERROR: No buildpack groups passed detection." + &formatdoc! {" + ===> DETECTING + ======== Output: heroku/python@{buildpack_version} ======== + No Python project files found (such as requirements.txt). + ======== Results ======== + fail: heroku/python@{buildpack_version} + ERROR: No buildpack groups passed detection. + "} ); }, ); } +// Determine package manager + #[test] #[ignore = "integration test"] -fn function_template() { +fn no_package_manager_detected() { TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/function_template"), + BuildConfig::new(builder(), "tests/fixtures/pyproject_toml_only") + .expected_pack_result(PackResult::Failure), |context| { - // Pip outputs git clone output to stderr for some reason, so stderr isn't empty. - // TODO: Decide whether this is a bug in pip and/or if we should work around it. - // assert_empty!(context.pack_stderr); + assert_contains!( + context.pack_stderr, + indoc! {" + [Error: No Python package manager files were found] + A Pip requirements file was not found in your application's source code. + This file is required so that your application's dependencies can be installed. + + Please add a file named exactly 'requirements.txt' to the root directory of your + application, containing a list of the packages required by your application. + + For more information on what this file should contain, see: + https://pip.pypa.io/en/stable/reference/requirements-file-format/ + "} + ); + }, + ); +} + +// runtime.txt parsing + +#[test] +#[ignore = "integration test"] +fn runtime_txt_invalid_version() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/runtime_txt_invalid_version") + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + &formatdoc! {" + [Error: Invalid Python version in runtime.txt] + The Python version specified in 'runtime.txt' is not in the correct format. + + The following file contents were found: + python-an.invalid.version + + However, the file contents must begin with a 'python-' prefix, followed by the + version specified as '..'. Comments are not supported. + + For example, to request Python {DEFAULT_PYTHON_VERSION}, the correct version format is: + python-{DEFAULT_PYTHON_VERSION} + + Please update 'runtime.txt' to use the correct version format, or else remove + the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). + + For a list of the supported Python versions, see: + https://devcenter.heroku.com/articles/python-support#supported-runtimes + "} + ); + }, + ); +} +#[test] +#[ignore = "integration test"] +fn runtime_txt_non_existent_version() { + rejects_non_existent_python_version( + "tests/fixtures/runtime_txt_non_existent_version", + "999.999.999", + ); +} + +// Python versions + +#[test] +#[ignore = "integration test"] +fn python_version_unspecified() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/python_version_unspecified"), + |context| { + assert_empty!(context.pack_stderr); assert_contains!( context.pack_stdout, - indoc! {" + &formatdoc! {" + ===> BUILDING + [Determining Python version] - No Python version specified, using the current default of 3.11.2. + No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes [Installing Python] - Downloading Python 3.11.2 + Downloading Python {DEFAULT_PYTHON_VERSION} Python installation successful [Installing Pip] - Installing pip 23.0, setuptools 67.1.0 and wheel 0.38.4 + Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} Installation completed [Installing dependencies using Pip] Pip cache created Running pip install - Collecting salesforce-functions - "} - ); - - assert_contains!( - context.pack_stdout, - indoc! {" + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 Pip install completed - - [Validating Salesforce Function] - Function passed validation. + ===> EXPORTING "} ); + }, + ); +} - context.start_container( - ContainerConfig::new() - .env("PORT", TEST_PORT.to_string()) - .expose_port(TEST_PORT), - |container| { - let address_on_host = container.address_for_port(TEST_PORT).unwrap(); - let url = format!("http://{}:{}", address_on_host.ip(), address_on_host.port()); - - // Retries needed since the server takes a moment to start up. - let mut attempts_remaining = 5; - let response = loop { - let response = ureq::post(&url).set("x-health-check", "true").call(); - if response.is_ok() || attempts_remaining == 0 { - break response; - } - attempts_remaining -= 1; - thread::sleep(Duration::from_secs(1)); - }; +#[test] +#[ignore = "integration test"] +fn python_3_7() { + // Python 3.7 is only available on Heroku-20 and older. + let fixture = "tests/fixtures/python_3.7"; + match builder().as_str() { + "heroku/buildpacks:20" => builds_with_python_version(fixture, LATEST_PYTHON_3_7), + _ => rejects_non_existent_python_version(fixture, LATEST_PYTHON_3_7), + }; +} - let server_log_output = container.logs_now(); - assert_contains!( - server_log_output.stderr, - &format!("Uvicorn running on http://0.0.0.0:{TEST_PORT}") - ); +#[test] +#[ignore = "integration test"] +fn python_3_8() { + // Python 3.8 is only available on Heroku-20 and older. + let fixture = "tests/fixtures/python_3.8"; + match builder().as_str() { + "heroku/buildpacks:20" => builds_with_python_version(fixture, LATEST_PYTHON_3_8), + _ => rejects_non_existent_python_version(fixture, LATEST_PYTHON_3_8), + }; +} - let body = response.unwrap().into_string().unwrap(); - assert_eq!(body, r#""OK""#); - }, - ); - }, - ); +#[test] +#[ignore = "integration test"] +fn python_3_9() { + builds_with_python_version("tests/fixtures/python_3.9", LATEST_PYTHON_3_9); } #[test] #[ignore = "integration test"] -fn function_repeat_build() { - TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/function_template"), - |context| { - let config = context.config.clone(); - context.rebuild(config, |rebuild_context| { - assert_contains!( - rebuild_context.pack_stdout, - indoc! {" - [Determining Python version] - No Python version specified, using the current default of 3.11.2. - To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes - - [Installing Python] - Re-using cached Python 3.11.2 - - [Installing Pip] - Re-using cached pip 23.0, setuptools 67.1.0 and wheel 0.38.4 - - [Installing dependencies using Pip] - Re-using cached pip-cache - Running pip install - Collecting salesforce-functions - "} - ); - }); - }, - ); +fn python_3_10() { + builds_with_python_version("tests/fixtures/python_3.10", LATEST_PYTHON_3_10); } #[test] #[ignore = "integration test"] -fn runtime_txt_python_version_unavailable() { +fn python_3_11() { + builds_with_python_version("tests/fixtures/python_3.11", LATEST_PYTHON_3_11); +} + +fn builds_with_python_version(fixture_path: &str, python_version: &str) { + let mut config = BuildConfig::new(builder(), fixture_path); + // Checks that potentially broken user-provided env vars are not being passed unfiltered to + // subprocesses we launch (such as `pip install`), thanks to `clear-env` in `buildpack.toml`. + config.env("PYTHONHOME", "/invalid-path"); + + TestRunner::default().build(config, |context| { + assert_empty!(context.pack_stderr); + assert_contains!( + context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + Using Python version {python_version} specified in runtime.txt + + [Installing Python] + Downloading Python {python_version} + Python installation successful + + [Installing Pip] + Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} + Installation completed + + [Installing dependencies using Pip] + Pip cache created + Running pip install + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + Pip install completed + ===> EXPORTING + "} + ); + // There's no sensible default process type we can set for Python apps. + assert_contains!(context.pack_stdout, "no default process type"); + + // Validate the Python/Pip install works as expected at runtime. + let command_output = context.run_shell_command( + indoc! {r#" + set -euo pipefail + + # Check that we installed the correct Python version, and that the command + # 'python' works (since it's a symlink to the actual 'python3' binary). + python --version + + # Check that the Python binary is using its own 'libpython' and not the system one: + # https://github.com/docker-library/python/issues/784 + # Note: This has to handle Python 3.9 and older not being built in shared library mode. + libpython_path=$(ldd /layers/heroku_python/python/bin/python | grep libpython || true) + if [[ -n "${libpython_path}" && "${libpython_path}" != *"=> /layers/"* ]]; then + echo "The Python binary is not using the correct libpython!" + echo "${libpython_path}" + exit 1 + fi + + # Check all required dynamically linked libraries can be found in the runtime image. + if find /layers -name '*.so' -exec ldd '{}' + | grep 'not found'; then + echo "The above dynamically linked libraries were not found!" + exit 1 + fi + + # Check that: + # - Pip is available at runtime too (and not just during the build). + # - The correct versions of pip/setuptools/wheel were installed. + # - Pip uses (via 'PYTHONUSERBASE') the user site-packages in the dependencies + # layer, and so can find the typing-extensions package installed there. + # - The "pip update available" warning is not shown (since it should be suppressed). + # - The system site-packages directory is protected against running 'pip install' + # without having passed '--user'. + pip list + pip install --dry-run typing-extensions + "#} + ); + assert_empty!(command_output.stderr); + assert_contains!( + command_output.stdout, + &formatdoc! {" + Python {python_version} + Package Version + ----------------- ------- + pip {PIP_VERSION} + setuptools {SETUPTOOLS_VERSION} + typing_extensions 4.4.0 + wheel {WHEEL_VERSION} + Defaulting to user installation because normal site-packages is not writeable + Requirement already satisfied: typing-extensions in /layers/heroku_python/dependencies/lib/" + } + ); + }); +} + +fn rejects_non_existent_python_version(fixture_path: &str, python_version: &str) { let builder = builder(); TestRunner::default().build( - BuildConfig::new( - &builder, - "tests/fixtures/runtime_txt_python_version_unavailable", - ) - .expected_pack_result(PackResult::Failure), + BuildConfig::new(&builder, fixture_path).expected_pack_result(PackResult::Failure), |context| { let expected_stack = match builder.as_str() { "heroku/buildpacks:20" => "heroku-20", @@ -162,10 +327,10 @@ fn runtime_txt_python_version_unavailable() { context.pack_stderr, &formatdoc! {" [Error: Requested Python version is not available] - The requested Python version (999.999.999) is not available for this stack ({expected_stack}). + The requested Python version ({python_version}) is not available for this stack ({expected_stack}). Please update the version in 'runtime.txt' to a supported Python version, or else - remove the file to instead use the default version (currently Python 3.11.2). + remove the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). For a list of the supported Python versions, see: https://devcenter.heroku.com/articles/python-support#supported-runtimes @@ -175,49 +340,251 @@ fn runtime_txt_python_version_unavailable() { ); } +// Pip + #[test] #[ignore = "integration test"] -fn runtime_txt_python_version_invalid() { +fn pip_editable_git_compiled() { + // This tests that: + // - Git from the stack image can be found (ie: the system PATH has been correctly propagated to pip). + // - The editable mode repository clone is saved into the dependencies layer not the app dir. + // - Compiling a source distribution package (as opposed to a pre-built wheel) works. + // - The Python headers can be found in the `include/pythonX.Y/` directory of the Python layer. + // - Headers/libraries from the stack image can be found (in this case, for libpq-dev). TestRunner::default().build( - BuildConfig::new( - builder(), - "tests/fixtures/runtime_txt_python_version_invalid", - ) - .expected_pack_result(PackResult::Failure), + BuildConfig::new(builder(), "tests/fixtures/pip_editable_git_compiled"), + |context| { + assert_contains!( + context.pack_stdout, + "Cloning https://github.com/psycopg/psycopg2 (to revision 2_9_5) to /layers/heroku_python/dependencies/src/psycopg2" + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn pip_invalid_requirement() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/pip_invalid_requirement") + .expected_pack_result(PackResult::Failure), |context| { + // Ideally we could test a combined stdout/stderr, however libcnb-test doesn't support this: + // https://github.com/heroku/libcnb.rs/issues/536 + assert_contains!( + context.pack_stdout, + &formatdoc! {" + [Installing dependencies using Pip] + Pip cache created + Running pip install + "} + ); assert_contains!( context.pack_stderr, - indoc! {" - [Error: Invalid Python version in runtime.txt] - The Python version specified in 'runtime.txt' is not in the correct format. + &formatdoc! {" + ERROR: Invalid requirement: 'an-invalid-requirement!' (from line 1 of requirements.txt) - The following file contents were found: - python-an.invalid.version + [Error: Unable to install dependencies using pip] + The 'pip install' command to install the application's dependencies from + 'requirements.txt' failed (exit status: 1). - However, the file contents must begin with a 'python-' prefix, followed by the - version specified as '..'. Comments are not supported. + See the log output above for more information. + "} + ); + }, + ); +} + +// Caching + +#[test] +#[ignore = "integration test"] +fn cache_used_for_repeat_builds() { + let config = BuildConfig::new(builder(), "tests/fixtures/python_3.11"); + + TestRunner::default().build(&config, |context| { + context.rebuild(&config, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + ===> BUILDING - For example, to request Python 3.11.2, the correct version format is: - python-3.11.2 + [Determining Python version] + Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt - Please update 'runtime.txt' to use the correct version format, or else remove - the file to instead use the default version (currently Python 3.11.2). + [Installing Python] + Re-using cached Python {LATEST_PYTHON_3_11} - For a list of the supported Python versions, see: - https://devcenter.heroku.com/articles/python-support#supported-runtimes + [Installing Pip] + Re-using cached pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} + + [Installing dependencies using Pip] + Re-using cached pip-cache + Running pip install + Collecting typing-extensions==4.4.0 + Using cached typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + Pip install completed + ===> EXPORTING + "} + ); + }); + }); +} + +#[test] +#[ignore = "integration test"] +fn cache_discarded_on_python_version_change() { + let builder = builder(); + let config_before = BuildConfig::new(&builder, "tests/fixtures/python_3.10"); + let config_after = BuildConfig::new(&builder, "tests/fixtures/python_3.11"); + + TestRunner::default().build(config_before, |context| { + context.rebuild(config_after, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt + Discarding cached Python {LATEST_PYTHON_3_10} + Discarding cached pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} + + [Installing Python] + Downloading Python {LATEST_PYTHON_3_11} + Python installation successful + + [Installing Pip] + Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} + Installation completed + + [Installing dependencies using Pip] + Discarding cached pip-cache + Pip cache created + Running pip install + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + Pip install completed + ===> EXPORTING + "} + ); + }); + }); +} + +#[test] +#[ignore = "integration test"] +fn cache_discarded_on_stack_change() { + let fixture = "tests/fixtures/python_version_unspecified"; + let config_before = BuildConfig::new("heroku/buildpacks:20", fixture); + let config_after = BuildConfig::new("heroku/builder:22", fixture); + + TestRunner::default().build(config_before, |context| { + context.rebuild(config_after, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. + To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes + Discarding cached Python {DEFAULT_PYTHON_VERSION} + Discarding cached pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} + + [Installing Python] + Downloading Python {DEFAULT_PYTHON_VERSION} + Python installation successful + + [Installing Pip] + Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} + Installation completed + + [Installing dependencies using Pip] + Discarding cached pip-cache + Pip cache created + Running pip install + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + Pip install completed + ===> EXPORTING "} ); + }); + }); +} + +// Salesforce Functions + +#[test] +#[ignore = "integration test"] +fn salesforce_function_template() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/salesforce_function_template"), + |context| { + assert_empty!(context.pack_stderr); + assert_contains!( + context.pack_stdout, + indoc! {" + Pip install completed + + [Validating Salesforce Function] + Function passed validation. + ===> EXPORTING + "} + ); + assert_contains!(context.pack_stdout, "Setting default process type 'web'"); + + // Test that the `sf-functions-python` web process the buildpack configures works correctly. + context.start_container( + ContainerConfig::new() + .env("PORT", TEST_PORT.to_string()) + .expose_port(TEST_PORT), + |container| { + let address_on_host = container.address_for_port(TEST_PORT).unwrap(); + let url = format!("http://{}:{}", address_on_host.ip(), address_on_host.port()); + + // Retries needed since the server takes a moment to start up. + let mut attempts_remaining = 5; + let response = loop { + let response = ureq::post(&url).set("x-health-check", "true").call(); + if response.is_ok() || attempts_remaining == 0 { + break response; + } + attempts_remaining -= 1; + thread::sleep(Duration::from_secs(1)); + }; + + let server_log_output = container.logs_now(); + assert_contains!( + server_log_output.stderr, + &format!("Uvicorn running on http://0.0.0.0:{TEST_PORT}") + ); + + let body = response.unwrap().into_string().unwrap(); + assert_eq!(body, r#""OK""#); + }, + ); }, ); } #[test] #[ignore = "integration test"] -fn function_missing_functions_package() { +fn salesforce_function_missing_package() { TestRunner::default().build( BuildConfig::new( builder(), - "tests/fixtures/function_missing_functions_package", + "tests/fixtures/salesforce_function_missing_package", ) .expected_pack_result(PackResult::Failure), |context| { @@ -241,11 +608,11 @@ fn function_missing_functions_package() { #[test] #[ignore = "integration test"] -fn function_fails_self_check() { +fn salesforce_function_fails_self_check() { TestRunner::default().build( BuildConfig::new( builder(), - "tests/fixtures/function_fails_self_check", + "tests/fixtures/salesforce_function_fails_self_check", ) .expected_pack_result(PackResult::Failure), |context| { @@ -257,34 +624,33 @@ fn function_fails_self_check() { there is a problem with the Python Salesforce Function in this project. Details: - Function failed validation: 'invalid' isn't a valid Salesforce REST API version. Update the 'salesforce-api-version' key in project.toml to a version that uses the form 'X.Y', such as '56.0'. - "} + Function failed validation: 'invalid' isn't a valid Salesforce REST API version." + } ); }, ); } -// TODO: -// -// Detect -// - no Python files -// -// Python versions -// - Default -// - 3.11. -// - 3.11. (show update warning) -// - 3.10. -// - 3.9. -// - 3.8 (unsupported, show reason) -// - 3.7 (unsupported, show reason) -// - 3.6 (unsupported, explain EOL) -// - various invalid version strings -// -// Caching -// - Python version change -// - Stack change -// - Various Pip cache invalidation types (package additions/removals etc) -// - No-op -// -// Other -// - that pip install can find Python headers +#[test] +#[ignore = "integration test"] +fn project_toml_invalid() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/project_toml_invalid") + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {r#" + [Error: Invalid project.toml] + A parsing/validation error error occurred whilst loading the project.toml file. + + Details: TOML parse error at line 4, column 1 + | + 4 | [com.salesforce] + | ^^^^^^^^^^^^^^^^ + missing field `type` + "#} + ); + }, + ); +} From a8febffb648b532e87a7c00373379757f94a00c8 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 20 Feb 2023 13:51:48 +0000 Subject: [PATCH 51/71] Try `--test-threads 10` for integration tests in CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfaa73c..f8a8c90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,4 +63,4 @@ jobs: uses: buildpacks/github-actions/setup-pack@v5.0.1 - name: Run integration tests # Runs only tests annotated with the `ignore` attribute (which in this repo, are the integration tests). - run: cargo test --locked -- --ignored + run: cargo test --locked -- --ignored --test-threads 10 From a7e3148b7f940c125342878cfaa473cdaa58898a Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 20 Feb 2023 13:56:30 +0000 Subject: [PATCH 52/71] `--test-threads 6` --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8a8c90..c64c802 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,4 +63,4 @@ jobs: uses: buildpacks/github-actions/setup-pack@v5.0.1 - name: Run integration tests # Runs only tests annotated with the `ignore` attribute (which in this repo, are the integration tests). - run: cargo test --locked -- --ignored --test-threads 10 + run: cargo test --locked -- --ignored --test-threads 6 From 544dba9d6d578f2a4c8288ebfa0b4720311c439c Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 20 Feb 2023 14:01:30 +0000 Subject: [PATCH 53/71] `--test-threads 4` --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c64c802..53a1088 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,4 +63,4 @@ jobs: uses: buildpacks/github-actions/setup-pack@v5.0.1 - name: Run integration tests # Runs only tests annotated with the `ignore` attribute (which in this repo, are the integration tests). - run: cargo test --locked -- --ignored --test-threads 6 + run: cargo test --locked -- --ignored --test-threads 4 From 04d0c1667d61ff31b2f3a962dfd84296374f9566 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 20 Feb 2023 14:08:25 +0000 Subject: [PATCH 54/71] `--test-threads 5` --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53a1088..17a97df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,4 +63,4 @@ jobs: uses: buildpacks/github-actions/setup-pack@v5.0.1 - name: Run integration tests # Runs only tests annotated with the `ignore` attribute (which in this repo, are the integration tests). - run: cargo test --locked -- --ignored --test-threads 4 + run: cargo test --locked -- --ignored --test-threads 5 From fa4b16e6cd55f0460c6890ca299c7fa57107fd78 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 24 Feb 2023 14:48:08 +0000 Subject: [PATCH 55/71] Update Swatinem/rust-cache to v2.2.1 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17a97df..0ecdf9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - name: Update Rust toolchain run: rustup update - name: Rust Cache - uses: Swatinem/rust-cache@v2.2.0 + uses: Swatinem/rust-cache@v2.2.1 - name: Clippy run: cargo clippy --all-targets --locked -- --deny warnings - name: rustfmt @@ -36,7 +36,7 @@ jobs: - name: Update Rust toolchain run: rustup update - name: Rust Cache - uses: Swatinem/rust-cache@v2.2.0 + uses: Swatinem/rust-cache@v2.2.1 - name: Run unit tests run: cargo test --locked @@ -58,7 +58,7 @@ jobs: - name: Install Rust linux-musl target run: rustup target add x86_64-unknown-linux-musl - name: Rust Cache - uses: Swatinem/rust-cache@v2.2.0 + uses: Swatinem/rust-cache@v2.2.1 - name: Install Pack CLI uses: buildpacks/github-actions/setup-pack@v5.0.1 - name: Run integration tests From 457a017f6632cffdab8de425b3f35d4fd826fa3e Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 24 Feb 2023 16:04:28 +0000 Subject: [PATCH 56/71] Improve cache and logging --- src/layers/pip_cache.rs | 54 ++++--- src/layers/pip_dependencies.rs | 2 - src/layers/python.rs | 267 ++++++++++++++++++++++++--------- src/main.rs | 6 +- src/package_manager.rs | 19 +++ tests/integration.rs | 102 +++++++------ 6 files changed, 307 insertions(+), 143 deletions(-) diff --git a/src/layers/pip_cache.rs b/src/layers/pip_cache.rs index 86d27d2..d4d1595 100644 --- a/src/layers/pip_cache.rs +++ b/src/layers/pip_cache.rs @@ -1,3 +1,4 @@ +use crate::package_manager::PackagingToolVersions; use crate::python_version::PythonVersion; use crate::PythonBuildpack; use libcnb::build::BuildContext; @@ -11,13 +12,10 @@ use std::path::Path; /// Layer containing Pip's cache of HTTP requests/downloads and built package wheels. pub(crate) struct PipCacheLayer<'a> { + /// The Python version used for this build. pub python_version: &'a PythonVersion, -} - -#[derive(Clone, Deserialize, PartialEq, Serialize)] -pub(crate) struct PipCacheLayerMetadata { - python_version: String, - stack: StackId, + /// The pip, setuptools and wheel versions used for this build. + pub packaging_tool_versions: &'a PackagingToolVersions, } impl Layer for PipCacheLayer<'_> { @@ -37,8 +35,7 @@ impl Layer for PipCacheLayer<'_> { context: &BuildContext, _layer_path: &Path, ) -> Result, ::Error> { - log_info("Pip cache created"); - let layer_metadata = generate_layer_metadata(&context.stack_id, self.python_version); + let layer_metadata = self.generate_layer_metadata(&context.stack_id); LayerResultBuilder::new(layer_metadata).build() } @@ -47,30 +44,37 @@ impl Layer for PipCacheLayer<'_> { context: &BuildContext, layer_data: &LayerData, ) -> Result::Error> { - // TODO: Also invalidate based on time since layer creation? - // TODO: Decide what should be logged - if layer_data.content_metadata.metadata - == generate_layer_metadata(&context.stack_id, self.python_version) - { - log_info("Re-using cached pip-cache"); + let cached_metadata = &layer_data.content_metadata.metadata; + let new_metadata = &self.generate_layer_metadata(&context.stack_id); + + if cached_metadata == new_metadata { + log_info("Using cached pip download/wheel cache"); Ok(ExistingLayerStrategy::Keep) } else { - log_info("Discarding cached pip-cache"); + log_info("Discarding cached pip download/wheel cache"); Ok(ExistingLayerStrategy::Recreate) } } } -fn generate_layer_metadata( - stack_id: &StackId, - python_version: &PythonVersion, -) -> PipCacheLayerMetadata { - // TODO: Add timestamp field or similar (maybe not necessary if invalidating on pip/python change?) - // TODO: Invalidate on pip version change? - PipCacheLayerMetadata { - python_version: python_version.to_string(), - stack: stack_id.clone(), +impl<'a> PipCacheLayer<'a> { + fn generate_layer_metadata(&self, stack_id: &StackId) -> PipCacheLayerMetadata { + PipCacheLayerMetadata { + stack: stack_id.clone(), + python_version: self.python_version.to_string(), + packaging_tool_versions: self.packaging_tool_versions.clone(), + } } } -// TODO: Unit tests for cache invalidation handling? +/// Metadata stored in the generated layer that allows future builds to determine whether +/// the cached layer needs to be invalidated or not. +// Timestamp based cache invalidation isn't used here since the Python/pip/setuptools/wheel +// versions will change often enough that it isn't worth the added complexity. Ideally pip +// would support cleaning up its own cache: https://github.com/pypa/pip/issues/6956 +#[derive(Clone, Deserialize, PartialEq, Serialize)] +pub(crate) struct PipCacheLayerMetadata { + stack: StackId, + python_version: String, + packaging_tool_versions: PackagingToolVersions, +} diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index 9dc153f..e0fea8c 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -108,8 +108,6 @@ impl Layer for PipDependenciesLayer<'_> { ) .map_err(PipDependenciesLayerError::PipInstallCommand)?; - log_info("Pip install completed"); - LayerResultBuilder::new(GenericMetadata::default()) .env(layer_env) .build() diff --git a/src/layers/python.rs b/src/layers/python.rs index 32f0d71..e26f4d2 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -1,3 +1,4 @@ +use crate::package_manager::PackagingToolVersions; use crate::python_version::PythonVersion; use crate::utils::{self, CommandError, DownloadUnpackArchiveError}; use crate::{BuildpackError, PythonBuildpack}; @@ -7,7 +8,7 @@ use libcnb::data::layer_content_metadata::LayerTypes; use libcnb::layer::{ExistingLayerStrategy, Layer, LayerData, LayerResult, LayerResultBuilder}; use libcnb::layer_env::{LayerEnv, ModificationBehavior, Scope}; use libcnb::{Buildpack, Env}; -use libherokubuildpack::log::{log_header, log_info}; +use libherokubuildpack::log::log_info; use serde::{Deserialize, Serialize}; use std::fs::Permissions; use std::os::unix::prelude::PermissionsExt; @@ -15,25 +16,14 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{fs, io}; -const PIP_VERSION: &str = "23.0.1"; -const SETUPTOOLS_VERSION: &str = "67.3.2"; -const WHEEL_VERSION: &str = "0.38.4"; - /// Layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`. pub(crate) struct PythonLayer<'a> { /// Environment variables inherited from earlier buildpack steps. pub command_env: &'a Env, - /// The Python version that will be installed. + /// The Python version that this layer should install. pub python_version: &'a PythonVersion, -} - -#[derive(Clone, Deserialize, PartialEq, Serialize)] -pub(crate) struct PythonLayerMetadata { - stack: StackId, - python_version: String, - pip_version: String, - setuptools_version: String, - wheel_version: String, + /// The pip, setuptools and wheel versions that this layer should install. + pub packaging_tool_versions: &'a PackagingToolVersions, } impl Layer for PythonLayer<'_> { @@ -53,15 +43,13 @@ impl Layer for PythonLayer<'_> { context: &BuildContext, layer_path: &Path, ) -> Result, ::Error> { - log_header("Installing Python"); - // TODO: Move this URL generation somewhere else (ie manifest etc). let archive_url = format!( "https://heroku-buildpack-python.s3.us-east-1.amazonaws.com/{}/runtimes/python-{}.tar.gz", context.stack_id, self.python_version ); - log_info(format!("Downloading Python {}", self.python_version)); + log_info(format!("Installing Python {}", self.python_version)); utils::download_and_unpack_gzipped_archive(&archive_url, layer_path).map_err(|error| { match error { // TODO: Remove this once the Python version is validated against a manifest (at which @@ -75,7 +63,6 @@ impl Layer for PythonLayer<'_> { other_error => PythonLayerError::DownloadUnpackPythonArchive(other_error), } })?; - log_info("Python installation successful"); let layer_env = generate_layer_env(layer_path, self.python_version); let mut command_env = layer_env.apply(Scope::Build, self.command_env); @@ -87,8 +74,15 @@ impl Layer for PythonLayer<'_> { // explicitly set it for the Python invocations within this layer. command_env.insert("LD_LIBRARY_PATH", layer_path.join("lib")); - log_header("Installing Pip"); - log_info(format!("Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION}")); + let PackagingToolVersions { + pip_version, + setuptools_version, + wheel_version, + } = self.packaging_tool_versions; + + log_info(format!( + "Installing pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version}" + )); let python_binary = layer_path.join("bin/python"); let python_stdlib_dir = layer_path.join(format!( @@ -111,9 +105,9 @@ impl Layer for PythonLayer<'_> { "--no-cache-dir", "--no-input", "--quiet", - format!("pip=={PIP_VERSION}").as_str(), - format!("setuptools=={SETUPTOOLS_VERSION}").as_str(), - format!("wheel=={WHEEL_VERSION}").as_str(), + format!("pip=={pip_version}").as_str(), + format!("setuptools=={setuptools_version}").as_str(), + format!("wheel=={wheel_version}").as_str(), ]) .env_clear() .envs(&command_env), @@ -130,9 +124,7 @@ impl Layer for PythonLayer<'_> { fs::set_permissions(site_packages_dir, Permissions::from_mode(0o555)) .map_err(PythonLayerError::MakeSitePackagesReadOnlyIo)?; - log_info("Installation completed"); - - let layer_metadata = generate_layer_metadata(&context.stack_id, self.python_version); + let layer_metadata = self.generate_layer_metadata(&context.stack_id); LayerResultBuilder::new(layer_metadata) .env(layer_env) .build() @@ -143,43 +135,122 @@ impl Layer for PythonLayer<'_> { context: &BuildContext, layer_data: &LayerData, ) -> Result::Error> { - // TODO: Decide what should be logged in the cached case (+more granular reason?) - // Worth including what changed not only for cache invalidation, but also - // to help debug any issues (eg changed pip version causing issues) - let old_metadata = &layer_data.content_metadata.metadata; - let new_metadata = generate_layer_metadata(&context.stack_id, self.python_version); - if new_metadata == *old_metadata { - log_header("Installing Python"); - log_info(format!( - "Re-using cached Python {}", - old_metadata.python_version - )); + let cached_metadata = &layer_data.content_metadata.metadata; + let new_metadata = self.generate_layer_metadata(&context.stack_id); - log_header("Installing Pip"); - log_info(format!( - "Re-using cached pip {}, setuptools {} and wheel {}", - new_metadata.pip_version, - new_metadata.setuptools_version, - new_metadata.wheel_version - )); - - Ok(ExistingLayerStrategy::Keep) + if let Some(reason) = cache_invalidation_reason(cached_metadata, &new_metadata) { + log_info(format!("Discarding cache {reason}")); + Ok(ExistingLayerStrategy::Recreate) } else { log_info(format!( - "Discarding cached Python {}", - old_metadata.python_version + "Using cached Python {}", + cached_metadata.python_version )); + let PackagingToolVersions { + pip_version, + setuptools_version, + wheel_version, + } = &cached_metadata.packaging_tool_versions; log_info(format!( - "Discarding cached pip {}, setuptools {} and wheel {}", - old_metadata.pip_version, - old_metadata.setuptools_version, - old_metadata.wheel_version + "Using cached pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version}" )); - Ok(ExistingLayerStrategy::Recreate) + Ok(ExistingLayerStrategy::Keep) + } + } +} + +impl<'a> PythonLayer<'a> { + fn generate_layer_metadata(&self, stack_id: &StackId) -> PythonLayerMetadata { + PythonLayerMetadata { + stack: stack_id.clone(), + python_version: self.python_version.to_string(), + packaging_tool_versions: self.packaging_tool_versions.clone(), } } } +/// Metadata stored in the generated layer that allows future builds to determine whether +/// the cached layer needs to be invalidated or not. +#[derive(Clone, Deserialize, PartialEq, Serialize)] +pub(crate) struct PythonLayerMetadata { + stack: StackId, + python_version: String, + packaging_tool_versions: PackagingToolVersions, +} + +/// Compare cached layer metadata to the new layer metadata to determine if the cache should +/// be invalidated, and if so, for what reason. +fn cache_invalidation_reason( + cached_metadata: &PythonLayerMetadata, + new_metadata: &PythonLayerMetadata, +) -> Option { + // By destructuring here we ensure that if any additional fields are added to the layer + // metadata in the future, it forces them to be used as part of cache invalidation, + // otherwise Clippy would report unused variable errors. + let PythonLayerMetadata { + stack: cached_stack, + python_version: cached_python_version, + packaging_tool_versions: + PackagingToolVersions { + pip_version: cached_pip_version, + setuptools_version: cached_setuptools_version, + wheel_version: cached_wheel_version, + }, + } = cached_metadata; + + let PythonLayerMetadata { + stack, + python_version, + packaging_tool_versions: + PackagingToolVersions { + pip_version, + setuptools_version, + wheel_version, + }, + } = new_metadata; + + let mut reasons = Vec::new(); + + if cached_stack != stack { + reasons.push(format!( + "the stack has changed from {cached_stack} to {stack}" + )); + } + + if cached_python_version != python_version { + reasons.push(format!( + "the Python version has changed from {cached_python_version} to {python_version}" + )); + } + + if cached_pip_version != pip_version { + reasons.push(format!( + "the pip version has changed from {cached_pip_version} to {pip_version}" + )); + } + + if cached_setuptools_version != setuptools_version { + reasons.push(format!( + "the setuptools version has changed from {cached_setuptools_version} to {setuptools_version}" + )); + } + + if cached_wheel_version != wheel_version { + reasons.push(format!( + "the wheel version has changed from {cached_wheel_version} to {wheel_version}" + )); + } + + // If there is more than one reason then all are mentioned to hopefully prevent support + // tickets where build failures are blamed on a stack upgrade but were actually caused + // by the app's Python version being updated at the same time. + match reasons.as_slice() { + [] => None, + [reason] => Some(format!("since {reason}")), + reasons => Some(format!("since:\n - {}", reasons.join("\n - "))), + } +} + /// Environment variables that will be set by this layer. fn generate_layer_env(layer_path: &Path, python_version: &PythonVersion) -> LayerEnv { // Several of the env vars below are technically build-time only vars, however, we use @@ -324,19 +395,6 @@ fn bundled_pip_module_path(python_stdlib_dir: &Path) -> io::Result { )) } -fn generate_layer_metadata( - stack_id: &StackId, - python_version: &PythonVersion, -) -> PythonLayerMetadata { - PythonLayerMetadata { - stack: stack_id.clone(), - python_version: python_version.to_string(), - pip_version: PIP_VERSION.to_string(), - setuptools_version: SETUPTOOLS_VERSION.to_string(), - wheel_version: WHEEL_VERSION.to_string(), - } -} - /// Errors that can occur when installing Python and required packaging tools into a layer. #[derive(Debug)] pub(crate) enum PythonLayerError { @@ -358,8 +416,83 @@ impl From for BuildpackError { #[cfg(test)] mod tests { + use indoc::indoc; + use libcnb::data::stack_id; + use super::*; + #[test] + fn cache_invalidation_reason_unchanged() { + let metadata = PythonLayerMetadata { + stack: stack_id!("heroku-22"), + python_version: "3.11.0".to_string(), + packaging_tool_versions: PackagingToolVersions { + pip_version: "A.B.C".to_string(), + setuptools_version: "D.E.F".to_string(), + wheel_version: "G.H.I".to_string(), + }, + }; + assert_eq!(cache_invalidation_reason(&metadata, &metadata), None); + } + + #[test] + fn cache_invalidation_reason_single_change() { + let cached_metadata = PythonLayerMetadata { + stack: stack_id!("heroku-22"), + python_version: "3.11.0".to_string(), + packaging_tool_versions: PackagingToolVersions { + pip_version: "A.B.C".to_string(), + setuptools_version: "D.E.F".to_string(), + wheel_version: "G.H.I".to_string(), + }, + }; + let new_metadata = PythonLayerMetadata { + python_version: "3.11.1".to_string(), + ..cached_metadata.clone() + }; + assert_eq!( + cache_invalidation_reason(&cached_metadata, &new_metadata), + Some("since the Python version has changed from 3.11.0 to 3.11.1".to_string()) + ); + } + + #[test] + fn cache_invalidation_reason_all_changed() { + let cached_metadata = PythonLayerMetadata { + stack: stack_id!("heroku-20"), + python_version: "3.9.0".to_string(), + packaging_tool_versions: PackagingToolVersions { + pip_version: "A.B.C".to_string(), + setuptools_version: "D.E.F".to_string(), + wheel_version: "G.H.I".to_string(), + }, + }; + let new_metadata = PythonLayerMetadata { + stack: stack_id!("heroku-22"), + python_version: "3.11.1".to_string(), + packaging_tool_versions: PackagingToolVersions { + pip_version: "A.B.C-new".to_string(), + setuptools_version: "D.E.F-new".to_string(), + wheel_version: "G.H.I-new".to_string(), + }, + }; + assert_eq!( + cache_invalidation_reason(&cached_metadata, &new_metadata), + Some( + indoc! {" + since: + - the stack has changed from heroku-20 to heroku-22 + - the Python version has changed from 3.9.0 to 3.11.1 + - the pip version has changed from A.B.C to A.B.C-new + - the setuptools version has changed from D.E.F to D.E.F-new + - the wheel version has changed from G.H.I to G.H.I-new + "} + .trim() + .to_string() + ) + ); + } + #[test] fn python_layer_env() { let layer_env = generate_layer_env( diff --git a/src/main.rs b/src/main.rs index 832b37b..4c7ee42 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,7 @@ mod utils; use crate::layers::pip_cache::PipCacheLayer; use crate::layers::pip_dependencies::{PipDependenciesLayer, PipDependenciesLayerError}; use crate::layers::python::{PythonLayer, PythonLayerError}; -use crate::package_manager::{DeterminePackageManagerError, PackageManager}; +use crate::package_manager::{DeterminePackageManagerError, PackageManager, PackagingToolVersions}; use crate::project_descriptor::ProjectDescriptorError; use crate::python_version::PythonVersionError; use crate::salesforce_functions::CheckSalesforceFunctionError; @@ -61,6 +61,7 @@ impl Buildpack for PythonBuildpack { log_header("Determining Python version"); let python_version = python_version::determine_python_version(&context.app_dir) .map_err(BuildpackError::PythonVersion)?; + let packaging_tool_versions = PackagingToolVersions::default(); // We inherit the current process's env vars, since we want `PATH` and `HOME` to be set // so that later commands can find tools like Git in the stack image. Any user-provided @@ -68,11 +69,13 @@ impl Buildpack for PythonBuildpack { let mut command_env = Env::from_current(); // Create the layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`. + log_header("Installing Python and packaging tools"); let python_layer = context.handle_layer( layer_name!("python"), PythonLayer { command_env: &command_env, python_version: &python_version, + packaging_tool_versions: &packaging_tool_versions, }, )?; command_env = python_layer.env.apply(Scope::Build, &command_env); @@ -86,6 +89,7 @@ impl Buildpack for PythonBuildpack { layer_name!("pip-cache"), PipCacheLayer { python_version: &python_version, + packaging_tool_versions: &packaging_tool_versions, }, )?; let pip_layer = context.handle_layer( diff --git a/src/package_manager.rs b/src/package_manager.rs index 357a186..803de33 100644 --- a/src/package_manager.rs +++ b/src/package_manager.rs @@ -1,6 +1,8 @@ use std::io; use std::path::Path; +use serde::{Deserialize, Serialize}; + /// A ordered mapping of project filenames to their associated package manager. /// Earlier entries will take precedence if a project matches multiple package managers. pub(crate) const PACKAGE_MANAGER_FILE_MAPPING: [(&str, PackageManager); 1] = @@ -39,6 +41,23 @@ pub(crate) enum DeterminePackageManagerError { NoneFound, } +#[derive(Clone, Deserialize, PartialEq, Serialize)] +pub(crate) struct PackagingToolVersions { + pub pip_version: String, + pub setuptools_version: String, + pub wheel_version: String, +} + +impl Default for PackagingToolVersions { + fn default() -> Self { + Self { + pip_version: "23.0.1".to_string(), + setuptools_version: "67.4.0".to_string(), + wheel_version: "0.38.4".to_string(), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/integration.rs b/tests/integration.rs index 6a037e1..3fd9fe6 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -23,7 +23,7 @@ const LATEST_PYTHON_3_11: &str = "3.11.2"; const DEFAULT_PYTHON_VERSION: &str = LATEST_PYTHON_3_11; const PIP_VERSION: &str = "23.0.1"; -const SETUPTOOLS_VERSION: &str = "67.3.2"; +const SETUPTOOLS_VERSION: &str = "67.4.0"; const WHEEL_VERSION: &str = "0.38.4"; const DEFAULT_BUILDER: &str = "heroku/builder:22"; @@ -155,22 +155,16 @@ fn python_version_unspecified() { No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes - [Installing Python] - Downloading Python {DEFAULT_PYTHON_VERSION} - Python installation successful - - [Installing Pip] + [Installing Python and packaging tools] + Installing Python {DEFAULT_PYTHON_VERSION} Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - Installation completed [Installing dependencies using Pip] - Pip cache created Running pip install Collecting typing-extensions==4.4.0 Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) Installing collected packages: typing-extensions Successfully installed typing-extensions-4.4.0 - Pip install completed ===> EXPORTING "} ); @@ -234,22 +228,16 @@ fn builds_with_python_version(fixture_path: &str, python_version: &str) { [Determining Python version] Using Python version {python_version} specified in runtime.txt - [Installing Python] - Downloading Python {python_version} - Python installation successful - - [Installing Pip] + [Installing Python and packaging tools] + Installing Python {python_version} Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - Installation completed [Installing dependencies using Pip] - Pip cache created Running pip install Collecting typing-extensions==4.4.0 Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) Installing collected packages: typing-extensions Successfully installed typing-extensions-4.4.0 - Pip install completed ===> EXPORTING "} ); @@ -375,7 +363,6 @@ fn pip_invalid_requirement() { context.pack_stdout, &formatdoc! {" [Installing dependencies using Pip] - Pip cache created Running pip install "} ); @@ -413,20 +400,17 @@ fn cache_used_for_repeat_builds() { [Determining Python version] Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt - [Installing Python] - Re-using cached Python {LATEST_PYTHON_3_11} - - [Installing Pip] - Re-using cached pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} + [Installing Python and packaging tools] + Using cached Python {LATEST_PYTHON_3_11} + Using cached pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} [Installing dependencies using Pip] - Re-using cached pip-cache + Using cached pip download/wheel cache Running pip install Collecting typing-extensions==4.4.0 Using cached typing_extensions-4.4.0-py3-none-any.whl (26 kB) Installing collected packages: typing-extensions Successfully installed typing-extensions-4.4.0 - Pip install completed ===> EXPORTING "} ); @@ -451,26 +435,19 @@ fn cache_discarded_on_python_version_change() { [Determining Python version] Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt - Discarding cached Python {LATEST_PYTHON_3_10} - Discarding cached pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - [Installing Python] - Downloading Python {LATEST_PYTHON_3_11} - Python installation successful - - [Installing Pip] + [Installing Python and packaging tools] + Discarding cache since the Python version has changed from {LATEST_PYTHON_3_10} to {LATEST_PYTHON_3_11} + Installing Python {LATEST_PYTHON_3_11} Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - Installation completed [Installing dependencies using Pip] - Discarding cached pip-cache - Pip cache created + Discarding cached pip download/wheel cache Running pip install Collecting typing-extensions==4.4.0 Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) Installing collected packages: typing-extensions Successfully installed typing-extensions-4.4.0 - Pip install completed ===> EXPORTING "} ); @@ -496,26 +473,57 @@ fn cache_discarded_on_stack_change() { [Determining Python version] No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes - Discarding cached Python {DEFAULT_PYTHON_VERSION} - Discarding cached pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - [Installing Python] - Downloading Python {DEFAULT_PYTHON_VERSION} - Python installation successful + [Installing Python and packaging tools] + Discarding cache since the stack has changed from heroku-20 to heroku-22 + Installing Python {DEFAULT_PYTHON_VERSION} + Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - [Installing Pip] + [Installing dependencies using Pip] + Discarding cached pip download/wheel cache + Running pip install + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + ===> EXPORTING + "} + ); + }); + }); +} + +#[test] +#[ignore = "integration test"] +fn cache_discarded_on_multiple_changes() { + let config_before = BuildConfig::new("heroku/buildpacks:20", "tests/fixtures/python_3.10"); + let config_after = BuildConfig::new("heroku/builder:22", "tests/fixtures/python_3.11"); + + TestRunner::default().build(config_before, |context| { + context.rebuild(config_after, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt + + [Installing Python and packaging tools] + Discarding cache since: + - the stack has changed from heroku-20 to heroku-22 + - the Python version has changed from 3.10.10 to 3.11.2 + Installing Python {LATEST_PYTHON_3_11} Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - Installation completed [Installing dependencies using Pip] - Discarding cached pip-cache - Pip cache created + Discarding cached pip download/wheel cache Running pip install Collecting typing-extensions==4.4.0 Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) Installing collected packages: typing-extensions Successfully installed typing-extensions-4.4.0 - Pip install completed ===> EXPORTING "} ); @@ -535,8 +543,6 @@ fn salesforce_function_template() { assert_contains!( context.pack_stdout, indoc! {" - Pip install completed - [Validating Salesforce Function] Function passed validation. ===> EXPORTING From 583b04f771967ce8188979868a464f2f828c0192 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 27 Feb 2023 11:56:22 +0000 Subject: [PATCH 57/71] Refactor Python runtime archive URL generation --- src/layers/python.rs | 8 ++------ src/python_version.rs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/layers/python.rs b/src/layers/python.rs index e26f4d2..12e1a3e 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -43,13 +43,9 @@ impl Layer for PythonLayer<'_> { context: &BuildContext, layer_path: &Path, ) -> Result, ::Error> { - // TODO: Move this URL generation somewhere else (ie manifest etc). - let archive_url = format!( - "https://heroku-buildpack-python.s3.us-east-1.amazonaws.com/{}/runtimes/python-{}.tar.gz", - context.stack_id, self.python_version - ); - log_info(format!("Installing Python {}", self.python_version)); + + let archive_url = self.python_version.url(&context.stack_id); utils::download_and_unpack_gzipped_archive(&archive_url, layer_path).map_err(|error| { match error { // TODO: Remove this once the Python version is validated against a manifest (at which diff --git a/src/python_version.rs b/src/python_version.rs index 8052e22..813daf9 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -1,5 +1,6 @@ use crate::runtime_txt::{self, RuntimeTxtError}; use indoc::formatdoc; +use libcnb::data::buildpack::StackId; use libherokubuildpack::log::log_info; use std::fmt::{self, Display}; use std::path::Path; @@ -27,6 +28,13 @@ impl PythonVersion { patch, } } + + pub fn url(&self, stack_id: &StackId) -> String { + format!( + "https://heroku-buildpack-python.s3.us-east-1.amazonaws.com/{}/runtimes/python-{}.{}.{}.tar.gz", + stack_id, self.major, self.minor, self.patch + ) + } } impl Display for PythonVersion { @@ -69,8 +77,18 @@ pub(crate) enum PythonVersionError { #[cfg(test)] mod tests { + use libcnb::data::stack_id; + use super::*; + #[test] + fn python_version_url() { + assert_eq!( + PythonVersion::new(3, 11, 0).url(&stack_id!("heroku-22")), + "https://heroku-buildpack-python.s3.us-east-1.amazonaws.com/heroku-22/runtimes/python-3.11.0.tar.gz" + ); + } + #[test] fn determine_python_version_runtime_txt_valid() { assert_eq!( From 2a92049ca256651c9091bbce050e99fd2831ad09 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 27 Feb 2023 12:10:49 +0000 Subject: [PATCH 58/71] Update dependencies --- Cargo.lock | 191 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 128 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f1f553..4953594 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,9 +109,9 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "camino" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77df041dc383319cc661b428b6961a005db4d6808d5e12536931b1ca9556055" +checksum = "6031a462f977dd38968b6f23378356512feeace69cef817e1a4475108093cec3" dependencies = [ "serde", ] @@ -191,9 +191,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc831ee6a32dd495436e317595e639a587aa9907bef96fe6e6abc290ab6204e9" +checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" dependencies = [ "cc", "cxxbridge-flags", @@ -203,9 +203,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94331d54f1b1a8895cd81049f7eaaaef9d05a7dcb4d1fd08bf3ff0806246789d" +checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" dependencies = [ "cc", "codespan-reporting", @@ -218,15 +218,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48dcd35ba14ca9b40d6e4b4b39961f23d835dbb8eed74565ded361d93e1feb8a" +checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" [[package]] name = "cxxbridge-macro" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bbeb29798b407ccd82a3324ade1a7286e0d29851475990b612670f6f5124d2" +checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ "proc-macro2", "quote", @@ -239,6 +239,27 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "fancy-regex" version = "0.11.0" @@ -251,23 +272,23 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] [[package]] name = "filetime" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9" +checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -396,9 +417,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -525,6 +546,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-lifetimes" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +dependencies = [ + "libc", + "windows-sys 0.45.0", +] + [[package]] name = "itoa" version = "1.0.5" @@ -643,6 +674,12 @@ dependencies = [ "cc", ] +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + [[package]] name = "log" version = "0.4.17" @@ -669,23 +706,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi", - "windows-sys", -] - -[[package]] -name = "nom8" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" -dependencies = [ - "memchr", + "windows-sys 0.45.0", ] [[package]] @@ -719,9 +747,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "percent-encoding" @@ -824,15 +852,6 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - [[package]] name = "ring" version = "0.16.20" @@ -848,6 +867,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustix" +version = "0.36.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.45.0", +] + [[package]] name = "rustls" version = "0.20.8" @@ -913,9 +946,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" dependencies = [ "itoa", "ryu", @@ -971,9 +1004,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -996,9 +1029,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", @@ -1018,16 +1051,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if", "fastrand", - "libc", "redox_syscall", - "remove_dir_all", - "winapi", + "rustix", + "windows-sys 0.42.0", ] [[package]] @@ -1061,9 +1093,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", "serde", @@ -1079,9 +1111,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -1115,14 +1147,14 @@ dependencies = [ "num_cpus", "pin-project-lite", "socket2", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite", @@ -1131,9 +1163,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -1166,15 +1198,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.3" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6a7712b49e1775fb9a7b998de6635b299237f48b404dde71704f2e0e7f37e5" +checksum = "9a1eb0622d28f4b9c90adc4ea4b2b46b47663fde9ac5fafcb14a1369d5508825" dependencies = [ "indexmap", - "nom8", "serde", "serde_spanned", "toml_datetime", + "winnow", ] [[package]] @@ -1430,6 +1462,30 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.1" @@ -1472,6 +1528,15 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "winnow" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658" +dependencies = [ + "memchr", +] + [[package]] name = "xattr" version = "0.2.3" From 9bc83dd1f6974c7f50086f9c63c7cb454627f584 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 27 Feb 2023 12:17:27 +0000 Subject: [PATCH 59/71] Refactor PackagingToolVersions --- src/layers/pip_cache.rs | 2 +- src/layers/python.rs | 2 +- src/main.rs | 4 +++- src/package_manager.rs | 19 ------------------- src/packaging_tool_versions.rs | 20 ++++++++++++++++++++ 5 files changed, 25 insertions(+), 22 deletions(-) create mode 100644 src/packaging_tool_versions.rs diff --git a/src/layers/pip_cache.rs b/src/layers/pip_cache.rs index d4d1595..ef1cb39 100644 --- a/src/layers/pip_cache.rs +++ b/src/layers/pip_cache.rs @@ -1,4 +1,4 @@ -use crate::package_manager::PackagingToolVersions; +use crate::packaging_tool_versions::PackagingToolVersions; use crate::python_version::PythonVersion; use crate::PythonBuildpack; use libcnb::build::BuildContext; diff --git a/src/layers/python.rs b/src/layers/python.rs index 12e1a3e..6b06cd3 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -1,4 +1,4 @@ -use crate::package_manager::PackagingToolVersions; +use crate::packaging_tool_versions::PackagingToolVersions; use crate::python_version::PythonVersion; use crate::utils::{self, CommandError, DownloadUnpackArchiveError}; use crate::{BuildpackError, PythonBuildpack}; diff --git a/src/main.rs b/src/main.rs index 4c7ee42..81b6af4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod errors; mod layers; mod package_manager; +mod packaging_tool_versions; mod project_descriptor; mod python_version; mod runtime_txt; @@ -17,7 +18,8 @@ mod utils; use crate::layers::pip_cache::PipCacheLayer; use crate::layers::pip_dependencies::{PipDependenciesLayer, PipDependenciesLayerError}; use crate::layers::python::{PythonLayer, PythonLayerError}; -use crate::package_manager::{DeterminePackageManagerError, PackageManager, PackagingToolVersions}; +use crate::package_manager::{DeterminePackageManagerError, PackageManager}; +use crate::packaging_tool_versions::PackagingToolVersions; use crate::project_descriptor::ProjectDescriptorError; use crate::python_version::PythonVersionError; use crate::salesforce_functions::CheckSalesforceFunctionError; diff --git a/src/package_manager.rs b/src/package_manager.rs index 803de33..357a186 100644 --- a/src/package_manager.rs +++ b/src/package_manager.rs @@ -1,8 +1,6 @@ use std::io; use std::path::Path; -use serde::{Deserialize, Serialize}; - /// A ordered mapping of project filenames to their associated package manager. /// Earlier entries will take precedence if a project matches multiple package managers. pub(crate) const PACKAGE_MANAGER_FILE_MAPPING: [(&str, PackageManager); 1] = @@ -41,23 +39,6 @@ pub(crate) enum DeterminePackageManagerError { NoneFound, } -#[derive(Clone, Deserialize, PartialEq, Serialize)] -pub(crate) struct PackagingToolVersions { - pub pip_version: String, - pub setuptools_version: String, - pub wheel_version: String, -} - -impl Default for PackagingToolVersions { - fn default() -> Self { - Self { - pip_version: "23.0.1".to_string(), - setuptools_version: "67.4.0".to_string(), - wheel_version: "0.38.4".to_string(), - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/packaging_tool_versions.rs b/src/packaging_tool_versions.rs new file mode 100644 index 0000000..8ee281a --- /dev/null +++ b/src/packaging_tool_versions.rs @@ -0,0 +1,20 @@ +use serde::{Deserialize, Serialize}; + +/// The versions of various packaging tools used during the build. +/// These are always installed, and are independent of the chosen package manager. +#[derive(Clone, Deserialize, PartialEq, Serialize)] +pub(crate) struct PackagingToolVersions { + pub pip_version: String, + pub setuptools_version: String, + pub wheel_version: String, +} + +impl Default for PackagingToolVersions { + fn default() -> Self { + Self { + pip_version: "23.0.1".to_string(), + setuptools_version: "67.4.0".to_string(), + wheel_version: "0.38.4".to_string(), + } + } +} From 90489968ad5ad4ee78befb9ca08c4aaa7cf466a0 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:36:24 +0000 Subject: [PATCH 60/71] More rustdocs --- src/layers/pip_dependencies.rs | 4 ++-- src/layers/python.rs | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/layers/pip_dependencies.rs b/src/layers/pip_dependencies.rs index e0fea8c..71eaa0e 100644 --- a/src/layers/pip_dependencies.rs +++ b/src/layers/pip_dependencies.rs @@ -64,8 +64,8 @@ impl Layer for PipDependenciesLayer<'_> { // the repository around, since the directory is added to the Python path directly (via // the `.pth` file created in `site-packages`). By default Pip will store the repository // in the current working directory (the app dir), however, we would prefer it to be stored - // in the dependencies layer instead for consistency. (Plus if this layer were ever cached, - // storing the repository in the app dir would break on repeat-builds). + // in the dependencies layer instead for consistency. (Plus if the dependencies layer were + // ever cached, storing the repository in the app dir would break on repeat-builds). let src_dir = layer_path.join("src"); fs::create_dir(&src_dir).map_err(PipDependenciesLayerError::CreateSrcDirIo)?; diff --git a/src/layers/python.rs b/src/layers/python.rs index 6b06cd3..3efdd29 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -17,6 +17,19 @@ use std::process::Command; use std::{fs, io}; /// Layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`. +/// +/// We install both Python and the packaging tools into the same layer, since: +/// - We don't want to mix buildpack/packaging dependencies with the app's own dependencies +/// (for a start, we need pip installed to even install the user's own dependencies, plus +/// want to keep caching separate), so cannot install the packaging tools into the user +/// site-packages directory. +/// - We don't want to install the packaging tools into an arbitrary directory added to +/// `PYTHONPATH`, since directories added to `PYTHONPATH` take precedence over the Python +/// stdlib (unlike the system or user site-packages directories), and so can result in hard +/// to debug stdlib shadowing problems that users won't encounter locally. +/// - This leaves just the system site-packages directory, which exists within the Python +/// installation directory and Python does not support moving it elsewhere. +/// - It matches what both local and official Docker image environments do. pub(crate) struct PythonLayer<'a> { /// Environment variables inherited from earlier buildpack steps. pub command_env: &'a Env, From c60266619adb8e39ef28deae78e2feb17d39d8e7 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 28 Feb 2023 10:22:46 +0000 Subject: [PATCH 61/71] Refactor integration tests - Split them out into separate files - Import them into the crate so private APIs can be used Inspired by: https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html http://xion.io/post/code/rust-unit-test-placement.html https://doc.rust-lang.org/reference/items/modules.html#the-path-attribute --- src/main.rs | 10 +- tests/integration.rs | 662 ---------------------- tests/integration/detect.rs | 36 ++ tests/integration/mod.rs | 23 + tests/integration/package_manager.rs | 28 + tests/integration/pip.rs | 229 ++++++++ tests/integration/python_version.rs | 252 ++++++++ tests/integration/salesforce_functions.rs | 137 +++++ 8 files changed, 711 insertions(+), 666 deletions(-) delete mode 100644 tests/integration.rs create mode 100644 tests/integration/detect.rs create mode 100644 tests/integration/mod.rs create mode 100644 tests/integration/package_manager.rs create mode 100644 tests/integration/pip.rs create mode 100644 tests/integration/python_version.rs create mode 100644 tests/integration/salesforce_functions.rs diff --git a/src/main.rs b/src/main.rs index 81b6af4..d57d13b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -151,8 +151,10 @@ impl From for libcnb::Error { buildpack_main!(PythonBuildpack); +// The integration tests are imported into the crate so that they can have access to private +// APIs and constants, saving having to (a) run a dual binary/library crate, (b) expose APIs +// publicly for things only used for testing. See: +// https://doc.rust-lang.org/reference/items/modules.html#the-path-attribute #[cfg(test)] -mod tests { - // Suppress warnings due to the `unused_crate_dependencies` lint not handling integration tests well. - use libcnb_test as _; -} +#[path = "../tests/integration/mod.rs"] +mod integration_tests; diff --git a/tests/integration.rs b/tests/integration.rs deleted file mode 100644 index 3fd9fe6..0000000 --- a/tests/integration.rs +++ /dev/null @@ -1,662 +0,0 @@ -//! All integration tests are skipped by default (using the `ignore` attribute), -//! since performing builds is slow. To run the tests use: `cargo test -- --ignored` - -#![warn(clippy::pedantic)] - -use indoc::{formatdoc, indoc}; -use libcnb::data::buildpack::{BuildpackVersion, SingleBuildpackDescriptor}; -use libcnb_test::{ - assert_contains, assert_empty, BuildConfig, ContainerConfig, PackResult, TestRunner, -}; -use std::time::Duration; -use std::{env, fs, thread}; - -// At the moment these can't be imported from the buildpack, since integration -// tests cannot access any interfaces for binary-only crates. -// TODO: Explore moving integration tests into the crate, per: -// https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html -const LATEST_PYTHON_3_7: &str = "3.7.16"; -const LATEST_PYTHON_3_8: &str = "3.8.16"; -const LATEST_PYTHON_3_9: &str = "3.9.16"; -const LATEST_PYTHON_3_10: &str = "3.10.10"; -const LATEST_PYTHON_3_11: &str = "3.11.2"; -const DEFAULT_PYTHON_VERSION: &str = LATEST_PYTHON_3_11; - -const PIP_VERSION: &str = "23.0.1"; -const SETUPTOOLS_VERSION: &str = "67.4.0"; -const WHEEL_VERSION: &str = "0.38.4"; - -const DEFAULT_BUILDER: &str = "heroku/builder:22"; -const TEST_PORT: u16 = 12345; - -fn builder() -> String { - env::var("INTEGRATION_TEST_CNB_BUILDER").unwrap_or(DEFAULT_BUILDER.to_string()) -} - -fn buildpack_version() -> BuildpackVersion { - let buildpack_toml = fs::read_to_string("buildpack.toml").unwrap(); - let buildpack_descriptor = - toml::from_str::>>(&buildpack_toml).unwrap(); - buildpack_descriptor.buildpack.version -} - -// Detect - -#[test] -#[ignore = "integration test"] -fn detect_rejects_non_python_projects() { - let buildpack_version = buildpack_version(); - - TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/empty") - .expected_pack_result(PackResult::Failure), - |context| { - assert_contains!( - context.pack_stdout, - &formatdoc! {" - ===> DETECTING - ======== Output: heroku/python@{buildpack_version} ======== - No Python project files found (such as requirements.txt). - ======== Results ======== - fail: heroku/python@{buildpack_version} - ERROR: No buildpack groups passed detection. - "} - ); - }, - ); -} - -// Determine package manager - -#[test] -#[ignore = "integration test"] -fn no_package_manager_detected() { - TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/pyproject_toml_only") - .expected_pack_result(PackResult::Failure), - |context| { - assert_contains!( - context.pack_stderr, - indoc! {" - [Error: No Python package manager files were found] - A Pip requirements file was not found in your application's source code. - This file is required so that your application's dependencies can be installed. - - Please add a file named exactly 'requirements.txt' to the root directory of your - application, containing a list of the packages required by your application. - - For more information on what this file should contain, see: - https://pip.pypa.io/en/stable/reference/requirements-file-format/ - "} - ); - }, - ); -} - -// runtime.txt parsing - -#[test] -#[ignore = "integration test"] -fn runtime_txt_invalid_version() { - TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/runtime_txt_invalid_version") - .expected_pack_result(PackResult::Failure), - |context| { - assert_contains!( - context.pack_stderr, - &formatdoc! {" - [Error: Invalid Python version in runtime.txt] - The Python version specified in 'runtime.txt' is not in the correct format. - - The following file contents were found: - python-an.invalid.version - - However, the file contents must begin with a 'python-' prefix, followed by the - version specified as '..'. Comments are not supported. - - For example, to request Python {DEFAULT_PYTHON_VERSION}, the correct version format is: - python-{DEFAULT_PYTHON_VERSION} - - Please update 'runtime.txt' to use the correct version format, or else remove - the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). - - For a list of the supported Python versions, see: - https://devcenter.heroku.com/articles/python-support#supported-runtimes - "} - ); - }, - ); -} - -#[test] -#[ignore = "integration test"] -fn runtime_txt_non_existent_version() { - rejects_non_existent_python_version( - "tests/fixtures/runtime_txt_non_existent_version", - "999.999.999", - ); -} - -// Python versions - -#[test] -#[ignore = "integration test"] -fn python_version_unspecified() { - TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/python_version_unspecified"), - |context| { - assert_empty!(context.pack_stderr); - assert_contains!( - context.pack_stdout, - &formatdoc! {" - ===> BUILDING - - [Determining Python version] - No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. - To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes - - [Installing Python and packaging tools] - Installing Python {DEFAULT_PYTHON_VERSION} - Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - - [Installing dependencies using Pip] - Running pip install - Collecting typing-extensions==4.4.0 - Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) - Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.4.0 - ===> EXPORTING - "} - ); - }, - ); -} - -#[test] -#[ignore = "integration test"] -fn python_3_7() { - // Python 3.7 is only available on Heroku-20 and older. - let fixture = "tests/fixtures/python_3.7"; - match builder().as_str() { - "heroku/buildpacks:20" => builds_with_python_version(fixture, LATEST_PYTHON_3_7), - _ => rejects_non_existent_python_version(fixture, LATEST_PYTHON_3_7), - }; -} - -#[test] -#[ignore = "integration test"] -fn python_3_8() { - // Python 3.8 is only available on Heroku-20 and older. - let fixture = "tests/fixtures/python_3.8"; - match builder().as_str() { - "heroku/buildpacks:20" => builds_with_python_version(fixture, LATEST_PYTHON_3_8), - _ => rejects_non_existent_python_version(fixture, LATEST_PYTHON_3_8), - }; -} - -#[test] -#[ignore = "integration test"] -fn python_3_9() { - builds_with_python_version("tests/fixtures/python_3.9", LATEST_PYTHON_3_9); -} - -#[test] -#[ignore = "integration test"] -fn python_3_10() { - builds_with_python_version("tests/fixtures/python_3.10", LATEST_PYTHON_3_10); -} - -#[test] -#[ignore = "integration test"] -fn python_3_11() { - builds_with_python_version("tests/fixtures/python_3.11", LATEST_PYTHON_3_11); -} - -fn builds_with_python_version(fixture_path: &str, python_version: &str) { - let mut config = BuildConfig::new(builder(), fixture_path); - // Checks that potentially broken user-provided env vars are not being passed unfiltered to - // subprocesses we launch (such as `pip install`), thanks to `clear-env` in `buildpack.toml`. - config.env("PYTHONHOME", "/invalid-path"); - - TestRunner::default().build(config, |context| { - assert_empty!(context.pack_stderr); - assert_contains!( - context.pack_stdout, - &formatdoc! {" - ===> BUILDING - - [Determining Python version] - Using Python version {python_version} specified in runtime.txt - - [Installing Python and packaging tools] - Installing Python {python_version} - Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - - [Installing dependencies using Pip] - Running pip install - Collecting typing-extensions==4.4.0 - Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) - Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.4.0 - ===> EXPORTING - "} - ); - // There's no sensible default process type we can set for Python apps. - assert_contains!(context.pack_stdout, "no default process type"); - - // Validate the Python/Pip install works as expected at runtime. - let command_output = context.run_shell_command( - indoc! {r#" - set -euo pipefail - - # Check that we installed the correct Python version, and that the command - # 'python' works (since it's a symlink to the actual 'python3' binary). - python --version - - # Check that the Python binary is using its own 'libpython' and not the system one: - # https://github.com/docker-library/python/issues/784 - # Note: This has to handle Python 3.9 and older not being built in shared library mode. - libpython_path=$(ldd /layers/heroku_python/python/bin/python | grep libpython || true) - if [[ -n "${libpython_path}" && "${libpython_path}" != *"=> /layers/"* ]]; then - echo "The Python binary is not using the correct libpython!" - echo "${libpython_path}" - exit 1 - fi - - # Check all required dynamically linked libraries can be found in the runtime image. - if find /layers -name '*.so' -exec ldd '{}' + | grep 'not found'; then - echo "The above dynamically linked libraries were not found!" - exit 1 - fi - - # Check that: - # - Pip is available at runtime too (and not just during the build). - # - The correct versions of pip/setuptools/wheel were installed. - # - Pip uses (via 'PYTHONUSERBASE') the user site-packages in the dependencies - # layer, and so can find the typing-extensions package installed there. - # - The "pip update available" warning is not shown (since it should be suppressed). - # - The system site-packages directory is protected against running 'pip install' - # without having passed '--user'. - pip list - pip install --dry-run typing-extensions - "#} - ); - assert_empty!(command_output.stderr); - assert_contains!( - command_output.stdout, - &formatdoc! {" - Python {python_version} - Package Version - ----------------- ------- - pip {PIP_VERSION} - setuptools {SETUPTOOLS_VERSION} - typing_extensions 4.4.0 - wheel {WHEEL_VERSION} - Defaulting to user installation because normal site-packages is not writeable - Requirement already satisfied: typing-extensions in /layers/heroku_python/dependencies/lib/" - } - ); - }); -} - -fn rejects_non_existent_python_version(fixture_path: &str, python_version: &str) { - let builder = builder(); - - TestRunner::default().build( - BuildConfig::new(&builder, fixture_path).expected_pack_result(PackResult::Failure), - |context| { - let expected_stack = match builder.as_str() { - "heroku/buildpacks:20" => "heroku-20", - "heroku/builder:22" => "heroku-22", - _ => unimplemented!("Unknown builder!"), - }; - - assert_contains!( - context.pack_stderr, - &formatdoc! {" - [Error: Requested Python version is not available] - The requested Python version ({python_version}) is not available for this stack ({expected_stack}). - - Please update the version in 'runtime.txt' to a supported Python version, or else - remove the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). - - For a list of the supported Python versions, see: - https://devcenter.heroku.com/articles/python-support#supported-runtimes - "} - ); - }, - ); -} - -// Pip - -#[test] -#[ignore = "integration test"] -fn pip_editable_git_compiled() { - // This tests that: - // - Git from the stack image can be found (ie: the system PATH has been correctly propagated to pip). - // - The editable mode repository clone is saved into the dependencies layer not the app dir. - // - Compiling a source distribution package (as opposed to a pre-built wheel) works. - // - The Python headers can be found in the `include/pythonX.Y/` directory of the Python layer. - // - Headers/libraries from the stack image can be found (in this case, for libpq-dev). - TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/pip_editable_git_compiled"), - |context| { - assert_contains!( - context.pack_stdout, - "Cloning https://github.com/psycopg/psycopg2 (to revision 2_9_5) to /layers/heroku_python/dependencies/src/psycopg2" - ); - }, - ); -} - -#[test] -#[ignore = "integration test"] -fn pip_invalid_requirement() { - TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/pip_invalid_requirement") - .expected_pack_result(PackResult::Failure), - |context| { - // Ideally we could test a combined stdout/stderr, however libcnb-test doesn't support this: - // https://github.com/heroku/libcnb.rs/issues/536 - assert_contains!( - context.pack_stdout, - &formatdoc! {" - [Installing dependencies using Pip] - Running pip install - "} - ); - assert_contains!( - context.pack_stderr, - &formatdoc! {" - ERROR: Invalid requirement: 'an-invalid-requirement!' (from line 1 of requirements.txt) - - [Error: Unable to install dependencies using pip] - The 'pip install' command to install the application's dependencies from - 'requirements.txt' failed (exit status: 1). - - See the log output above for more information. - "} - ); - }, - ); -} - -// Caching - -#[test] -#[ignore = "integration test"] -fn cache_used_for_repeat_builds() { - let config = BuildConfig::new(builder(), "tests/fixtures/python_3.11"); - - TestRunner::default().build(&config, |context| { - context.rebuild(&config, |rebuild_context| { - assert_empty!(rebuild_context.pack_stderr); - assert_contains!( - rebuild_context.pack_stdout, - &formatdoc! {" - ===> BUILDING - - [Determining Python version] - Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt - - [Installing Python and packaging tools] - Using cached Python {LATEST_PYTHON_3_11} - Using cached pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - - [Installing dependencies using Pip] - Using cached pip download/wheel cache - Running pip install - Collecting typing-extensions==4.4.0 - Using cached typing_extensions-4.4.0-py3-none-any.whl (26 kB) - Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.4.0 - ===> EXPORTING - "} - ); - }); - }); -} - -#[test] -#[ignore = "integration test"] -fn cache_discarded_on_python_version_change() { - let builder = builder(); - let config_before = BuildConfig::new(&builder, "tests/fixtures/python_3.10"); - let config_after = BuildConfig::new(&builder, "tests/fixtures/python_3.11"); - - TestRunner::default().build(config_before, |context| { - context.rebuild(config_after, |rebuild_context| { - assert_empty!(rebuild_context.pack_stderr); - assert_contains!( - rebuild_context.pack_stdout, - &formatdoc! {" - ===> BUILDING - - [Determining Python version] - Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt - - [Installing Python and packaging tools] - Discarding cache since the Python version has changed from {LATEST_PYTHON_3_10} to {LATEST_PYTHON_3_11} - Installing Python {LATEST_PYTHON_3_11} - Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - - [Installing dependencies using Pip] - Discarding cached pip download/wheel cache - Running pip install - Collecting typing-extensions==4.4.0 - Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) - Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.4.0 - ===> EXPORTING - "} - ); - }); - }); -} - -#[test] -#[ignore = "integration test"] -fn cache_discarded_on_stack_change() { - let fixture = "tests/fixtures/python_version_unspecified"; - let config_before = BuildConfig::new("heroku/buildpacks:20", fixture); - let config_after = BuildConfig::new("heroku/builder:22", fixture); - - TestRunner::default().build(config_before, |context| { - context.rebuild(config_after, |rebuild_context| { - assert_empty!(rebuild_context.pack_stderr); - assert_contains!( - rebuild_context.pack_stdout, - &formatdoc! {" - ===> BUILDING - - [Determining Python version] - No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. - To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes - - [Installing Python and packaging tools] - Discarding cache since the stack has changed from heroku-20 to heroku-22 - Installing Python {DEFAULT_PYTHON_VERSION} - Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - - [Installing dependencies using Pip] - Discarding cached pip download/wheel cache - Running pip install - Collecting typing-extensions==4.4.0 - Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) - Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.4.0 - ===> EXPORTING - "} - ); - }); - }); -} - -#[test] -#[ignore = "integration test"] -fn cache_discarded_on_multiple_changes() { - let config_before = BuildConfig::new("heroku/buildpacks:20", "tests/fixtures/python_3.10"); - let config_after = BuildConfig::new("heroku/builder:22", "tests/fixtures/python_3.11"); - - TestRunner::default().build(config_before, |context| { - context.rebuild(config_after, |rebuild_context| { - assert_empty!(rebuild_context.pack_stderr); - assert_contains!( - rebuild_context.pack_stdout, - &formatdoc! {" - ===> BUILDING - - [Determining Python version] - Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt - - [Installing Python and packaging tools] - Discarding cache since: - - the stack has changed from heroku-20 to heroku-22 - - the Python version has changed from 3.10.10 to 3.11.2 - Installing Python {LATEST_PYTHON_3_11} - Installing pip {PIP_VERSION}, setuptools {SETUPTOOLS_VERSION} and wheel {WHEEL_VERSION} - - [Installing dependencies using Pip] - Discarding cached pip download/wheel cache - Running pip install - Collecting typing-extensions==4.4.0 - Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) - Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.4.0 - ===> EXPORTING - "} - ); - }); - }); -} - -// Salesforce Functions - -#[test] -#[ignore = "integration test"] -fn salesforce_function_template() { - TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/salesforce_function_template"), - |context| { - assert_empty!(context.pack_stderr); - assert_contains!( - context.pack_stdout, - indoc! {" - [Validating Salesforce Function] - Function passed validation. - ===> EXPORTING - "} - ); - assert_contains!(context.pack_stdout, "Setting default process type 'web'"); - - // Test that the `sf-functions-python` web process the buildpack configures works correctly. - context.start_container( - ContainerConfig::new() - .env("PORT", TEST_PORT.to_string()) - .expose_port(TEST_PORT), - |container| { - let address_on_host = container.address_for_port(TEST_PORT).unwrap(); - let url = format!("http://{}:{}", address_on_host.ip(), address_on_host.port()); - - // Retries needed since the server takes a moment to start up. - let mut attempts_remaining = 5; - let response = loop { - let response = ureq::post(&url).set("x-health-check", "true").call(); - if response.is_ok() || attempts_remaining == 0 { - break response; - } - attempts_remaining -= 1; - thread::sleep(Duration::from_secs(1)); - }; - - let server_log_output = container.logs_now(); - assert_contains!( - server_log_output.stderr, - &format!("Uvicorn running on http://0.0.0.0:{TEST_PORT}") - ); - - let body = response.unwrap().into_string().unwrap(); - assert_eq!(body, r#""OK""#); - }, - ); - }, - ); -} - -#[test] -#[ignore = "integration test"] -fn salesforce_function_missing_package() { - TestRunner::default().build( - BuildConfig::new( - builder(), - "tests/fixtures/salesforce_function_missing_package", - ) - .expected_pack_result(PackResult::Failure), - |context| { - assert_contains!( - context.pack_stderr, - indoc! {r#" - [Error: The Salesforce Functions package is not installed] - The 'sf-functions-python' program that is required for Python Salesforce - Functions could not be found. - - Check that the 'salesforce-functions' Python package is listed as a - dependency in 'requirements.txt'. - - If this project is not intended to be a Salesforce Function, remove the - 'type = "function"' declaration from 'project.toml' to skip this check. - "#} - ); - }, - ); -} - -#[test] -#[ignore = "integration test"] -fn salesforce_function_fails_self_check() { - TestRunner::default().build( - BuildConfig::new( - builder(), - "tests/fixtures/salesforce_function_fails_self_check", - ) - .expected_pack_result(PackResult::Failure), - |context| { - assert_contains!( - context.pack_stderr, - indoc! {" - [Error: The Salesforce Functions self-check failed] - The 'sf-functions-python check' command failed (exit status: 1), indicating - there is a problem with the Python Salesforce Function in this project. - - Details: - Function failed validation: 'invalid' isn't a valid Salesforce REST API version." - } - ); - }, - ); -} - -#[test] -#[ignore = "integration test"] -fn project_toml_invalid() { - TestRunner::default().build( - BuildConfig::new(builder(), "tests/fixtures/project_toml_invalid") - .expected_pack_result(PackResult::Failure), - |context| { - assert_contains!( - context.pack_stderr, - indoc! {r#" - [Error: Invalid project.toml] - A parsing/validation error error occurred whilst loading the project.toml file. - - Details: TOML parse error at line 4, column 1 - | - 4 | [com.salesforce] - | ^^^^^^^^^^^^^^^^ - missing field `type` - "#} - ); - }, - ); -} diff --git a/tests/integration/detect.rs b/tests/integration/detect.rs new file mode 100644 index 0000000..b227483 --- /dev/null +++ b/tests/integration/detect.rs @@ -0,0 +1,36 @@ +use crate::integration_tests::builder; +use indoc::formatdoc; +use libcnb::data::buildpack::{BuildpackVersion, SingleBuildpackDescriptor}; +use libcnb_test::{assert_contains, BuildConfig, PackResult, TestRunner}; +use std::fs; + +#[test] +#[ignore = "integration test"] +fn detect_rejects_non_python_projects() { + let buildpack_version = buildpack_version(); + + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/empty") + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stdout, + &formatdoc! {" + ===> DETECTING + ======== Output: heroku/python@{buildpack_version} ======== + No Python project files found (such as requirements.txt). + ======== Results ======== + fail: heroku/python@{buildpack_version} + ERROR: No buildpack groups passed detection. + "} + ); + }, + ); +} + +fn buildpack_version() -> BuildpackVersion { + let buildpack_toml = fs::read_to_string("buildpack.toml").unwrap(); + let buildpack_descriptor = + toml::from_str::>>(&buildpack_toml).unwrap(); + buildpack_descriptor.buildpack.version +} diff --git a/tests/integration/mod.rs b/tests/integration/mod.rs new file mode 100644 index 0000000..43afd4f --- /dev/null +++ b/tests/integration/mod.rs @@ -0,0 +1,23 @@ +//! All integration tests are skipped by default (using the `ignore` attribute), +//! since performing builds is slow. To run the tests use: `cargo test -- --ignored` + +use std::env; + +mod detect; +mod package_manager; +mod pip; +mod python_version; +mod salesforce_functions; + +const LATEST_PYTHON_3_7: &str = "3.7.16"; +const LATEST_PYTHON_3_8: &str = "3.8.16"; +const LATEST_PYTHON_3_9: &str = "3.9.16"; +const LATEST_PYTHON_3_10: &str = "3.10.10"; +const LATEST_PYTHON_3_11: &str = "3.11.2"; +const DEFAULT_PYTHON_VERSION: &str = LATEST_PYTHON_3_11; + +const DEFAULT_BUILDER: &str = "heroku/builder:22"; + +fn builder() -> String { + env::var("INTEGRATION_TEST_CNB_BUILDER").unwrap_or(DEFAULT_BUILDER.to_string()) +} diff --git a/tests/integration/package_manager.rs b/tests/integration/package_manager.rs new file mode 100644 index 0000000..8f26db7 --- /dev/null +++ b/tests/integration/package_manager.rs @@ -0,0 +1,28 @@ +use crate::integration_tests::builder; +use indoc::indoc; +use libcnb_test::{assert_contains, BuildConfig, PackResult, TestRunner}; + +#[test] +#[ignore = "integration test"] +fn no_package_manager_detected() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/pyproject_toml_only") + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {" + [Error: No Python package manager files were found] + A Pip requirements file was not found in your application's source code. + This file is required so that your application's dependencies can be installed. + + Please add a file named exactly 'requirements.txt' to the root directory of your + application, containing a list of the packages required by your application. + + For more information on what this file should contain, see: + https://pip.pypa.io/en/stable/reference/requirements-file-format/ + "} + ); + }, + ); +} diff --git a/tests/integration/pip.rs b/tests/integration/pip.rs new file mode 100644 index 0000000..f3c1723 --- /dev/null +++ b/tests/integration/pip.rs @@ -0,0 +1,229 @@ +use crate::integration_tests::{ + builder, DEFAULT_PYTHON_VERSION, LATEST_PYTHON_3_10, LATEST_PYTHON_3_11, +}; +use crate::packaging_tool_versions::PackagingToolVersions; +use indoc::formatdoc; +use libcnb_test::{assert_contains, assert_empty, BuildConfig, PackResult, TestRunner}; + +#[test] +#[ignore = "integration test"] +fn pip_editable_git_compiled() { + // This tests that: + // - Git from the stack image can be found (ie: the system PATH has been correctly propagated to pip). + // - The editable mode repository clone is saved into the dependencies layer not the app dir. + // - Compiling a source distribution package (as opposed to a pre-built wheel) works. + // - The Python headers can be found in the `include/pythonX.Y/` directory of the Python layer. + // - Headers/libraries from the stack image can be found (in this case, for libpq-dev). + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/pip_editable_git_compiled"), + |context| { + assert_contains!( + context.pack_stdout, + "Cloning https://github.com/psycopg/psycopg2 (to revision 2_9_5) to /layers/heroku_python/dependencies/src/psycopg2" + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn pip_install_error() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/pip_invalid_requirement") + .expected_pack_result(PackResult::Failure), + |context| { + // Ideally we could test a combined stdout/stderr, however libcnb-test doesn't support this: + // https://github.com/heroku/libcnb.rs/issues/536 + assert_contains!( + context.pack_stdout, + &formatdoc! {" + [Installing dependencies using Pip] + Running pip install + "} + ); + assert_contains!( + context.pack_stderr, + &formatdoc! {" + ERROR: Invalid requirement: 'an-invalid-requirement!' (from line 1 of requirements.txt) + + [Error: Unable to install dependencies using pip] + The 'pip install' command to install the application's dependencies from + 'requirements.txt' failed (exit status: 1). + + See the log output above for more information. + "} + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn cache_used_for_repeat_builds() { + let PackagingToolVersions { + pip_version, + setuptools_version, + wheel_version, + } = PackagingToolVersions::default(); + + let config = BuildConfig::new(builder(), "tests/fixtures/python_3.11"); + + TestRunner::default().build(&config, |context| { + context.rebuild(&config, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt + + [Installing Python and packaging tools] + Using cached Python {LATEST_PYTHON_3_11} + Using cached pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version} + + [Installing dependencies using Pip] + Using cached pip download/wheel cache + Running pip install + Collecting typing-extensions==4.4.0 + Using cached typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + ===> EXPORTING + "} + ); + }); + }); +} + +#[test] +#[ignore = "integration test"] +fn cache_discarded_on_python_version_change() { + let PackagingToolVersions { + pip_version, + setuptools_version, + wheel_version, + } = PackagingToolVersions::default(); + + let builder = builder(); + let config_before = BuildConfig::new(&builder, "tests/fixtures/python_3.10"); + let config_after = BuildConfig::new(&builder, "tests/fixtures/python_3.11"); + + TestRunner::default().build(config_before, |context| { + context.rebuild(config_after, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt + + [Installing Python and packaging tools] + Discarding cache since the Python version has changed from {LATEST_PYTHON_3_10} to {LATEST_PYTHON_3_11} + Installing Python {LATEST_PYTHON_3_11} + Installing pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version} + + [Installing dependencies using Pip] + Discarding cached pip download/wheel cache + Running pip install + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + ===> EXPORTING + "} + ); + }); + }); +} + +#[test] +#[ignore = "integration test"] +fn cache_discarded_on_stack_change() { + let PackagingToolVersions { + pip_version, + setuptools_version, + wheel_version, + } = PackagingToolVersions::default(); + + let fixture = "tests/fixtures/python_version_unspecified"; + let config_before = BuildConfig::new("heroku/buildpacks:20", fixture); + let config_after = BuildConfig::new("heroku/builder:22", fixture); + + TestRunner::default().build(config_before, |context| { + context.rebuild(config_after, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. + To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes + + [Installing Python and packaging tools] + Discarding cache since the stack has changed from heroku-20 to heroku-22 + Installing Python {DEFAULT_PYTHON_VERSION} + Installing pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version} + + [Installing dependencies using Pip] + Discarding cached pip download/wheel cache + Running pip install + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + ===> EXPORTING + "} + ); + }); + }); +} + +#[test] +#[ignore = "integration test"] +fn cache_discarded_on_multiple_changes() { + let PackagingToolVersions { + pip_version, + setuptools_version, + wheel_version, + } = PackagingToolVersions::default(); + + let config_before = BuildConfig::new("heroku/buildpacks:20", "tests/fixtures/python_3.10"); + let config_after = BuildConfig::new("heroku/builder:22", "tests/fixtures/python_3.11"); + + TestRunner::default().build(config_before, |context| { + context.rebuild(config_after, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + Using Python version {LATEST_PYTHON_3_11} specified in runtime.txt + + [Installing Python and packaging tools] + Discarding cache since: + - the stack has changed from heroku-20 to heroku-22 + - the Python version has changed from 3.10.10 to 3.11.2 + Installing Python {LATEST_PYTHON_3_11} + Installing pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version} + + [Installing dependencies using Pip] + Discarding cached pip download/wheel cache + Running pip install + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + ===> EXPORTING + "} + ); + }); + }); +} diff --git a/tests/integration/python_version.rs b/tests/integration/python_version.rs new file mode 100644 index 0000000..6a82e7f --- /dev/null +++ b/tests/integration/python_version.rs @@ -0,0 +1,252 @@ +// runtime.txt parsing + +use crate::integration_tests::{ + builder, DEFAULT_PYTHON_VERSION, LATEST_PYTHON_3_10, LATEST_PYTHON_3_11, LATEST_PYTHON_3_7, + LATEST_PYTHON_3_8, LATEST_PYTHON_3_9, +}; +use crate::packaging_tool_versions::PackagingToolVersions; +use indoc::{formatdoc, indoc}; +use libcnb_test::{assert_contains, assert_empty, BuildConfig, PackResult, TestRunner}; + +#[test] +#[ignore = "integration test"] +fn python_version_unspecified() { + let PackagingToolVersions { + pip_version, + setuptools_version, + wheel_version, + } = PackagingToolVersions::default(); + + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/python_version_unspecified"), + |context| { + assert_empty!(context.pack_stderr); + assert_contains!( + context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. + To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes + + [Installing Python and packaging tools] + Installing Python {DEFAULT_PYTHON_VERSION} + Installing pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version} + + [Installing dependencies using Pip] + Running pip install + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + ===> EXPORTING + "} + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn python_3_7() { + // Python 3.7 is only available on Heroku-20 and older. + let fixture = "tests/fixtures/python_3.7"; + match builder().as_str() { + "heroku/buildpacks:20" => builds_with_python_version(fixture, LATEST_PYTHON_3_7), + _ => rejects_non_existent_python_version(fixture, LATEST_PYTHON_3_7), + }; +} + +#[test] +#[ignore = "integration test"] +fn python_3_8() { + // Python 3.8 is only available on Heroku-20 and older. + let fixture = "tests/fixtures/python_3.8"; + match builder().as_str() { + "heroku/buildpacks:20" => builds_with_python_version(fixture, LATEST_PYTHON_3_8), + _ => rejects_non_existent_python_version(fixture, LATEST_PYTHON_3_8), + }; +} + +#[test] +#[ignore = "integration test"] +fn python_3_9() { + builds_with_python_version("tests/fixtures/python_3.9", LATEST_PYTHON_3_9); +} + +#[test] +#[ignore = "integration test"] +fn python_3_10() { + builds_with_python_version("tests/fixtures/python_3.10", LATEST_PYTHON_3_10); +} + +#[test] +#[ignore = "integration test"] +fn python_3_11() { + builds_with_python_version("tests/fixtures/python_3.11", LATEST_PYTHON_3_11); +} + +#[test] +#[ignore = "integration test"] +fn runtime_txt_invalid_version() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/runtime_txt_invalid_version") + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + &formatdoc! {" + [Error: Invalid Python version in runtime.txt] + The Python version specified in 'runtime.txt' is not in the correct format. + + The following file contents were found: + python-an.invalid.version + + However, the file contents must begin with a 'python-' prefix, followed by the + version specified as '..'. Comments are not supported. + + For example, to request Python {DEFAULT_PYTHON_VERSION}, the correct version format is: + python-{DEFAULT_PYTHON_VERSION} + + Please update 'runtime.txt' to use the correct version format, or else remove + the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). + + For a list of the supported Python versions, see: + https://devcenter.heroku.com/articles/python-support#supported-runtimes + "} + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn runtime_txt_non_existent_version() { + rejects_non_existent_python_version( + "tests/fixtures/runtime_txt_non_existent_version", + "999.999.999", + ); +} + +fn builds_with_python_version(fixture_path: &str, python_version: &str) { + let PackagingToolVersions { + pip_version, + setuptools_version, + wheel_version, + } = PackagingToolVersions::default(); + + let mut config = BuildConfig::new(builder(), fixture_path); + // Checks that potentially broken user-provided env vars are not being passed unfiltered to + // subprocesses we launch (such as `pip install`), thanks to `clear-env` in `buildpack.toml`. + config.env("PYTHONHOME", "/invalid-path"); + + TestRunner::default().build(config, |context| { + assert_empty!(context.pack_stderr); + assert_contains!( + context.pack_stdout, + &formatdoc! {" + ===> BUILDING + + [Determining Python version] + Using Python version {python_version} specified in runtime.txt + + [Installing Python and packaging tools] + Installing Python {python_version} + Installing pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version} + + [Installing dependencies using Pip] + Running pip install + Collecting typing-extensions==4.4.0 + Downloading typing_extensions-4.4.0-py3-none-any.whl (26 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.4.0 + ===> EXPORTING + "} + ); + // There's no sensible default process type we can set for Python apps. + assert_contains!(context.pack_stdout, "no default process type"); + + // Validate the Python/Pip install works as expected at runtime. + let command_output = context.run_shell_command( + indoc! {r#" + set -euo pipefail + + # Check that we installed the correct Python version, and that the command + # 'python' works (since it's a symlink to the actual 'python3' binary). + python --version + + # Check that the Python binary is using its own 'libpython' and not the system one: + # https://github.com/docker-library/python/issues/784 + # Note: This has to handle Python 3.9 and older not being built in shared library mode. + libpython_path=$(ldd /layers/heroku_python/python/bin/python | grep libpython || true) + if [[ -n "${libpython_path}" && "${libpython_path}" != *"=> /layers/"* ]]; then + echo "The Python binary is not using the correct libpython!" + echo "${libpython_path}" + exit 1 + fi + + # Check all required dynamically linked libraries can be found in the runtime image. + if find /layers -name '*.so' -exec ldd '{}' + | grep 'not found'; then + echo "The above dynamically linked libraries were not found!" + exit 1 + fi + + # Check that: + # - Pip is available at runtime too (and not just during the build). + # - The correct versions of pip/setuptools/wheel were installed. + # - Pip uses (via 'PYTHONUSERBASE') the user site-packages in the dependencies + # layer, and so can find the typing-extensions package installed there. + # - The "pip update available" warning is not shown (since it should be suppressed). + # - The system site-packages directory is protected against running 'pip install' + # without having passed '--user'. + pip list + pip install --dry-run typing-extensions + "#} + ); + assert_empty!(command_output.stderr); + assert_contains!( + command_output.stdout, + &formatdoc! {" + Python {python_version} + Package Version + ----------------- ------- + pip {pip_version} + setuptools {setuptools_version} + typing_extensions 4.4.0 + wheel {wheel_version} + Defaulting to user installation because normal site-packages is not writeable + Requirement already satisfied: typing-extensions in /layers/heroku_python/dependencies/lib/" + } + ); + }); +} + +fn rejects_non_existent_python_version(fixture_path: &str, python_version: &str) { + let builder = builder(); + + TestRunner::default().build( + BuildConfig::new(&builder, fixture_path).expected_pack_result(PackResult::Failure), + |context| { + let expected_stack = match builder.as_str() { + "heroku/buildpacks:20" => "heroku-20", + "heroku/builder:22" => "heroku-22", + _ => unimplemented!("Unknown builder!"), + }; + + assert_contains!( + context.pack_stderr, + &formatdoc! {" + [Error: Requested Python version is not available] + The requested Python version ({python_version}) is not available for this stack ({expected_stack}). + + Please update the version in 'runtime.txt' to a supported Python version, or else + remove the file to instead use the default version (currently Python {DEFAULT_PYTHON_VERSION}). + + For a list of the supported Python versions, see: + https://devcenter.heroku.com/articles/python-support#supported-runtimes + "} + ); + }, + ); +} diff --git a/tests/integration/salesforce_functions.rs b/tests/integration/salesforce_functions.rs new file mode 100644 index 0000000..c6d4ed8 --- /dev/null +++ b/tests/integration/salesforce_functions.rs @@ -0,0 +1,137 @@ +use crate::integration_tests::builder; +use indoc::indoc; +use libcnb_test::{ + assert_contains, assert_empty, BuildConfig, ContainerConfig, PackResult, TestRunner, +}; +use std::thread; +use std::time::Duration; + +const TEST_PORT: u16 = 12345; + +#[test] +#[ignore = "integration test"] +fn salesforce_function_template() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/salesforce_function_template"), + |context| { + assert_empty!(context.pack_stderr); + assert_contains!( + context.pack_stdout, + indoc! {" + [Validating Salesforce Function] + Function passed validation. + ===> EXPORTING + "} + ); + assert_contains!(context.pack_stdout, "Setting default process type 'web'"); + + // Test that the `sf-functions-python` web process the buildpack configures works correctly. + context.start_container( + ContainerConfig::new() + .env("PORT", TEST_PORT.to_string()) + .expose_port(TEST_PORT), + |container| { + let address_on_host = container.address_for_port(TEST_PORT).unwrap(); + let url = format!("http://{}:{}", address_on_host.ip(), address_on_host.port()); + + // Retries needed since the server takes a moment to start up. + let mut attempts_remaining = 5; + let response = loop { + let response = ureq::post(&url).set("x-health-check", "true").call(); + if response.is_ok() || attempts_remaining == 0 { + break response; + } + attempts_remaining -= 1; + thread::sleep(Duration::from_secs(1)); + }; + + let server_log_output = container.logs_now(); + assert_contains!( + server_log_output.stderr, + &format!("Uvicorn running on http://0.0.0.0:{TEST_PORT}") + ); + + let body = response.unwrap().into_string().unwrap(); + assert_eq!(body, r#""OK""#); + }, + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn salesforce_function_missing_package() { + TestRunner::default().build( + BuildConfig::new( + builder(), + "tests/fixtures/salesforce_function_missing_package", + ) + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {r#" + [Error: The Salesforce Functions package is not installed] + The 'sf-functions-python' program that is required for Python Salesforce + Functions could not be found. + + Check that the 'salesforce-functions' Python package is listed as a + dependency in 'requirements.txt'. + + If this project is not intended to be a Salesforce Function, remove the + 'type = "function"' declaration from 'project.toml' to skip this check. + "#} + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn salesforce_function_fails_self_check() { + TestRunner::default().build( + BuildConfig::new( + builder(), + "tests/fixtures/salesforce_function_fails_self_check", + ) + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {" + [Error: The Salesforce Functions self-check failed] + The 'sf-functions-python check' command failed (exit status: 1), indicating + there is a problem with the Python Salesforce Function in this project. + + Details: + Function failed validation: 'invalid' isn't a valid Salesforce REST API version." + } + ); + }, + ); +} + +#[test] +#[ignore = "integration test"] +fn project_toml_invalid() { + TestRunner::default().build( + BuildConfig::new(builder(), "tests/fixtures/project_toml_invalid") + .expected_pack_result(PackResult::Failure), + |context| { + assert_contains!( + context.pack_stderr, + indoc! {r#" + [Error: Invalid project.toml] + A parsing/validation error error occurred whilst loading the project.toml file. + + Details: TOML parse error at line 4, column 1 + | + 4 | [com.salesforce] + | ^^^^^^^^^^^^^^^^ + missing field `type` + "#} + ); + }, + ); +} From ff1c62c5acba15fc1938e00422995252b63793a6 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 28 Feb 2023 19:03:24 +0000 Subject: [PATCH 62/71] Add work item numbers to some of the TODOs --- src/errors.rs | 4 ++-- src/python_version.rs | 2 +- src/utils.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 9c4e52a..efdd9f2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -106,7 +106,7 @@ fn on_python_version_error(error: PythonVersionError) { "reading the (optional) runtime.txt file", &io_error, ), - // TODO: Write the supported Python versions inline, instead of linking out to Dev Center. + // TODO: (W-12613425) Write the supported Python versions inline, instead of linking out to Dev Center. RuntimeTxtError::Parse(ParseRuntimeTxtError { cleaned_contents }) => { let PythonVersion { major, @@ -192,7 +192,7 @@ fn on_python_layer_error(error: PythonLayerError) { &io_error, ), // This error will change once the Python version is validated against a manifest. - // TODO: Write the supported Python versions inline, instead of linking out to Dev Center. + // TODO: (W-12613425) Write the supported Python versions inline, instead of linking out to Dev Center. PythonLayerError::PythonArchiveNotFound { python_version, stack, diff --git a/src/python_version.rs b/src/python_version.rs index 813daf9..ae80443 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -60,7 +60,7 @@ pub(crate) fn determine_python_version( return Ok(runtime_txt_version); } - // TODO: Write this content inline, instead of linking out to Dev Center. + // TODO: (W-12613425) Write this content inline, instead of linking out to Dev Center. // Also adjust wording to mention pinning as a use-case, not just using a different version. log_info(formatdoc! {" No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. diff --git a/src/utils.rs b/src/utils.rs index b2b0a54..e0ca74b 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -52,8 +52,8 @@ pub(crate) fn download_and_unpack_gzipped_archive( uri: &str, destination: &Path, ) -> Result<(), DownloadUnpackArchiveError> { - // TODO: Timeouts: https://docs.rs/ureq/latest/ureq/struct.AgentBuilder.html?search=timeout - // TODO: Retries + // TODO: (W-12613141) Add a timeout: https://docs.rs/ureq/latest/ureq/struct.AgentBuilder.html?search=timeout + // TODO: (W-12613168) Add retries for certain failure modes, eg: https://github.com/algesten/ureq/blob/05b9a82a380af013338c4f42045811fc15689a6b/src/error.rs#L39-L63 let response = ureq::get(uri) .call() .map_err(DownloadUnpackArchiveError::Request)?; From f22600c0dec03850acab7b1564e49b33a694831e Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 1 Mar 2023 10:44:22 +0000 Subject: [PATCH 63/71] Misc cleanup --- src/layers/python.rs | 5 ++--- src/python_version.rs | 5 ++--- src/runtime_txt.rs | 2 +- tests/fixtures/runtime_txt_non_existent_version/runtime.txt | 2 +- tests/integration/python_version.rs | 2 +- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/layers/python.rs b/src/layers/python.rs index 3efdd29..55b8a90 100644 --- a/src/layers/python.rs +++ b/src/layers/python.rs @@ -340,7 +340,7 @@ fn generate_layer_env(layer_path: &Path, python_version: &PythonVersion) -> Laye // // One option to solve all of the above, would be to delete the `.pyc` files from the image // at the end of the buildpack's build phase, however: - // - This means they need to be regenerated at app start boot, slowing boot times. + // - This means they need to be regenerated at app boot, slowing boot times. // (For a simple Django project on a Perf-M, boot time increases from ~0.5s to ~1.5s.) // - If any other later buildpack runs any of the Python files added by this buildpack, then // the timestamp based `.pyc` files will be created again, re-introducing non-determinism. @@ -425,11 +425,10 @@ impl From for BuildpackError { #[cfg(test)] mod tests { + use super::*; use indoc::indoc; use libcnb::data::stack_id; - use super::*; - #[test] fn cache_invalidation_reason_unchanged() { let metadata = PythonLayerMetadata { diff --git a/src/python_version.rs b/src/python_version.rs index ae80443..3055353 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -77,9 +77,8 @@ pub(crate) enum PythonVersionError { #[cfg(test)] mod tests { - use libcnb::data::stack_id; - use super::*; + use libcnb::data::stack_id; #[test] fn python_version_url() { @@ -98,7 +97,7 @@ mod tests { assert_eq!( determine_python_version(Path::new("tests/fixtures/runtime_txt_non_existent_version")) .unwrap(), - PythonVersion::new(999, 999, 999) + PythonVersion::new(999, 888, 777) ); } diff --git a/src/runtime_txt.rs b/src/runtime_txt.rs index 00ce4b4..0ccd5a3 100644 --- a/src/runtime_txt.rs +++ b/src/runtime_txt.rs @@ -200,7 +200,7 @@ mod tests { ); assert_eq!( read_version(Path::new("tests/fixtures/runtime_txt_non_existent_version")).unwrap(), - Some(PythonVersion::new(999, 999, 999)) + Some(PythonVersion::new(999, 888, 777)) ); } diff --git a/tests/fixtures/runtime_txt_non_existent_version/runtime.txt b/tests/fixtures/runtime_txt_non_existent_version/runtime.txt index e67d1c2..f5bde40 100644 --- a/tests/fixtures/runtime_txt_non_existent_version/runtime.txt +++ b/tests/fixtures/runtime_txt_non_existent_version/runtime.txt @@ -1 +1 @@ -python-999.999.999 +python-999.888.777 diff --git a/tests/integration/python_version.rs b/tests/integration/python_version.rs index 6a82e7f..967edc1 100644 --- a/tests/integration/python_version.rs +++ b/tests/integration/python_version.rs @@ -125,7 +125,7 @@ fn runtime_txt_invalid_version() { fn runtime_txt_non_existent_version() { rejects_non_existent_python_version( "tests/fixtures/runtime_txt_non_existent_version", - "999.999.999", + "999.888.777", ); } From b189360053295c675cf23eb3b8476d6162d208a4 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 1 Mar 2023 10:49:15 +0000 Subject: [PATCH 64/71] Address review comments --- src/packaging_tool_versions.rs | 2 ++ src/python_version.rs | 1 + tests/integration/python_version.rs | 2 -- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/packaging_tool_versions.rs b/src/packaging_tool_versions.rs index 8ee281a..6112160 100644 --- a/src/packaging_tool_versions.rs +++ b/src/packaging_tool_versions.rs @@ -2,6 +2,8 @@ use serde::{Deserialize, Serialize}; /// The versions of various packaging tools used during the build. /// These are always installed, and are independent of the chosen package manager. +/// Strings are unused instead of a semver version, since these packages don't use +/// semver, and we never introspect the version parts anyway. #[derive(Clone, Deserialize, PartialEq, Serialize)] pub(crate) struct PackagingToolVersions { pub pip_version: String, diff --git a/src/python_version.rs b/src/python_version.rs index 3055353..456725e 100644 --- a/src/python_version.rs +++ b/src/python_version.rs @@ -30,6 +30,7 @@ impl PythonVersion { } pub fn url(&self, stack_id: &StackId) -> String { + // TODO: (W-11474658) Switch to tracking versions/URLs via a manifest file. format!( "https://heroku-buildpack-python.s3.us-east-1.amazonaws.com/{}/runtimes/python-{}.{}.{}.tar.gz", stack_id, self.major, self.minor, self.patch diff --git a/tests/integration/python_version.rs b/tests/integration/python_version.rs index 967edc1..bce028c 100644 --- a/tests/integration/python_version.rs +++ b/tests/integration/python_version.rs @@ -1,5 +1,3 @@ -// runtime.txt parsing - use crate::integration_tests::{ builder, DEFAULT_PYTHON_VERSION, LATEST_PYTHON_3_10, LATEST_PYTHON_3_11, LATEST_PYTHON_3_7, LATEST_PYTHON_3_8, LATEST_PYTHON_3_9, From e95f00e85110de2faecebaa8904a8e895b735336 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 1 Mar 2023 13:06:28 +0000 Subject: [PATCH 65/71] Add a release script --- .github/workflows/release.yml | 74 +++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..68fd596 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,74 @@ +name: Release Buildpack + +on: + workflow_dispatch: + +permissions: + contents: write + +env: + BUILDPACK_DOCKER_REPO: docker.io/heroku/buildpack-python + CARGO_TERM_COLOR: always + +jobs: + # Releases the buildpack to Docker Hub and registers it with the CNB Buildpack Registry. + # This release process intentionally does not create a .cnb file release for now, since + # there are currently no use-cases that need it for Python. + release: + name: Release heroku/python + runs-on: ubuntu-latest + steps: + # Setup + - name: Checkout + uses: actions/checkout@v3 + - name: Install musl-tools + run: sudo apt-get install musl-tools --no-install-recommends + - name: Update Rust toolchain + run: rustup update + - name: Install Rust linux-musl target + run: rustup target add x86_64-unknown-linux-musl + - name: Rust Cache + uses: Swatinem/rust-cache@v2.2.1 + - name: Install libcnb-cargo + run: cargo install libcnb-cargo + - name: Install Pack CLI + uses: buildpacks/github-actions/setup-pack@v5.0.1 + - name: Install yj and crane + uses: buildpacks/github-actions/setup-tools@v5.0.1 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: docker.io + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + # Build + - name: Compile the buildpack + run: cargo libcnb package --release + + # Publish + - name: Read buildpack metadata + run: | + echo "buildpack_id=$(yj -t < buildpack.toml | jq -r .buildpack.id)" >> $GITHUB_ENV + echo "buildpack_version=$(yj -t < buildpack.toml | jq -r .buildpack.version)" >> $GITHUB_ENV + - name: Publish the buildpack to Docker Hub + run: pack buildpack package --path target/buildpack/release/heroku_python --publish "${{ env.BUILDPACK_DOCKER_REPO }}:${{ env.buildpack_version }}" + - name: Calculate the buildpack image digest + run: echo "buildpack_digest=$(crane digest ${{ env.BUILDPACK_DOCKER_REPO }}:${{ env.buildpack_version }})" >> $GITHUB_ENV + - name: Register the new version with the CNB Buildpack Registry + uses: docker://ghcr.io/buildpacks/actions/registry/request-add-entry:5.0.1 + with: + token: ${{ secrets.CNB_REGISTRY_RELEASE_BOT_GITHUB_TOKEN }} + id: ${{ env.buildpack_id }} + version: ${{ env.buildpack_version }} + address: ${{ env.BUILDPACK_DOCKER_REPO }}@${{ env.buildpack_digest }} + - name: Create GitHub release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ env.buildpack_version }} + release_name: v${{ env.buildpack_version }} + body: | + See the [CHANGELOG](./CHANGELOG.md) for details. + draft: false From 630d74fd004fd7443f9554b9b12a6f146f29fc97 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 1 Mar 2023 13:42:05 +0000 Subject: [PATCH 66/71] Add duplicate version check to release workflow Taken from: https://github.com/heroku/buildpacks-ruby/blob/d988c4371cc5aef11e1c7375fad4f7ed87475de8/.github/workflows/release.yml#L69-L75 --- .github/workflows/release.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68fd596..65b39fc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,6 +51,12 @@ jobs: run: | echo "buildpack_id=$(yj -t < buildpack.toml | jq -r .buildpack.id)" >> $GITHUB_ENV echo "buildpack_version=$(yj -t < buildpack.toml | jq -r .buildpack.version)" >> $GITHUB_ENV + - name: Check version is unique on Docker Hub + run: | + if docker manifest inspect "${{ env.BUILDPACK_DOCKER_REPO }}:${{ env.buildpack_version }}" > /dev/null; then + echo "Duplicate version found on Docker Hub ${{ env.BUILDPACK_DOCKER_REPO }}:${{ env.buildpack_version }}" + exit 1 + fi - name: Publish the buildpack to Docker Hub run: pack buildpack package --path target/buildpack/release/heroku_python --publish "${{ env.BUILDPACK_DOCKER_REPO }}:${{ env.buildpack_version }}" - name: Calculate the buildpack image digest From 7dce0d59152e12228735f45784b2fa5c6744a99e Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 3 Mar 2023 13:51:09 +0000 Subject: [PATCH 67/71] Fix comment typo --- src/packaging_tool_versions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packaging_tool_versions.rs b/src/packaging_tool_versions.rs index 6112160..483fd86 100644 --- a/src/packaging_tool_versions.rs +++ b/src/packaging_tool_versions.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; /// The versions of various packaging tools used during the build. /// These are always installed, and are independent of the chosen package manager. -/// Strings are unused instead of a semver version, since these packages don't use +/// Strings are used instead of a semver version, since these packages don't use /// semver, and we never introspect the version parts anyway. #[derive(Clone, Deserialize, PartialEq, Serialize)] pub(crate) struct PackagingToolVersions { From 192230568b35c7ac32e20677e4d11838dc8b0659 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 6 Mar 2023 10:52:00 +0000 Subject: [PATCH 68/71] Clarify comment about env var inheritance --- src/main.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index d57d13b..6a2b637 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,9 +65,11 @@ impl Buildpack for PythonBuildpack { .map_err(BuildpackError::PythonVersion)?; let packaging_tool_versions = PackagingToolVersions::default(); - // We inherit the current process's env vars, since we want `PATH` and `HOME` to be set - // so that later commands can find tools like Git in the stack image. Any user-provided - // env vars will still be excluded, due to the use of `clear-env` in `buildpack.toml`. + // We inherit the current process's env vars, since we want `PATH` and `HOME` from the OS + // to be set, so that later commands can find tools like Git in the stack image. We exclude + // user-provided env vars (by setting `clear-env` to true in `buildpack.toml`) to prevent an + // app's env vars from breaking internal buildpack commands. Any buildpack steps that need + // user-provided env vars must explicitly retrieve them via `context.platform.env`. let mut command_env = Env::from_current(); // Create the layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`. From 55349e40c0ffff73d681d70ee575bc6b62f1798f Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 6 Mar 2023 10:59:59 +0000 Subject: [PATCH 69/71] Fix comment typo in package_manager.rs Co-authored-by: Josh W Lewis --- src/package_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/package_manager.rs b/src/package_manager.rs index 357a186..0246cd4 100644 --- a/src/package_manager.rs +++ b/src/package_manager.rs @@ -1,7 +1,7 @@ use std::io; use std::path::Path; -/// A ordered mapping of project filenames to their associated package manager. +/// An ordered mapping of project filenames to their associated package manager. /// Earlier entries will take precedence if a project matches multiple package managers. pub(crate) const PACKAGE_MANAGER_FILE_MAPPING: [(&str, PackageManager); 1] = [("requirements.txt", PackageManager::Pip)]; From b2b796ff0aed6e6d5ad0a11f515ac77967435310 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 6 Mar 2023 11:21:57 +0000 Subject: [PATCH 70/71] Update to setuptools 67.5.0 --- src/packaging_tool_versions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packaging_tool_versions.rs b/src/packaging_tool_versions.rs index 483fd86..73822e1 100644 --- a/src/packaging_tool_versions.rs +++ b/src/packaging_tool_versions.rs @@ -15,7 +15,7 @@ impl Default for PackagingToolVersions { fn default() -> Self { Self { pip_version: "23.0.1".to_string(), - setuptools_version: "67.4.0".to_string(), + setuptools_version: "67.5.0".to_string(), wheel_version: "0.38.4".to_string(), } } From dd390e949a06987e476061d8bd1393029340627a Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 6 Mar 2023 11:22:06 +0000 Subject: [PATCH 71/71] Refresh Cargo.lock --- Cargo.lock | 74 +++++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4953594..5a5f12f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,9 +191,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" +checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72" dependencies = [ "cc", "cxxbridge-flags", @@ -203,9 +203,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" +checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613" dependencies = [ "cc", "codespan-reporting", @@ -218,15 +218,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" +checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97" [[package]] name = "cxxbridge-macro" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56" dependencies = [ "proc-macro2", "quote", @@ -377,9 +377,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -533,9 +533,9 @@ dependencies = [ [[package]] name = "indoc" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe2b9d82064e8a0226fddb3547f37f28eaa46d0fc210e275d835f08cf3b76a7" +checksum = "9f2cb48b81b1dc9f39676bf99f5499babfec7cd8fe14307f7b3d747208fb5690" [[package]] name = "instant" @@ -558,9 +558,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" @@ -869,9 +869,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.8" +version = "0.36.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" dependencies = [ "bitflags", "errno", @@ -895,15 +895,15 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "sct" @@ -946,9 +946,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ "itoa", "ryu", @@ -957,9 +957,9 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a5ec9fa74a20ebbe5d9ac23dac1fc96ba0ecfe9f50f2843b52e537b10fbcb4e" +checksum = "395627de918015623b32e7669714206363a7fc00382bf477e72c1f7533e8eafc" dependencies = [ "proc-macro2", "quote", @@ -1013,9 +1013,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -1073,18 +1073,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" dependencies = [ "proc-macro2", "quote", @@ -1135,9 +1135,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -1147,7 +1147,7 @@ dependencies = [ "num_cpus", "pin-project-lite", "socket2", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -1249,9 +1249,9 @@ checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -1530,9 +1530,9 @@ checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winnow" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658" +checksum = "c95fb4ff192527911dd18eb138ac30908e7165b8944e528b6af93aa4c842d345" dependencies = [ "memchr", ]