diff --git a/CHANGELOG.md b/CHANGELOG.md index dc1e65506..f0e0c71e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,12 @@ ## [Unreleased] -- Buildpack error messages are now more consistently formatted and use colour. ([#1639](https://github.com/heroku/heroku-buildpack-python/pull/1639)) +- Moved the SQLite3 install step prior to installing dependencies when using Pipenv. This now matches the behaviour when using pip and allows dependencies to actually use the headers. ([#1640](https://github.com/heroku/heroku-buildpack-python/pull/1640)) +- Stopped exposing the `SKIP_PIP_INSTALL` env var to `bin/post_compile` and other subprocesses when using Pipenv. ([#1640](https://github.com/heroku/heroku-buildpack-python/pull/1640)) +- Stopped creating `.heroku/python/requirements-{declared,installed}.txt` files when using pip. ([#1640](https://github.com/heroku/heroku-buildpack-python/pull/1640)) +- Stopped creating a placeholder `requirements.txt` file when an app only has a `setup.py` file and no other package manager files. Instead pip is now invoked directly using `--editable .`. ([#1640](https://github.com/heroku/heroku-buildpack-python/pull/1640)) +- Improved buildpack metrics for package manager detection and duration of install steps. ([#1640](https://github.com/heroku/heroku-buildpack-python/pull/1640)) +- Updated buildpack-generated error messages to use colour and be more consistently formatted. ([#1639](https://github.com/heroku/heroku-buildpack-python/pull/1639)) ## [v256] - 2024-09-07 diff --git a/bin/compile b/bin/compile index 9bd6482b7..d598039e9 100755 --- a/bin/compile +++ b/bin/compile @@ -16,6 +16,10 @@ BUILDPACK_DIR=$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd) source "${BUILDPACK_DIR}/bin/utils" source "${BUILDPACK_DIR}/lib/metadata.sh" source "${BUILDPACK_DIR}/lib/output.sh" +source "${BUILDPACK_DIR}/lib/package_manager.sh" +source "${BUILDPACK_DIR}/lib/pip.sh" +source "${BUILDPACK_DIR}/lib/pipenv.sh" +source "${BUILDPACK_DIR}/lib/utils.sh" compile_start_time=$(nowms) @@ -97,12 +101,10 @@ cd "$BUILD_DIR" # - The build is executed, modifying `~/.heroku/{known-paths}`. # - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache. -# Create the cache directory, if it doesn't exist. mkdir -p "$CACHE_DIR/.heroku" # Restore old artifacts from the cache. mkdir -p .heroku - # The Python installation. cp -R "$CACHE_DIR/.heroku/python" .heroku/ &>/dev/null || true # A plain text file which contains the current stack being used (used for cache busting). @@ -116,10 +118,7 @@ if [[ -d "$CACHE_DIR/.heroku/src" ]]; then cp -R "$CACHE_DIR/.heroku/src" .heroku/ &>/dev/null || true fi -# The pre_compile hook. Customers rely on this. Don't remove it. -# This part of the code is used to allow users to customize their build experience -# without forking the buildpack by providing a `bin/pre_compile` script, which gets -# run inline with the buildpack automatically. +# Runs a `bin/pre_compile` script if found in the app source, allowing build customisation. source "${BUILDPACK_DIR}/bin/steps/hooks/pre_compile" # Sticky runtimes. If there was a previous build, and it used a given version of Python, @@ -128,43 +127,15 @@ if [[ -f "$CACHE_DIR/.heroku/python-version" ]]; then CACHED_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version") fi -# We didn't always record the stack version. This code is in place because of that. +# We didn't always record the stack version. if [[ -f "$CACHE_DIR/.heroku/python-stack" ]]; then CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack") else CACHED_PYTHON_STACK=$STACK fi -# TODO: Move this into a new package manager handling implementation when adding Poetry support. -# We intentionally don't mention `setup.py` here since it's being removed soon. -if [[ ! -f requirements.txt && ! -f Pipfile && ! -f setup.py ]]; then - display_error <<-EOF - Error: Couldn't find any supported Python package manager files. - - A Python app on Heroku must have either a 'requirements.txt' or - 'Pipfile' package manager file in the root directory of its - source code. - - Currently the root directory of your app contains: - - $(ls -1 --indicator-style=slash "${BUILD_DIR}") - - If your app already has a package manager file, check that it: - - 1. Is in the top level directory (not a subdirectory). - 2. Has the correct spelling (the filenames are case-sensitive). - 3. Isn't listed in '.gitignore' or '.slugignore'. - - Otherwise, add a package manager file to your app. If your app has - no dependencies, then create an empty 'requirements.txt' file. - - For help with using Python on Heroku, see: - https://devcenter.heroku.com/articles/getting-started-with-python - https://devcenter.heroku.com/articles/python-support - EOF - meta_set "failure_reason" "package-manager-not-found" - exit 1 -fi +PACKAGE_MANAGER=$(package_manager::determine_package_manager "${BUILD_DIR}") +meta_set "package_manager" "${PACKAGE_MANAGER}" # Pipenv Python version support. # Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version). @@ -189,9 +160,9 @@ else echo "${DEFAULT_PYTHON_VERSION}" >runtime.txt fi -# Create the directory for .profile.d, if it doesn't exist. +# The directory for the .profile.d scripts. mkdir -p "$(dirname "$PROFILE_PATH")" -# Create the directory for editable source code installation, if it doesn't exist. +# The directory for editable VCS dependencies. mkdir -p /app/.heroku/src # On Heroku CI, builds happen in `/app`. Otherwise, on the Heroku platform, @@ -207,46 +178,58 @@ if [[ "$(realpath "${BUILD_DIR}")" != "$(realpath /app)" ]]; then # Note: .heroku/src is copied in later. fi -# Download / Install Python, from pre-build binaries available on Amazon S3. -# This step also bootstraps pip / setuptools. +# Download and install Python using pre-built binaries from S3. install_python_start_time=$(nowms) source "${BUILDPACK_DIR}/bin/steps/python" meta_time "python_install_duration" "${install_python_start_time}" -# Install Pipenv dependencies, if a Pipfile was provided. -source "${BUILDPACK_DIR}/bin/steps/pipenv" - -# If no requirements.txt file given, assume `setup.py develop` is intended. -# This allows for people to ship a setup.py application to Heroku - -if [[ ! -f requirements.txt ]] && [[ ! -f Pipfile ]]; then - meta_set "setup_py_only" "true" - echo "-e ." >requirements.txt -else - meta_set "setup_py_only" "false" -fi +# Install the package manager and related tools. +package_manager_install_start_time=$(nowms) +bundled_pip_module_path="$(utils::bundled_pip_module_path "${BUILD_DIR}")" +case "${PACKAGE_MANAGER}" in + pip) + pip::install_pip_setuptools_wheel "${bundled_pip_module_path}" + ;; + pipenv) + # TODO: Stop installing pip when using Pipenv. + pip::install_pip_setuptools_wheel "${bundled_pip_module_path}" + pipenv::install_pipenv + ;; + *) + abort_internal_error "Unhandled package manager" + ;; +esac +meta_time "package_manager_install_duration" "${package_manager_install_start_time}" # SQLite3 support. -# This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the -# libsqlite3-0 library since that exists on the stack image. +# Installs the sqlite3 dev headers and sqlite3 binary but not the +# libsqlite3-0 library since that exists in the base image. install_sqlite_start_time=$(nowms) source "${BUILDPACK_DIR}/bin/steps/sqlite3" buildpack_sqlite3_install meta_time "sqlite_install_duration" "${install_sqlite_start_time}" -# pip install -# ----------- - -# Install dependencies with pip (where the magic happens). -source "${BUILDPACK_DIR}/bin/steps/pip-install" +# Install app dependencies. +dependencies_install_start_time=$(nowms) +case "${PACKAGE_MANAGER}" in + pip) + pip::install_dependencies + ;; + pipenv) + pipenv::install_dependencies + ;; + *) + abort_internal_error "Unhandled package manager" + ;; +esac +meta_time "dependencies_install_duration" "${dependencies_install_start_time}" # Support for NLTK corpora. nltk_downloader_start_time=$(nowms) sub_env "${BUILDPACK_DIR}/bin/steps/nltk" meta_time "nltk_downloader_duration" "${nltk_downloader_start_time}" -# Support for editable installations. Here, we are copying pip–created src directory, -# and copying it into the proper place (the logical place to do this was early, but it must be done here). +# Support for editable installations. # In CI, $BUILD_DIR is /app. # Realpath is used to support use-cases where one of the locations is a symlink to the other. if [[ "$(realpath "${BUILD_DIR}")" != "$(realpath /app)" ]]; then @@ -256,9 +239,6 @@ fi # Django collectstatic support. # The buildpack automatically runs collectstatic for Django applications. -# This is the cause for the majority of build failures on the Python platform. -# These failures are intentional — if collectstatic (which can be tricky, at times) fails, -# your build fails. collectstatic_start_time=$(nowms) sub_env "${BUILDPACK_DIR}/bin/steps/collectstatic" meta_time "django_collectstatic_duration" "${collectstatic_start_time}" @@ -304,7 +284,7 @@ fi cp "${BUILDPACK_DIR}/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH" cp "${BUILDPACK_DIR}/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH" -# Experimental post_compile hook. Don't remove this. +# Runs a `bin/post_compile` script if found in the app source, allowing build customisation. source "${BUILDPACK_DIR}/bin/steps/hooks/post_compile" # Store new artifacts in the cache. diff --git a/bin/report b/bin/report index 9210d7371..5697a7f57 100755 --- a/bin/report +++ b/bin/report @@ -67,6 +67,7 @@ STRING_FIELDS=( failure_reason nltk_downloader package_manager + package_manager_multiple_found pip_version pipenv_version python_version_major @@ -81,6 +82,7 @@ ALL_OTHER_FIELDS=( dependencies_install_duration django_collectstatic_duration nltk_downloader_duration + package_manager_install_duration pipenv_has_lockfile post_compile_hook post_compile_hook_duration diff --git a/bin/steps/pip-install b/bin/steps/pip-install deleted file mode 100755 index 86ecfd0d3..000000000 --- a/bin/steps/pip-install +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -if [[ -z "$SKIP_PIP_INSTALL" ]]; then - pip_install_start_time=$(nowms) - meta_set "package_manager" "pip" - - puts-step "Installing requirements with pip" - - # Set Pip env vars - # This reads certain environment variables set on the Heroku app config - # and makes them accessible to the pip install process. - # - # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used. - if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then - PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")" - export PIP_EXTRA_INDEX_URL - fi - - set +e - - /app/.heroku/python/bin/pip install -r requirements.txt --exists-action=w --src='/app/.heroku/src' --disable-pip-version-check --no-cache-dir --progress-bar off 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent - PIP_STATUS="${PIPESTATUS[0]}" - set -e - - show-warnings - - if [[ ! $PIP_STATUS -eq 0 ]]; then - meta_set "failure_reason" "pip-install" - exit 1 - fi - - cp requirements.txt .heroku/python/requirements-declared.txt - /app/.heroku/python/bin/pip freeze --disable-pip-version-check >.heroku/python/requirements-installed.txt - - # Install test dependencies, for CI. - if [[ -n "$INSTALL_TEST" ]]; then - if [[ -f requirements-test.txt ]]; then - puts-step "Installing test dependencies..." - /app/.heroku/python/bin/pip install -r requirements-test.txt --exists-action=w --src='/app/.heroku/src' --disable-pip-version-check --no-cache-dir 2>&1 | cleanup | indent - fi - fi - - meta_time "dependencies_install_duration" "${pip_install_start_time}" -fi diff --git a/bin/steps/pipenv-python-version b/bin/steps/pipenv-python-version index b6614cf99..6b4bd46e5 100755 --- a/bin/steps/pipenv-python-version +++ b/bin/steps/pipenv-python-version @@ -1,8 +1,10 @@ #!/usr/bin/env bash +# TODO: Move this to lib/ as part of the refactoring for .python-version support. + # Detect Python-version with Pipenv. -if [[ -f $BUILD_DIR/Pipfile ]]; then +if [[ "${PACKAGE_MANAGER}" == "pipenv" ]]; then if [[ ! -f $BUILD_DIR/runtime.txt ]]; then if [[ ! -f $BUILD_DIR/Pipfile.lock ]]; then diff --git a/bin/steps/python b/bin/steps/python index 3c1e36e57..56af2778e 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -166,34 +166,3 @@ else hash -r fi - -PIP_VERSION=$(get_requirement_version 'pip') -SETUPTOOLS_VERSION=$(get_requirement_version 'setuptools') -WHEEL_VERSION=$(get_requirement_version 'wheel') - -puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}" - -meta_set "pip_version" "${PIP_VERSION}" -meta_set "setuptools_version" "${SETUPTOOLS_VERSION}" -meta_set "wheel_version" "${WHEEL_VERSION}" - -# Python bundles Pip within its standard library, which we can use to install our chosen -# pip version from PyPI, saving us from having to download the usual pip bootstrap script. -# We have to use a glob since the bundled wheel filename contains the pip version, which -# differs between Python versions. We also have to handle the case where there are multiple -# matching pip wheels, since in some versions of Python (eg 3.9.0) multiple versions of pip -# were accidentally bundled upstream. Note: This implementation relies upon `nullglob` being -# set, which is the case thanks to the `bin/utils` that was run earlier. -BUNDLED_PIP_WHEEL_LIST=(.heroku/python/lib/python*/ensurepip/_bundled/pip-*.whl) -BUNDLED_PIP_WHEEL="${BUNDLED_PIP_WHEEL_LIST[0]}" - -if [[ -z "${BUNDLED_PIP_WHEEL}" ]]; then - display_error "Error: Failed to locate the bundled pip wheel." - meta_set "failure_reason" "bundled-pip-not-found" - exit 1 -fi - -/app/.heroku/python/bin/python "${BUNDLED_PIP_WHEEL}/pip" install --quiet --disable-pip-version-check --no-cache-dir \ - "pip==${PIP_VERSION}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}" - -hash -r diff --git a/buildpack.toml b/buildpack.toml index 07f83d4cf..4ccd6f242 100644 --- a/buildpack.toml +++ b/buildpack.toml @@ -7,6 +7,7 @@ files = [ "builds/", "etc/publish.sh", "spec/", + ".editorconfig", ".gitignore", ".rubocop.yml", ".shellcheckrc", diff --git a/lib/kvstore.sh b/lib/kvstore.sh index 17e2dd0d0..4bfdc85ac 100644 --- a/lib/kvstore.sh +++ b/lib/kvstore.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# TODO: Switch this file to using namespaced functions like `kvstore::`. + # Taken from: https://github.com/heroku/heroku-buildpack-nodejs/blob/main/lib/kvstore.sh kv_create() { diff --git a/lib/metadata.sh b/lib/metadata.sh index 7d2a39b7a..3def5ded3 100644 --- a/lib/metadata.sh +++ b/lib/metadata.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# TODO: Switch this file to using namespaced functions like `metadata::`. + # Based on: https://github.com/heroku/heroku-buildpack-nodejs/blob/main/lib/metadata.sh source "${BUILDPACK_DIR}/lib/kvstore.sh" diff --git a/lib/output.sh b/lib/output.sh index e063ddd88..246e689bf 100644 --- a/lib/output.sh +++ b/lib/output.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# TODO: Switch this file to using namespaced functions like `output::`. + ANSI_RED='\033[1;31m' ANSI_RESET='\033[0m' diff --git a/lib/package_manager.sh b/lib/package_manager.sh new file mode 100644 index 000000000..d776c08a5 --- /dev/null +++ b/lib/package_manager.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +function package_manager::determine_package_manager() { + local build_dir="${1}" + local package_managers_found=() + + if [[ -f "${build_dir}/Pipfile.lock" ]]; then + package_managers_found+=(pipenv) + meta_set "pipenv_has_lockfile" "true" + elif [[ -f "${build_dir}/Pipfile" ]]; then + # TODO: Start requiring a Pipfile.lock and make this branch a "missing lockfile" error instead. + package_managers_found+=(pipenv) + meta_set "pipenv_has_lockfile" "false" + fi + + if [[ -f "${build_dir}/requirements.txt" ]]; then + package_managers_found+=(pip) + fi + + # TODO: Deprecate/sunset this fallback, since using setup.py declared dependencies is + # not a best practice, and we can only guess as to which package manager to use. + if ((${#package_managers_found[@]} == 0)) && [[ -f "${build_dir}/setup.py" ]]; then + package_managers_found+=(pip) + meta_set "setup_py_only" "true" + else + meta_set "setup_py_only" "false" + fi + + case "${#package_managers_found[@]}" in + 1) + echo "${package_managers_found[0]}" + return 0 + ;; + 0) + display_error <<-EOF + Error: Couldn't find any supported Python package manager files. + + A Python app on Heroku must have either a 'requirements.txt' or + 'Pipfile' package manager file in the root directory of its + source code. + + Currently the root directory of your app contains: + + $(ls -1 --indicator-style=slash "${build_dir}" || true) + + If your app already has a package manager file, check that it: + + 1. Is in the top level directory (not a subdirectory). + 2. Has the correct spelling (the filenames are case-sensitive). + 3. Isn't listed in '.gitignore' or '.slugignore'. + + Otherwise, add a package manager file to your app. If your app has + no dependencies, then create an empty 'requirements.txt' file. + + For help with using Python on Heroku, see: + https://devcenter.heroku.com/articles/getting-started-with-python + https://devcenter.heroku.com/articles/python-support + EOF + meta_set "failure_reason" "package-manager-not-found" + return 1 + ;; + *) + # If multiple package managers are found, use the first found above. + # TODO: Turn this case into an error since it results in support tickets from users + # who don't realise they have multiple package manager files and think their changes + # aren't taking effect. (We'll need to wait until after Poetry support has landed, + # and people have had a chance to migrate from the third-party Poetry buildpack, + # since using it results in both a requirements.txt and a poetry.lock.) + echo "${package_managers_found[0]}" + meta_set "package_manager_multiple_found" "$( + IFS=, + echo "${package_managers_found[*]}" + )" + return 0 + ;; + esac +} diff --git a/lib/pip.sh b/lib/pip.sh new file mode 100644 index 000000000..4ba2a6812 --- /dev/null +++ b/lib/pip.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +function pip::install_pip_setuptools_wheel() { + # We use the pip wheel bundled within Python's standard library to install our chosen + # pip version, since it's faster than `ensurepip` followed by an upgrade in place. + local bundled_pip_module_path="${1}" + + # TODO: Either make these `local` or move elsewhere as part of the cache invalidation refactoring. + PIP_VERSION=$(get_requirement_version 'pip') + SETUPTOOLS_VERSION=$(get_requirement_version 'setuptools') + WHEEL_VERSION=$(get_requirement_version 'wheel') + meta_set "pip_version" "${PIP_VERSION}" + meta_set "setuptools_version" "${SETUPTOOLS_VERSION}" + meta_set "wheel_version" "${WHEEL_VERSION}" + + puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}" + + /app/.heroku/python/bin/python "${bundled_pip_module_path}" install --quiet --disable-pip-version-check --no-cache-dir \ + "pip==${PIP_VERSION}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}" +} + +function pip::install_dependencies() { + puts-step "Installing requirements with pip" + + # Make select pip config vars set on the Heroku app available to pip. + # TODO: Expose all config vars (after suitable checks are added for unsafe env vars) + # to allow for the env var interpolation feature of requirements files to work. + # + # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used. + if [[ -r "${ENV_DIR}/PIP_EXTRA_INDEX_URL" ]]; then + PIP_EXTRA_INDEX_URL="$(cat "${ENV_DIR}/PIP_EXTRA_INDEX_URL")" + export PIP_EXTRA_INDEX_URL + fi + + # TODO: Deprecate/sunset this missing requirements file fallback. + if [[ -f setup.py && ! -f requirements.txt ]]; then + args=(--editable .) + else + args=(-r requirements.txt) + fi + + set +e + /app/.heroku/python/bin/pip install "${args[@]}" --exists-action=w --src='/app/.heroku/src' --disable-pip-version-check --no-cache-dir --progress-bar off 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent + local PIP_STATUS="${PIPESTATUS[0]}" + set -e + + show-warnings + + if [[ ! ${PIP_STATUS} -eq 0 ]]; then + meta_set "failure_reason" "pip-install" + return 1 + fi + + # Install test dependencies, for Heroku CI. + if [[ -n "${INSTALL_TEST}" ]]; then + if [[ -f requirements-test.txt ]]; then + puts-step "Installing test dependencies..." + /app/.heroku/python/bin/pip install -r requirements-test.txt --exists-action=w --src='/app/.heroku/src' --disable-pip-version-check --no-cache-dir 2>&1 | cleanup | indent + fi + fi +} diff --git a/bin/steps/pipenv b/lib/pipenv.sh old mode 100755 new mode 100644 similarity index 60% rename from bin/steps/pipenv rename to lib/pipenv.sh index 6974ae2d5..56ee6b5c2 --- a/bin/steps/pipenv +++ b/lib/pipenv.sh @@ -3,54 +3,46 @@ # export CLINT_FORCE_COLOR=1 # export PIPENV_FORCE_COLOR=1 +function pipenv::install_pipenv() { + # TODO: Either make this `local` or move elsewhere as part of the cache invalidation refactoring. + PIPENV_VERSION=$(get_requirement_version 'pipenv') + meta_set "pipenv_version" "${PIPENV_VERSION}" + + puts-step "Installing Pipenv ${PIPENV_VERSION}" + + # TODO: Install Pipenv into a venv so it isn't leaked into the app environment. + # TODO: Explore viability of making Pipenv only be available during the build, to reduce slug size. + /app/.heroku/python/bin/pip install --quiet --disable-pip-version-check --no-cache-dir "pipenv==${PIPENV_VERSION}" +} + # Previous versions of the buildpack used to cache the checksum of the lockfile to allow # for skipping pipenv install if the lockfile was unchanged. However, this is not always safe # to do (the lockfile can refer to dependencies that can change independently of the lockfile, # for example, when using a local non-editable file dependency), so we no longer ever skip # install, and instead defer to pipenv to determine whether install is actually a no-op. -rm -f .heroku/python/Pipfile.lock.sha256 - -if [[ -f Pipfile ]]; then - pipenv_install_start_time=$(nowms) - meta_set "package_manager" "pipenv" - - # Skip installing dependencies using pip later. - # TODO: Stop leaking this env var into subshells such as post_compile hooks. - export SKIP_PIP_INSTALL=1 - - # Set Pip env vars - # This reads certain environment variables set on the Heroku app config - # and makes them accessible to the pip install process. +function pipenv::install_dependencies() { + # Make select pip config vars set on the Heroku app available to the pip used by Pipenv. + # TODO: Expose all config vars (after suitable checks are added for unsafe env vars). # # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used. - if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then - PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")" + if [[ -r "${ENV_DIR}/PIP_EXTRA_INDEX_URL" ]]; then + PIP_EXTRA_INDEX_URL="$(cat "${ENV_DIR}/PIP_EXTRA_INDEX_URL")" export PIP_EXTRA_INDEX_URL fi - PIPENV_VERSION=$(get_requirement_version 'pipenv') - meta_set "pipenv_version" "${PIPENV_VERSION}" - - /app/.heroku/python/bin/pip install --quiet --disable-pip-version-check --no-cache-dir "pipenv==${PIPENV_VERSION}" - # Install the test dependencies, for CI. # TODO: This is currently inconsistent with the non-test path, since it assumes (but doesn't check for) a lockfile. - if [[ -n "$INSTALL_TEST" ]]; then - meta_set "pipenv_has_lockfile" "true" - puts-step "Installing test dependencies" + if [[ -n "${INSTALL_TEST}" ]]; then + puts-step "Installing test dependencies with Pipenv" /app/.heroku/python/bin/pipenv install --dev --system --deploy --extra-pip-args='--src=/app/.heroku/src' 2>&1 | cleanup | indent # Install the dependencies. elif [[ ! -f Pipfile.lock ]]; then - meta_set "pipenv_has_lockfile" "false" - puts-step "Installing dependencies with Pipenv ${PIPENV_VERSION}" + puts-step "Installing dependencies with Pipenv" /app/.heroku/python/bin/pipenv install --system --skip-lock --extra-pip-args='--src=/app/.heroku/src' 2>&1 | indent else - meta_set "pipenv_has_lockfile" "true" - puts-step "Installing dependencies with Pipenv ${PIPENV_VERSION}" + puts-step "Installing dependencies with Pipenv" /app/.heroku/python/bin/pipenv install --system --deploy --extra-pip-args='--src=/app/.heroku/src' 2>&1 | indent fi - - meta_time "dependencies_install_duration" "${pipenv_install_start_time}" -fi +} diff --git a/lib/utils.sh b/lib/utils.sh new file mode 100644 index 000000000..fb7da23a6 --- /dev/null +++ b/lib/utils.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Python bundles Pip within its standard library, which we can use to install our chosen +# pip version from PyPI, saving us from having to download the usual pip bootstrap script. +function utils::bundled_pip_module_path() { + local build_dir="${1}" + + # We have to use a glob since the bundled wheel filename contains the pip version, which + # differs between Python versions. We also have to handle the case where there are multiple + # matching pip wheels, since in some versions of Python (eg 3.9.0) multiple versions of pip + # were accidentally bundled upstream. Note: This implementation relies upon `nullglob` being + # set, which is the case thanks to the `bin/utils` that was run earlier. + local bundled_pip_wheel_list=("${build_dir}"/.heroku/python/lib/python*/ensurepip/_bundled/pip-*.whl) + local bundled_pip_wheel="${bundled_pip_wheel_list[0]}" + + if [[ -z "${bundled_pip_wheel}" ]]; then + display_error "Error: Failed to locate the bundled pip wheel." + meta_set "failure_reason" "bundled-pip-not-found" + return 1 + fi + + echo "${bundled_pip_wheel}/pip" +} + +function utils::abort_internal_error() { + local message="${1}" + display_error "Internal error: ${message} (line $(caller || true))." + exit 1 +} diff --git a/spec/hatchet/ci_spec.rb b/spec/hatchet/ci_spec.rb index ad8053abf..f6403c1ac 100644 --- a/spec/hatchet/ci_spec.rb +++ b/spec/hatchet/ci_spec.rb @@ -96,10 +96,11 @@ To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes -----> Installing python-#{DEFAULT_PYTHON_VERSION} -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - -----> Installing test dependencies + -----> Installing Pipenv #{PIPENV_VERSION} + -----> Installing SQLite3 + -----> Installing test dependencies with Pipenv Installing dependencies from Pipfile.lock \\(.+\\)... Installing dependencies from Pipfile.lock \\(.+\\)... - -----> Installing SQLite3 -----> Skipping Django collectstatic since the env var DISABLE_COLLECTSTATIC is set. -----> Running post-compile hook CI=true @@ -115,7 +116,6 @@ PIP_NO_PYTHON_VERSION_WARNING=1 PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config: PYTHONUNBUFFERED=1 - SKIP_PIP_INSTALL=1 -----> Inline app detected LANG=en_US.UTF-8 LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib: @@ -152,10 +152,11 @@ To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes -----> Using cached install of python-#{DEFAULT_PYTHON_VERSION} -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - -----> Installing test dependencies + -----> Installing Pipenv #{PIPENV_VERSION} + -----> Installing SQLite3 + -----> Installing test dependencies with Pipenv Installing dependencies from Pipfile.lock \\(.+\\)... Installing dependencies from Pipfile.lock \\(.+\\)... - -----> Installing SQLite3 -----> Skipping Django collectstatic since the env var DISABLE_COLLECTSTATIC is set. -----> Running post-compile hook REGEX diff --git a/spec/hatchet/pip_spec.rb b/spec/hatchet/pip_spec.rb index 5351975e0..b57910e60 100644 --- a/spec/hatchet/pip_spec.rb +++ b/spec/hatchet/pip_spec.rb @@ -220,7 +220,7 @@ remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing SQLite3 remote: -----> Installing requirements with pip - remote: Obtaining file:///tmp/build_.* \\(from -r requirements.txt \\(line 1\\)\\) + remote: Obtaining file:///tmp/build_.* remote: Preparing metadata \\(setup.py\\): started remote: Preparing metadata \\(setup.py\\): finished with status 'done' remote: .+ diff --git a/spec/hatchet/pipenv_spec.rb b/spec/hatchet/pipenv_spec.rb index ccd5dee93..f8b810b3f 100644 --- a/spec/hatchet/pipenv_spec.rb +++ b/spec/hatchet/pipenv_spec.rb @@ -10,9 +10,10 @@ remote: -----> Using Python version specified in Pipfile.lock remote: -----> Installing python-#{python_version} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} - remote: Installing dependencies from Pipfile.lock \\(.+\\)... + remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... REGEX end end @@ -49,7 +50,9 @@ remote: To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes remote: -----> Installing python-#{DEFAULT_PYTHON_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} + remote: -----> Installing Pipenv #{PIPENV_VERSION} + remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv remote: The flag --skip-lock has been reintroduced \\(but is not recommended\\). Without remote: the lock resolver it is difficult to manage multiple package indexes, and hash remote: checking is not provided. However it can help manage installs with current @@ -57,7 +60,6 @@ remote: Pipfile.lock not found, creating... .+ remote: Installing dependencies from Pipfile... - remote: -----> Installing SQLite3 REGEX end end @@ -78,9 +80,10 @@ remote: To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes remote: -----> Installing python-#{DEFAULT_PYTHON_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} - remote: Installing dependencies from Pipfile.lock \\(.+\\)... + remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... remote: -----> Inline app detected remote: LANG=en_US.UTF-8 remote: LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib: @@ -195,9 +198,10 @@ remote: ! remote: -----> Installing python-#{LATEST_PYTHON_3_8} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} - remote: Installing dependencies from Pipfile.lock \\(.+\\)... + remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... REGEX end end @@ -260,9 +264,10 @@ remote: ! remote: -----> Installing python-3.12.2 remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} - remote: Installing dependencies from Pipfile.lock \\(.+\\)... + remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... REGEX end end @@ -319,9 +324,10 @@ remote: -----> Using Python version specified in runtime.txt remote: -----> Installing python-#{LATEST_PYTHON_3_12} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} - remote: Installing dependencies from Pipfile.lock \\(.+\\)... + remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... REGEX end end @@ -340,9 +346,10 @@ remote: To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes remote: -----> Using cached install of python-#{DEFAULT_PYTHON_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} - remote: Installing dependencies from Pipfile.lock \\(.+\\)... + remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... remote: -----> Discovering process types REGEX end @@ -366,9 +373,10 @@ remote: To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes remote: -----> Using cached install of python-#{DEFAULT_PYTHON_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} - remote: Installing dependencies from Pipfile.lock \\(.+\\)... + remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... remote: -----> Discovering process types REGEX end @@ -385,9 +393,10 @@ remote: -----> Using Python version specified in Pipfile.lock remote: -----> Installing python-#{LATEST_PYTHON_3_12} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} - remote: Installing dependencies from Pipfile.lock \\(.+\\)... + remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... REGEX end end @@ -404,7 +413,9 @@ remote: To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes remote: -----> Installing python-#{DEFAULT_PYTHON_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing dependencies with Pipenv #{PIPENV_VERSION} + remote: -----> Installing Pipenv #{PIPENV_VERSION} + remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv remote: Your Pipfile.lock \\(.+\\) is out of date. Expected: \\(.+\\). remote: .+ remote: ERROR:: Aborting deploy