Add support for Heroku-20 (heroku#968)

This adds support for the upcoming Heroku-20 stack.

The Heroku-20 Dockerfile is identical to that for Heroku-18, other
than the base image, and stack-related env var changes.

The initial Python versions made available will be those in:
https://devcenter.heroku.com/articles/python-support#supported-runtimes
https://devcenter.heroku.com/articles/python-support#supported-pypy-runtimes

...minus CPython 2.7, since it's EOL.

Which are:
* `python-3.6.12`
* `python-3.7.9`
* `python-3.8.6`
* `python-3.9.0`
* `pypy2.7-7.3.2`
* `pypy3.6-7.3.2`

Note: Unlike CPython 2.7, the PyPy 2.7 branch is still supported:
https://doc.pypy.org/en/latest/faq.html#how-long-will-pypy-support-python2

In addition, I've generated binaries for each patch release immediately
prior to the latest versions (with the exception of 3.9.0, since there
isn't one), otherwise it's not possible to run the "out of date Python"
warning tests.

The binaries were generated using the process here:
https://github.com/heroku/heroku-buildpack-python/blob/main/builds/README.md

Specifically:

```
make deploy-runtimes STACKS='heroku-20' \
  RUNTIMES='python-3.6.11 python-3.6.12 python-3.7.8 python-3.7.9 python-3.8.5 python-3.8.6 python-3.9.0 pypy2.7-7.3.1 pypy2.7-7.3.2 pypy3.6-7.3.1 pypy3.6-7.3.2' \
  ENV_FILE=...
```

Binaries for the GDAL/GEOS/PROJ feature have not been generated, since
it's deprecated and due for removal shortly:
https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-now-deprecated

Note: Like the Python 3.9.0 release, this uses the new S3 bucket, so
apps will need to be using a recent version of the buildpack in order
to build on Heroku-20:
https://devcenter.heroku.com/articles/python-support#checking-the-python-buildpack-version

Closes @W-7485877@.

Author: edmorley

.travis.yml

@@ -34,6 +34,10 @@ env:
     - STACK=heroku-18 TEST_CMD=test/run-deps
     - STACK=heroku-18 TEST_CMD=test/run-versions
     - STACK=heroku-18 TEST_CMD=test/run-features
+
+    - STACK=heroku-20 TEST_CMD=test/run-deps
+    - STACK=heroku-20 TEST_CMD=test/run-versions
+    - STACK=heroku-20 TEST_CMD=test/run-features
   global:
     - HATCHET_RETRIES=3
     - IS_RUNNING_ON_CI=true

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add support for Heroku-20 (#968).
 
 ## v182 (2020-10-06)
 

bin/steps/gdal

@@ -21,8 +21,11 @@ source "$BIN_DIR/utils"
 # If GDAL exists within requirements, use vendored gdal.
 if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
 
-  if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
+  if [[ ! -f ".heroku/vendor/bin/gdalserver" && "${STACK}" == 'heroku-20' ]]; then
+    puts-warn "The buildpack's built-in GDAL functonality is not supported on Heroku-20."
+    puts-warn "Please use this buildpack instead: https://github.com/heroku/heroku-geo-buildpack"
 
+  elif [[ ! -f ".heroku/vendor/bin/gdalserver" ]]; then
     puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated."
     puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack"
 

bin/steps/geo-libs

@@ -20,8 +20,11 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
-# If GDAL exists within requirements, use vendored gdal.
-if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
+if [[ "$BUILD_WITH_GEO_LIBRARIES" && "${STACK}" == 'heroku-20' ]]; then
+  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20."
+  puts-warn "Please use this buildpack for GDAL, GEOS and PROJ: https://github.com/heroku/heroku-geo-buildpack"
+  puts-warn "To hide this message, unset the BUILD_WITH_GEO_LIBRARIES variable using: heroku config:unset BUILD_WITH_GEO_LIBRARIES"
+elif [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
   mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"
 
   puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."

builds/heroku-20.Dockerfile

@@ -0,0 +1,20 @@
+FROM heroku/heroku:20-build
+
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="heroku-buildpack-python" \
+    S3_PREFIX="heroku-20/" \
+    STACK="heroku-20"
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt /app/
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+
+COPY . /app

test/fixtures/pipenv-full-version/Pipfile

@@ -6,4 +6,4 @@ verify_ssl = true
 requests = "*"
 
 [requires]
-python_full_version = "3.6.3"
+python_full_version = "3.7.8"

test/fixtures/pipenv-full-version/Pipfile.lock

@@ -1 +1 @@
-python-3.6.7
+python-3.6.11

test/fixtures/python3_6_warn/runtime.txt

@@ -1 +1 @@
-python-3.7.1
+python-3.7.8

test/fixtures/python3_7_warn/runtime.txt

@@ -1 +1 @@
-python-3.8.0
+python-3.8.5

test/fixtures/python3_8_warn/runtime.txt

@@ -21,16 +21,13 @@ testGEOS() {
   local env_dir="$(mktmpdir)"
   echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
   compile 'geos' '' "${env_dir}"
-  assertCaptured "geos"
-  assertCapturedSuccess
-}
-
-testGEOSDeprecation() {
-  local env_dir="$(mktmpdir)"
-  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
-  compile 'geos' '' "${env_dir}"
-  assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated.
- !     An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
+  if [[ $STACK == "heroku-20" ]]; then
+    assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20."
+  else
+    assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
+  fi
+  # This should assertCapturedError on Heroku-20, but the test doesn't actually
+  # install anything that uses GEOS so succeeds (see W-8145375)
   assertCapturedSuccess
 }
 
@@ -55,6 +52,11 @@ testPsycopg2() {
 }
 
 testPysqlite() {
+  # pysqlite does not support Python 3 (since the sqlite3 stdlib can be used there),
+  # so we have to test with Python 2, which we've not made available for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "pysqlite"
   assertCaptured "pysqlite"
   assertCapturedSuccess

test/run-deps

@@ -68,6 +68,10 @@ testPipenvPythonVersion3_6() {
 }
 
 testPipenvPythonVersion2_7() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "pipenv-version2"
   assertCaptured "Installing ${LATEST_27}"
   # Can't use `assertCapturedSuccess` since stderr contains:
@@ -76,8 +80,12 @@ testPipenvPythonVersion2_7() {
 }
 
 testPipenvPythonFullVersion() {
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
+  fi
   compile "pipenv-full-version"
-  assertCaptured "3.6.3"
+  assertCaptured "3.7.8"
   # Can't use `assertCapturedSuccess` since stderr contains:
   # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
   assertCapturedSuccessWithStdErr

test/run-features

@@ -14,6 +14,10 @@ testPythonVersionUnspecified() {
 }
 
 testPython2_7() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python2"
   assertCaptured "Installing ${LATEST_27}"
   assertCaptured "python-2-7-eol-faq";
@@ -24,6 +28,10 @@ testPython2_7() {
 }
 
 testPython2_7_warn() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python2_warn"
   assertCaptured "Installing python-2.7.15"
   assertCaptured "python-2-7-eol-faq";
@@ -33,6 +41,10 @@ testPython2_7_warn() {
 }
 
 testPython3_4() {
+  # Python 3.4 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_4"
   assertCaptured "Installing ${LATEST_34}"
   assertNotCaptured "security update"
@@ -44,6 +56,10 @@ testPython3_4() {
 }
 
 testPython3_4_warn() {
+  # Python 3.4 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_4_warn"
   assertCaptured "Installing python-3.4.9"
   assertCaptured "security update!"
@@ -54,6 +70,10 @@ testPython3_4_warn() {
 }
 
 testPython3_5() {
+  # Python 3.5 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_5"
   assertCaptured "Installing ${LATEST_35}"
   assertNotCaptured "security update"
@@ -63,6 +83,10 @@ testPython3_5() {
 }
 
 testPython3_5_warn() {
+  # Python 3.5 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_5_warn"
   assertCaptured "Installing python-3.5.6"
   assertCaptured "security update!"
@@ -81,7 +105,7 @@ testPython3_6() {
 
 testPython3_6_warn() {
   compile "python3_6_warn"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
   assertCaptured "security update!"
   assertCaptured "${LATEST_36}"
   assertCapturedSuccess
@@ -106,7 +130,7 @@ testPython3_7_warn() {
     return
   fi
   compile "python3_7_warn"
-  assertCaptured "Installing python-3.7.1"
+  assertCaptured "Installing python-3.7.8"
   assertCaptured "security update!"
   assertCaptured "${LATEST_37}"
   assertCapturedSuccess
@@ -131,7 +155,7 @@ testPython3_8_warn() {
     return
   fi
   compile "python3_8_warn"
-  assertCaptured "Installing python-3.8.0"
+  assertCaptured "Installing python-3.8.5"
   assertCaptured "security update!"
   assertCaptured "${LATEST_38}"
   assertCapturedSuccess
@@ -208,7 +232,7 @@ testPypy2_7_warn() {
 testStickyPythonVersion() {
   local cache_dir="$(mktmpdir)"
   compile "python3_6_warn" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
   assertCapturedSuccess
   compile "python_version_unspecified" "$cache_dir"
   assertNotCaptured "Installing python"
@@ -217,16 +241,16 @@ testStickyPythonVersion() {
   # Whilst this file seems like an implementation detail (so something that should
   # not be tested), we must guarantee the filename remains consistent for backwards
   # compatibility across buildpack versions for already-built apps.
-  assertFile "python-3.6.7" ".heroku/python-version"
+  assertFile "python-3.6.11" ".heroku/python-version"
 }
 
 testPythonVersionChange() {
   local cache_dir="$(mktmpdir)"
   compile "python3_6_warn" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
   assertCapturedSuccess
   compile "python3_6" "$cache_dir"
-  assertCaptured "Found python-3.6.7, removing"
+  assertCaptured "Found python-3.6.11, removing"
   assertCapturedSuccess
 }