Skip to content

Commit a98ef91

Browse files
authored
Tests: Clean up the Python version unit tests (heroku#1092)
* Fixes the "Installing <version>" assertions so that they don't false positive against the "please upgrade to <version>" output. * Removes modification of test fixtures during tests, since it can lead to failures depending on test order, and confusion when debugging. * Updates the PyPy version warning tests to use a slightly newer (but still not latest) PyPy version, which means that the test now passes on Cedar-14 and can be unskipped. * Switches to using an empty requirements file for version tests that duplicate the main test, to save spending time installing dependencies unnecessarily. * Switches the NLTK test to using the default buildpack Python version, rather than an ancient Python 3.6. * Skips the Python 3.7/3.8 tests on Cedar-14 rather than asserting failure, since we know they'll never pass due to Cedar-14's libssl being older than required. * Removes redundant `testSqliteInstall` test since it duplicates the Python version install tests. Longer term I'll be moving many of the unit tests to Hatchet, however this at least makes the tests more dependable in the meantime. Closes @W-8060219@. Closes @W-8176779@. [skip changelog]
1 parent e67235f commit a98ef91

File tree

18 files changed

+89
-118
lines changed

18 files changed

+89
-118
lines changed

test/fixtures/nltk/runtime.txt

Lines changed: 0 additions & 1 deletion
This file was deleted.
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
pypy2.7-7.2.0
1+
pypy2.7-7.3.1
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
pypy3.6-7.2.0
1+
pypy3.6-7.3.1
Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
requests
Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
flask
Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
flask

test/fixtures/python3_6/runtime.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
python-3.6.8
1+
python-3.6.12
Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
requests

test/fixtures/python3_7/runtime.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
python-3.7.2
1+
python-3.7.9
Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
requests

test/fixtures/python3_8/runtime.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
python-3.8.2
1+
python-3.8.6
Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
requests

test/fixtures/pythonDefault/requirements.txt

Lines changed: 0 additions & 1 deletion
This file was deleted.

test/run-deps

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,6 @@ testPysqlite() {
6060
assertCapturedSuccess
6161
}
6262

63-
testSqliteInstall() {
64-
compile "pythonDefault"
65-
assertNotCaptured "Sqlite3 failed to install."
66-
assertCapturedSuccess
67-
}
68-
6963
testCffi() {
7064
compile "cffi"
7165
assertCaptured "cffi"

test/run-features

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,22 +59,23 @@ testPipenvLock() {
5959
assertCapturedSuccessWithStdErr
6060
}
6161

62-
testPipenvVersion() {
62+
testPipenvPythonVersion3_6() {
6363
compile "pipenv-version"
64-
assertCaptured $DEFAULT_PYTHON_VERSION
64+
assertCaptured "Installing ${LATEST_36}"
6565
# Can't use `assertCapturedSuccess` since stderr contains:
6666
# "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
6767
assertCapturedSuccessWithStdErr
6868
}
6969

70-
testPipenvVersion2() {
70+
testPipenvPythonVersion2_7() {
7171
compile "pipenv-version2"
72-
assertCaptured $LATEST_27
72+
assertCaptured "Installing ${LATEST_27}"
7373
# Can't use `assertCapturedSuccess` since stderr contains:
7474
# "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
7575
assertCapturedSuccessWithStdErr
7676
}
77-
testPipenvFullVersion() {
77+
78+
testPipenvPythonFullVersion() {
7879
compile "pipenv-full-version"
7980
assertCaptured "3.6.3"
8081
# Can't use `assertCapturedSuccess` since stderr contains:

test/run-versions

Lines changed: 78 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -4,59 +4,58 @@
44
# shellcheck source=bin/default_pythons
55
source "bin/default_pythons"
66

7-
testPythonDefault() {
8-
updateVersion "pythonDefault" $DEFAULT_PYTHON_VERSION
9-
compile "pythonDefault"
10-
assertCaptured $DEFAULT_PYTHON_VERSION
7+
testPythonVersionUnspecified() {
8+
compile "python_version_unspecified"
9+
assertCaptured "Installing ${DEFAULT_PYTHON_VERSION}"
1110
assertNotCaptured "security update"
1211
assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
1312
assertCaptured "Installing SQLite3"
1413
assertCapturedSuccess
1514
}
1615

17-
testPython2() {
18-
updateVersion "python2" $LATEST_27
19-
echo $LATEST_27 > "runtime.txt"
20-
compile "python2"
21-
assertCaptured $LATEST_27
22-
assertCaptured "python-2-7-eol-faq";
23-
assertNotCaptured "security update"
24-
assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
25-
assertCaptured "Installing SQLite3"
26-
assertCapturedSuccess
16+
testPython2_7() {
17+
compile "python2"
18+
assertCaptured "Installing ${LATEST_27}"
19+
assertCaptured "python-2-7-eol-faq";
20+
assertNotCaptured "security update"
21+
assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
22+
assertCaptured "Installing SQLite3"
23+
assertCapturedSuccess
2724
}
2825

29-
testPython2_warn() {
30-
compile "python2_warn"
31-
assertCaptured "python-2.7.15"
32-
assertCaptured "python-2-7-eol-faq";
33-
assertCaptured "Only the latest version"
34-
assertCaptured "Installing SQLite3"
35-
assertCapturedSuccess
26+
testPython2_7_warn() {
27+
compile "python2_warn"
28+
assertCaptured "Installing python-2.7.15"
29+
assertCaptured "python-2-7-eol-faq";
30+
assertCaptured "Only the latest version"
31+
assertCaptured "${LATEST_27}"
32+
assertCapturedSuccess
3633
}
3734

3835
testPython3_4() {
3936
compile "python3_4"
40-
assertCaptured $LATEST_34
37+
assertCaptured "Installing ${LATEST_34}"
4138
assertNotCaptured "security update"
4239
assertCaptured "Installing pip 19.1.1, setuptools 43.0.0 and wheel 0.33.6"
40+
assertCaptured "Installing SQLite3"
4341
# Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
4442
# and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
4543
assertCapturedSuccessWithStdErr
4644
}
4745

4846
testPython3_4_warn() {
4947
compile "python3_4_warn"
50-
assertCaptured "python-3.4.9"
48+
assertCaptured "Installing python-3.4.9"
5149
assertCaptured "security update!"
50+
assertCaptured "${LATEST_34}"
5251
# Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
5352
# and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
5453
assertCapturedSuccessWithStdErr
5554
}
5655

5756
testPython3_5() {
5857
compile "python3_5"
59-
assertCaptured $LATEST_35
58+
assertCaptured "Installing ${LATEST_35}"
6059
assertNotCaptured "security update"
6160
assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
6261
assertCaptured "Installing SQLite3"
@@ -65,15 +64,15 @@ testPython3_5() {
6564

6665
testPython3_5_warn() {
6766
compile "python3_5_warn"
68-
assertCaptured "python-3.5.6"
67+
assertCaptured "Installing python-3.5.6"
6968
assertCaptured "security update!"
69+
assertCaptured "${LATEST_35}"
7070
assertCapturedSuccess
7171
}
7272

7373
testPython3_6() {
74-
updateVersion "python3_6" $LATEST_36
7574
compile "python3_6"
76-
assertCaptured $LATEST_36
75+
assertCaptured "Installing ${LATEST_36}"
7776
assertNotCaptured "security update"
7877
assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
7978
assertCaptured "Installing SQLite3"
@@ -82,62 +81,60 @@ testPython3_6() {
8281

8382
testPython3_6_warn() {
8483
compile "python3_6_warn"
85-
assertCaptured "python-3.6.7"
84+
assertCaptured "Installing python-3.6.7"
8685
assertCaptured "security update!"
87-
assertCaptured "Installing SQLite3"
86+
assertCaptured "${LATEST_36}"
8887
assertCapturedSuccess
8988
}
9089

9190
testPython3_7() {
92-
updateVersion "python3_7" $LATEST_37
93-
compile "python3_7"
94-
if [[ $STACK = "cedar-14" ]]; then
95-
assertCapturedError
96-
else
97-
assertNotCaptured "security update"
98-
assertCaptured $LATEST_37
99-
assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
100-
assertCaptured "Installing SQLite3"
101-
assertCapturedSuccess
91+
# Python 3.7+ requires newer libssl than is present on Cedar-14.
92+
if [[ "${STACK}" = "cedar-14" ]]; then
93+
return
10294
fi
95+
compile "python3_7"
96+
assertCaptured "Installing ${LATEST_37}"
97+
assertNotCaptured "security update"
98+
assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
99+
assertCaptured "Installing SQLite3"
100+
assertCapturedSuccess
103101
}
104102

105103
testPython3_7_warn() {
106-
compile "python3_7_warn"
107-
if [[ $STACK = "cedar-14" ]]; then
108-
assertCapturedError
109-
else
110-
assertCaptured "python-3.7.1"
111-
assertCaptured "security update!"
112-
assertCaptured "Installing SQLite3"
113-
assertCapturedSuccess
104+
# Python 3.7+ requires newer libssl than is present on Cedar-14.
105+
if [[ "${STACK}" = "cedar-14" ]]; then
106+
return
114107
fi
108+
compile "python3_7_warn"
109+
assertCaptured "Installing python-3.7.1"
110+
assertCaptured "security update!"
111+
assertCaptured "${LATEST_37}"
112+
assertCapturedSuccess
115113
}
116114

117115
testPython3_8() {
118-
updateVersion "python3_8" $LATEST_38
119-
compile "python3_8"
120-
if [[ $STACK = "cedar-14" ]]; then
121-
assertCapturedError
122-
else
123-
assertNotCaptured "security update"
124-
assertCaptured $LATEST_38
125-
assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
126-
assertCaptured "Installing SQLite3"
127-
assertCapturedSuccess
116+
# Python 3.7+ requires newer libssl than is present on Cedar-14.
117+
if [[ "${STACK}" = "cedar-14" ]]; then
118+
return
128119
fi
120+
compile "python3_8"
121+
assertCaptured "Installing ${LATEST_38}"
122+
assertNotCaptured "security update"
123+
assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
124+
assertCaptured "Installing SQLite3"
125+
assertCapturedSuccess
129126
}
130127

131128
testPython3_8_warn() {
132-
compile "python3_8_warn"
133-
if [[ $STACK = "cedar-14" ]]; then
134-
assertCapturedError
135-
else
136-
assertCaptured "python-3.8.0"
137-
assertCaptured "security update!"
138-
assertCaptured "Installing SQLite3"
139-
assertCapturedSuccess
129+
# Python 3.7+ requires newer libssl than is present on Cedar-14.
130+
if [[ "${STACK}" = "cedar-14" ]]; then
131+
return
140132
fi
133+
compile "python3_8_warn"
134+
assertCaptured "Installing python-3.8.0"
135+
assertCaptured "security update!"
136+
assertCaptured "${LATEST_38}"
137+
assertCapturedSuccess
141138
}
142139

143140
testPython3_9() {
@@ -146,8 +143,8 @@ testPython3_9() {
146143
return
147144
fi
148145
compile "python3_9"
146+
assertCaptured "Installing ${LATEST_39}"
149147
assertNotCaptured "security update"
150-
assertCaptured $LATEST_39
151148
assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
152149
assertCaptured "Installing SQLite3"
153150
assertCapturedSuccess
@@ -163,8 +160,9 @@ testPython3_9_warn() {
163160
return
164161
fi
165162
compile "python3_9_warn"
166-
assertCaptured "python-3.9.0"
163+
assertCaptured "Installing python-3.9.0"
167164
assertCaptured "security update!"
165+
assertCaptured "${LATEST_39}"
168166
assertCapturedSuccess
169167
}
170168

@@ -177,53 +175,44 @@ testPythonVersionInvalid() {
177175

178176
testPypy3_6() {
179177
compile "pypy3_6"
180-
assertCaptured "Installing pypy"
178+
assertCaptured "Installing ${LATEST_PYPY_36}"
181179
assertNotCaptured "security update"
182-
assertCaptured "$LATEST_PYPY_36"
183180
assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
184181
assertCapturedSuccess
185182
}
186183

187184
testPypy3_6_warn() {
188185
compile "pypy3_6_warn"
189-
if [[ $STACK = "cedar-14" ]]; then
190-
assertCapturedError
191-
else
192-
assertCaptured "Installing pypy"
193-
assertCaptured "security update!"
194-
assertCaptured "$LATEST_PYPY_36"
195-
assertCapturedSuccess
196-
fi
186+
assertCaptured "Installing pypy3.6-7.3.1"
187+
assertCaptured "security update!"
188+
assertCaptured "${LATEST_PYPY_36}"
189+
assertCapturedSuccess
197190
}
198191

199192
testPypy2_7() {
200193
compile "pypy2_7"
201-
assertCaptured "Installing pypy"
194+
assertCaptured "Installing ${LATEST_PYPY_27}"
202195
assertNotCaptured "security update"
203-
assertCaptured "$LATEST_PYPY_27"
204196
assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
205197
assertCapturedSuccess
206198
}
207199

208200
testPypy2_7_warn() {
209201
compile "pypy2_7_warn"
210-
if [[ $STACK = "cedar-14" ]]; then
211-
assertCapturedError
212-
else
213-
assertCaptured "Installing pypy"
214-
assertCaptured "security update!"
215-
assertCaptured "$LATEST_PYPY_27"
216-
assertCapturedSuccess
217-
fi
202+
assertCaptured "Installing pypy2.7-7.3.1"
203+
assertCaptured "security update!"
204+
assertCaptured "${LATEST_PYPY_27}"
205+
assertCapturedSuccess
218206
}
219207

220208
testStickyPythonVersion() {
221209
local cache_dir="$(mktmpdir)"
222210
compile "python3_6_warn" "$cache_dir"
223211
assertCaptured "Installing python-3.6.7"
224212
assertCapturedSuccess
225-
compile "no-runtime-txt" "$cache_dir"
226-
assertCaptured "Installing python-3.6.7"
213+
compile "python_version_unspecified" "$cache_dir"
214+
assertNotCaptured "Installing python"
215+
assertCaptured "security update!"
227216
assertCapturedSuccess
228217
# Whilst this file seems like an implementation detail (so something that should
229218
# not be tested), we must guarantee the filename remains consistent for backwards

test/utils

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,6 @@ resetCapture()
5656
unset rtrn # deprecated
5757
}
5858

59-
updateVersion()
60-
{
61-
echo "$2" > "test/fixtures/${1}/runtime.txt"
62-
}
63-
6459
assertCapturedEquals()
6560
{
6661
assertEquals "$@" "$(cat ${STD_OUT})"

0 commit comments

Comments
 (0)