CASMPET-7288: Minimize Python packages virtual environment inherits f…

…rom system (#637)

README.md

@@ -83,4 +83,6 @@ These will be installed into the virtual environment when the RPM is built.
 
 Note that for a few dependencies, the dependency is fulfilled by inheriting the package from the system Python
 installation. This should not normally be done. If it does need to be done, be sure to list the required system
-Python package in the [csm-testing.spec](csm-testing.spec) file (along with applicable version constraints, if any).
+Python package in the [csm-testing-requirements.spec](inc/csm-testing-requirements.spec) file (along with applicable
+version constraints, if any). Also be sure to update [requirements.txt](requirements.txt) -- see that file for details
+on how to list a package that is being inherited from the system Python installation.

constraints.txt

@@ -1,7 +1,32 @@
-colorama>=0.4.5,<0.5
+attrs==22.2.0
+boto3==1.23.10
+botocore==1.26.10
+cachetools==4.2.4
+# certifi version should match the one in CSM 1.6 NCN images, since the virtual environment will be using it
+certifi==2018.1.18
+charset-normalizer==2.0.12
+colorama==0.4.5
 cray-product-catalog>=2.6,<2.7
-PyYAML>=6.0.1,<6.1
-# CASMPET-7271: This requests version is specified to match the version in the base NCN images,
-# to avoid version conflicts
-requests==2.25.1
+google-auth==2.22.0
+idna==3.10
+importlib-metadata==4.8.3
+jmespath==0.10.0
+jsonschema==3.2.0
+# CSM 1.6 uses Kubernetes v1.24
+kubernetes==24.2.0
+oauthlib==3.2.2
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pyrsistent==0.18.0
+python-dateutil==2.9.0
+PyYAML==6.0.1
+requests==2.27.1
+requests-oauthlib==2.0.0
 requests-retry-session>=0.1,<0.2
+rsa==4.9
+s3transfer==0.5.2
+six==1.16.0
+typing-extensions==4.1.1
+urllib3==1.26.20
+websocket-client==1.3.1
+zipp==3.6.0

csm-testing.spec

@@ -150,8 +150,8 @@ install -D -m 0644 -t %{buildroot}%{_unitdir}-preset/   systemd/90-goss-servers.
 %python_exec -m venv --system-site-packages %{buildroot}%{python_venv_dir}
 
 # Install csm-testing module + dependencies into virtual env
-%{buildroot}%{python_venv_bin} -m pip install --upgrade pip setuptools --no-cache --ignore-installed -c constraints.txt
-%{buildroot}%{python_venv_bin} -m pip install csm_testing*.whl -r requirements.txt --disable-pip-version-check --no-cache --ignore-installed
+%{buildroot}%{python_venv_bin} -m pip install --upgrade pip setuptools --no-cache-dir --ignore-installed -c constraints.txt
+%{buildroot}%{python_venv_bin} -m pip install csm_testing*.whl -r requirements.txt --disable-pip-version-check --no-cache-dir --ignore-installed
 
 # List what is in the virtual env, for the purposes of build logging
 %{buildroot}%{python_venv_bin} -m pip list --format freeze --local

inc/csm-testing-requirements.spec

@@ -55,12 +55,5 @@ Requires: util-linux-systemd
 
 Requires: (python%{python_version_nodots}-base or python3-base >= %{py_version})
 
-Requires: %{py_rpm_prefix}-boto3
-Requires: %{py_rpm_prefix}-botocore
 Requires: %{py_rpm_prefix}-certifi
-Requires: %{py_rpm_prefix}-chardet
-Requires: %{py_rpm_prefix}-idna
-Requires: %{py_rpm_prefix}-kubernetes
 Requires: %{py_rpm_prefix}-rados
-Requires: %{py_rpm_prefix}-requests
-Requires: %{py_rpm_prefix}-urllib3

requirements.txt

@@ -3,9 +3,13 @@
 --index-url https://arti.hpc.amslabs.hpecorp.net:443/artifactory/api/pypi/pypi-remote/simple
 --extra-index-url http://artifactory.algol60.net/artifactory/csm-python-modules/simple
 -c constraints.txt
+boto3
+botocore
 colorama
 cray-product-catalog
+kubernetes
 PyYAML
+requests
 requests-retry-session
 
 # There are some additional Python packages that test scripts can rely on that are
@@ -22,24 +26,7 @@ requests-retry-session
 
 #system_python:rados
 
-# For other Python modules that are closely coupled to software on the NCNs, we also rely on the
-# corresponding system Python packages. That way we do not need to make PRs to csm-testing as
-# these software versions change.
-
-#system_python:boto3
-#system_python:botocore
-#system_python:kubernetes
-
-# For the requests module, we want to use the system version to avoid SSL certificate verify issues.
-# There are other ways around this, but since our virtual environment is already inheriting from
-# the system Python install, this is the simplest way.
-
-#system_python:requests
-
-# CASMPET-7271: We also want to use the system version of the requests dependencies,
-# in order to avoid SSL errors or dependency warnings
+# CASMPET-7271: We also want to use the system version of certifi, in order to avoid SSL errors or
+# dependency warnings
 
 #system_python:certifi
-#system_python:chardet
-#system_python:idna
-#system_python:urllib3