Clean up Python requirements (#1199)

import-automation/executor/app/executor/import_executor.py

@@ -445,10 +445,8 @@ def _import_one_helper(
         with tempfile.TemporaryDirectory() as tmpdir:
             requirements_path = os.path.join(absolute_import_dir,
                                              self.config.requirements_filename)
-            central_requirements_path = os.path.join(
-                repo_dir, self.config.requirements_filename)
             interpreter_path, process = _create_venv(
-                (central_requirements_path, requirements_path),
+                (requirements_path),
                 tmpdir,
                 timeout=self.config.venv_create_timeout,
             )

import-automation/executor/requirements.txt

@@ -1,15 +1,40 @@
-requests
-protobuf
+# Requirements for Python scripts in this repo that have automation enabled!
+
+absl-py
+arcgis2geojson
+chardet
+croniter
+dataclasses==0.6
+datacommons==1.4.3
+frozendict
+func-timeout==4.3.5
+geojson==2.5.0
 google-auth
-google-cloud-logging
-google-cloud-run
-google-cloud-storage
+google-cloud-bigquery
 google-cloud-datastore
+google-cloud-run
+google-cloud-storage>=2.7.0
+google-cloud-logging==3.4.0
 google-cloud-scheduler==2.10.0
-gspread
-flask
+gspread==5.12.0
 gunicorn
-pytz
-absl-py
-croniter
+lxml==4.9.1
+numpy==1.26.4
+openpyxl>=3.1.0
 pandas
+protobuf
+psutil
+pylint
+pytest
+pytz
+requests==2.27.1
+requests_cache
+retry==0.9.2
+shapely==1.8.5
+urllib3==1.26.8
+xarray==0.19.0
+xlrd
+zipp
+beautifulsoup4
+ratelimit
+xlsxwriter==3.2.0

requirements_all.txt

@@ -1,46 +1,22 @@
 # Requirements for all Python code in this repo, except for import-automation
 
-absl-py
-arcgis2geojson
+# Add new modules to one of:
+# - import-automation/executor/requirements.txt: if the script is used in import automation.
+# - script-folder/requirements.txt: the module is only needed for that script.
+# - requirements_all.txt (here): anything not related to import automation.
+
+-r import-automation/executor/requirements.txt
+
 chembl-webresource-client>=0.10.2
-chardet
-dataclasses==0.6
-datacommons==1.4.3
 deepdiff==6.3.0
 earthengine-api
 flask_restful==0.3.9
-frozendict
-func-timeout==4.3.5
-geojson==2.5.0
 geopandas==0.8.1
 geopy
-google-cloud-run
-google-cloud-bigquery
-google-cloud-storage>=2.7.0
-google-cloud-logging==3.4.0
-google-cloud-scheduler==2.10.0
-gspread==5.12.0
-lxml==4.9.1
 matplotlib==3.3.0
 netCDF4==1.6.4
-numpy==1.26.4
-openpyxl>=3.1.0
-pandas
-psutil
-pylint
-pytest
 rasterio
 rdp==0.8
-requests==2.27.1
-retry==0.9.2
 s2sphere==0.2.5
-shapely==1.8.5
 tabula-py
-urllib3==1.26.8
-xarray==0.19.0
-xlrd
 yapf
-zipp
-beautifulsoup4
-ratelimit
-xlsxwriter==3.2.0