Fix Python requirement logic (#1201)

datacommonsorg · Jan 27, 2025 · d7d7302 · d7d7302
1 parent ee824c1
commit d7d7302
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 11 deletions.
diff --git a/import-automation/executor/app/executor/import_executor.py b/import-automation/executor/app/executor/import_executor.py
@@ -353,9 +353,12 @@ def _invoke_import_validation(self, repo_dir: str, relative_import_dir: str,
             validation_output_path = os.path.join(absolute_import_dir,
                                                   'validation')
             config_file = import_spec.get('validation_config_file', '')
-            if not config_file:
-                config_file = self.config.validation_config_file
-            config_file_path = os.path.join(REPO_DIR, config_file)
+            if config_file:
+                config_file_path = os.path.join(absolute_import_dir,
+                                                config_file)
+            else:
+                config_file_path = os.path.join(
+                    repo_dir, self.config.validation_config_file)
             logging.info(f'Validation config file: {config_file_path}')
 
             # Download previous import data.
@@ -445,8 +448,11 @@ def _import_one_helper(
         with tempfile.TemporaryDirectory() as tmpdir:
             requirements_path = os.path.join(absolute_import_dir,
                                              self.config.requirements_filename)
+            central_requirements_path = os.path.join(
+                repo_dir, 'import-automation', 'executor',
+                self.config.requirements_filename)
             interpreter_path, process = _create_venv(
-                (requirements_path),
+                (central_requirements_path, requirements_path),
                 tmpdir,
                 timeout=self.config.venv_create_timeout,
             )

diff --git a/import-automation/executor/requirements.txt b/import-automation/executor/requirements.txt
@@ -2,20 +2,21 @@
 
 absl-py
 arcgis2geojson
+beautifulsoup4
 chardet
 croniter
 dataclasses==0.6
-datacommons==1.4.3
+datacommons
 frozendict
 func-timeout==4.3.5
 geojson==2.5.0
 google-auth
 google-cloud-bigquery
 google-cloud-datastore
 google-cloud-run
-google-cloud-storage>=2.7.0
-google-cloud-logging==3.4.0
-google-cloud-scheduler==2.10.0
+google-cloud-storage
+google-cloud-logging
+google-cloud-scheduler
 gspread==5.12.0
 gunicorn
 lxml==4.9.1
@@ -27,14 +28,13 @@ psutil
 pylint
 pytest
 pytz
+ratelimit
 requests==2.27.1
 requests_cache
 retry==0.9.2
 shapely==1.8.5
 urllib3==1.26.8
 xarray==0.19.0
 xlrd
-zipp
-beautifulsoup4
-ratelimit
 xlsxwriter==3.2.0
+zipp