From d7d7302edb23f086520c0dd770ac771062791f8c Mon Sep 17 00:00:00 2001
From: Vishal Gupta <vishg@google.com>
Date: Mon, 27 Jan 2025 18:17:17 +0530
Subject: [PATCH] Fix Python requirement logic (#1201)

---
 .../executor/app/executor/import_executor.py       | 14 ++++++++++----
 import-automation/executor/requirements.txt        | 14 +++++++-------
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/import-automation/executor/app/executor/import_executor.py b/import-automation/executor/app/executor/import_executor.py
index 0788d7c4e..33740da08 100644
--- a/import-automation/executor/app/executor/import_executor.py
+++ b/import-automation/executor/app/executor/import_executor.py
@@ -353,9 +353,12 @@ def _invoke_import_validation(self, repo_dir: str, relative_import_dir: str,
             validation_output_path = os.path.join(absolute_import_dir,
                                                   'validation')
             config_file = import_spec.get('validation_config_file', '')
-            if not config_file:
-                config_file = self.config.validation_config_file
-            config_file_path = os.path.join(REPO_DIR, config_file)
+            if config_file:
+                config_file_path = os.path.join(absolute_import_dir,
+                                                config_file)
+            else:
+                config_file_path = os.path.join(
+                    repo_dir, self.config.validation_config_file)
             logging.info(f'Validation config file: {config_file_path}')
 
             # Download previous import data.
@@ -445,8 +448,11 @@ def _import_one_helper(
         with tempfile.TemporaryDirectory() as tmpdir:
             requirements_path = os.path.join(absolute_import_dir,
                                              self.config.requirements_filename)
+            central_requirements_path = os.path.join(
+                repo_dir, 'import-automation', 'executor',
+                self.config.requirements_filename)
             interpreter_path, process = _create_venv(
-                (requirements_path),
+                (central_requirements_path, requirements_path),
                 tmpdir,
                 timeout=self.config.venv_create_timeout,
             )
diff --git a/import-automation/executor/requirements.txt b/import-automation/executor/requirements.txt
index 7f489f74a..92692ffb3 100644
--- a/import-automation/executor/requirements.txt
+++ b/import-automation/executor/requirements.txt
@@ -2,10 +2,11 @@
 
 absl-py
 arcgis2geojson
+beautifulsoup4
 chardet
 croniter
 dataclasses==0.6
-datacommons==1.4.3
+datacommons
 frozendict
 func-timeout==4.3.5
 geojson==2.5.0
@@ -13,9 +14,9 @@ google-auth
 google-cloud-bigquery
 google-cloud-datastore
 google-cloud-run
-google-cloud-storage>=2.7.0
-google-cloud-logging==3.4.0
-google-cloud-scheduler==2.10.0
+google-cloud-storage
+google-cloud-logging
+google-cloud-scheduler
 gspread==5.12.0
 gunicorn
 lxml==4.9.1
@@ -27,6 +28,7 @@ psutil
 pylint
 pytest
 pytz
+ratelimit
 requests==2.27.1
 requests_cache
 retry==0.9.2
@@ -34,7 +36,5 @@ shapely==1.8.5
 urllib3==1.26.8
 xarray==0.19.0
 xlrd
-zipp
-beautifulsoup4
-ratelimit
 xlsxwriter==3.2.0
+zipp