Fix unit tests (#1208)

cloudbuild.py.yaml

@@ -10,13 +10,6 @@ steps:
     - |
       ./run_tests.sh -r
 
-# We can't just test everything under our Cloud Build working directory because
-# that's where python installs all of its packages also, and we don't really
-# want to test all of that other unrelated code everytime.
-#
-# The unittest module only supports one -s path at a time, so we need multiple rules
-# to get both places where we submit code.
-#
 - id: python_test
   name: python:3.11
   entrypoint: /bin/sh
@@ -25,9 +18,7 @@ steps:
   args:
     - -c
     - |
-      ./run_tests.sh -p util/
-      ./run_tests.sh -p import-automation/executor
-      ./run_tests.sh -p scripts/
+      ./run_tests.sh -a
 
 - id: python_format_check
   name: python:3.11

import-automation/executor/test/integration_test.py

@@ -27,10 +27,10 @@
     # The GitHub params belong to the public Data Commons gmail account.
     # Auth tokens, user name and other details can be found in the inbox
     # and in the inbox of teammates.
-    'github_repo_owner_username': os.environ['_GITHUB_REPO_OWNER_USERNAME'],
+    'github_repo_owner_username': os.getenv('_GITHUB_REPO_OWNER_USERNAME', ''),
     'github_repo_name': 'data-demo',
-    'github_auth_username': os.environ['_GITHUB_AUTH_USERNAME'],
-    'github_auth_access_token': os.environ['_GITHUB_AUTH_ACCESS_TOKEN']
+    'github_auth_username': os.getenv('_GITHUB_AUTH_USERNAME', ''),
+    'github_auth_access_token': os.getenv('_GITHUB_AUTH_ACCESS_TOKEN', '')
 }
 
 

tools/import_differ/import_differ.py

@@ -21,7 +21,7 @@
 from absl import flags
 from absl import logging
 
-import differ_utils
+from tools.import_differ import differ_utils
 
 SAMPLE_COUNT = 3
 GROUPBY_COLUMNS = 'variableMeasured,observationAbout,observationDate,measurementMethod,unit,observationPeriod'

tools/import_differ/import_differ_test.py

@@ -17,9 +17,8 @@
 import unittest
 
 from pandas.testing import assert_frame_equal
-from import_differ import ImportDiffer
-
-import differ_utils
+from tools.import_differ import import_differ
+from tools.import_differ import differ_utils
 
 module_dir = os.path.dirname(__file__)
 
@@ -37,8 +36,9 @@ def test_diff_analysis(self):
         previous_data = os.path.join(module_dir, 'test', 'previous.mcf')
         output_location = os.path.join(module_dir, 'test')
 
-        differ = ImportDiffer(current_data, previous_data, output_location,
-                              groupby_columns, value_columns)
+        differ = import_differ.ImportDiffer(current_data, previous_data,
+                                            output_location, groupby_columns,
+                                            value_columns)
         current = differ_utils.load_mcf_file(current_data)
         previous = differ_utils.load_mcf_file(previous_data)
 

tools/import_validation/import_validation.py

@@ -22,7 +22,7 @@
 import json
 
 FLAGS = flags.FLAGS
-flags.DEFINE_string('config_file', 'validation_config.json',
+flags.DEFINE_string('validation_config_file', 'validation_config.json',
                     'Path to the validation config file.')
 flags.DEFINE_string('differ_output_location', '.',
                     'Path to the differ output data folder.')
@@ -61,7 +61,7 @@ class ImportValidation:
   Class to perform validations for import automation.
 
   Usage:
-  $ python import_validation.py --config_file=<path> \
+  $ python import_validation.py --validation_config_file=<path> \
     --differ_output_location=<path> --stats_summary_location=<path>
 
   Each import can provide configuration (JSON) to select which validation
@@ -129,7 +129,7 @@ def run_validations(self):
 
 def main(_):
     validation = ImportValidation(
-        FLAGS.config_file,
+        FLAGS.validation_config_file,
         os.path.join(FLAGS.differ_output_location, POINT_ANALAYSIS_FILE),
         os.path.join(FLAGS.stats_summary_location, STATS_SUMMARY_FILE),
         os.paht.join(FLAGS.validation_output_location, VALIDATION_OUTPUT_FILE))

tools/import_validation/import_validation_test.py

@@ -17,7 +17,7 @@
 import unittest
 
 from pandas.testing import assert_frame_equal
-from import_validation import ImportValidation
+from tools.import_validation import import_validation
 
 module_dir = os.path.dirname(__file__)
 
@@ -32,8 +32,11 @@ def test_validation(self):
         result_file = os.path.join(module_dir, 'test', 'test_output.csv')
         config_file = os.path.join(module_dir, 'test', 'test_config.json')
         differ_output = os.path.join(module_dir, 'test', 'differ_output.csv')
+        validation_output = os.path.join(module_dir, 'validation_output.csv')
 
-        validation = ImportValidation(config_file, differ_output, '')
+        validation = import_validation.ImportValidation(config_file,
+                                                        differ_output, '',
+                                                        validation_output)
         validation.run_validations()
 
         expected = pd.read_csv(result_file)