Merge pull request #699 from cmu-delphi/gs-pull-from-bigquery

Convert Google Symptoms pipeline to pull data from BigQuery

Author: krivard

google_symptoms/Makefile

@@ -16,8 +16,13 @@ lint:
 	. env/bin/activate; pydocstyle $(dir)
 
 test:
-	. env/bin/activate ; (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing)
+	. env/bin/activate ;
+	(cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing)
 
 clean:
 	rm -rf env
 	rm -f params.json
+
+run:
+	env/bin/python -m $(dir)
+	env/bin/python -m delphi_utils.archive --archive_type s3 --indicator_prefix $(dir)

google_symptoms/README.md

@@ -1,8 +1,8 @@
 # Google Symptoms
 
-We import the normalized symptom search term popularity data from the Google 
-Research's Open COVID-19 Data project and export the county-level and state-level 
-data as-is.  We also aggregate the data to the MSA and HRR levels. For detailed 
+We import the normalized symptom search term popularity data from the Google
+Research's Open COVID-19 Data project via BigQuery and export the county-level and state-level
+data as-is.  We also aggregate the data to the MSA and HRR levels. For detailed
 information see the files `DETAILS.md` contained in this directory.
 
 ## Running the Indicator
@@ -17,19 +17,20 @@ make install
 ```
 
 This command will install the package in editable mode, so you can make changes that
-will automatically propagate to the installed package. 
+will automatically propagate to the installed package.
 
-All of the user-changable parameters are stored in `params.json`. To execute the module
+All of the user-changable parameters are stored in `params.json`. You will need to
+acquire a BigQuery API key with affiliated billing to fetch data. To execute the module
 and produce the output datasets (by default, in `receiving`), run the following.
 
 ```
 env/bin/python -m delphi_google_symptoms
 ```
 
-If you want to enter the virtual environment in your shell, 
-you can run `source env/bin/activate`. Run `deactivate` to leave the virtual environment. 
+If you want to enter the virtual environment in your shell,
+you can run `source env/bin/activate`. Run `deactivate` to leave the virtual environment.
 
-Once you are finished, you can remove the virtual environment and 
+Once you are finished, you can remove the virtual environment and
 params file with the following:
 
 ```
@@ -58,7 +59,7 @@ To run individual tests, run the following:
 ```
 
 The output will show the number of unit tests that passed and failed, along
-with the percentage of code covered by the tests. 
+with the percentage of code covered by the tests.
 
 None of the linting or unit tests should fail, and the code lines that are not covered by unit tests should be small and
-should not include critical sub-routines. 
+should not include critical sub-routines.

google_symptoms/delphi_google_symptoms/constants.py

@@ -12,12 +12,12 @@
 COMBINED_METRIC = "sum_anosmia_ageusia"
 SMOOTHERS = ["raw", "smoothed"]
 GEO_RESOLUTIONS = [
-        "state",
-        "county",
-        "msa",
-        "hrr",
-        "hhs",
-        "nation"
+    "state",
+    "county",
+    "msa",
+    "hrr",
+    "hhs",
+    "nation"
 ]
 
 seven_day_moving_average = partial(kday_moving_average, k=7)
@@ -26,19 +26,19 @@
     "smoothed":      (seven_day_moving_average, lambda d: d),
 }
 
-STATE_TO_ABBREV = {'Alabama':'al',
+STATE_TO_ABBREV = {'Alabama': 'al',
                    'Alaska': 'ak',
-#                   'American Samoa': 'as',
+                   #                   'American Samoa': 'as',
                    'Arizona': 'az',
                    'Arkansas': 'ar',
                    'California': 'ca',
                    'Colorado': 'co',
                    'Connecticut': 'ct',
                    'Delaware': 'de',
-#                   'District of Columbia': 'dc',
+                   #                   'District of Columbia': 'dc',
                    'Florida': 'fl',
                    'Georgia': 'ga',
-#                   'Guam': 'gu',
+                   #                   'Guam': 'gu',
                    'Hawaii': 'hi',
                    'Idaho': 'id',
                    'Illinois': 'il',
@@ -59,24 +59,24 @@
                    'Nevada': 'nv',
                    'New_Hampshire': 'nh',
                    'New_Jersey': 'nj',
-                   'New_Mexico':'nm',
+                   'New_Mexico': 'nm',
                    'New_York': 'ny',
                    'North_Carolina': 'nc',
                    'North_Dakota': 'nd',
-#                   'Northern Mariana Islands': 'mp',
+                   #                   'Northern Mariana Islands': 'mp',
                    'Ohio': 'oh',
                    'Oklahoma': 'ok',
                    'Oregon': 'or',
                    'Pennsylvania': 'pa',
-#                   'Puerto Rico': 'pr',
+                   #                   'Puerto Rico': 'pr',
                    'Rhode_Island': 'ri',
                    'South_Carolina': 'sc',
                    'South_Dakota': 'sd',
                    'Tennessee': 'tn',
                    'Texas': 'tx',
                    'Utah': 'ut',
                    'Vermont': 'vt',
-#                   'Virgin Islands': 'vi',
+                   #                   'Virgin Islands': 'vi',
                    'Virginia': 'va',
                    'Washington': 'wa',
                    'West_Virginia': 'wv',