Integrate events importer into RSI. (#272)

datacommonsorg · Jan 18, 2024 · 347309f · 347309f
1 parent 93aa394
commit 347309f
Show file tree

Hide file tree

Showing 17 changed files with 606 additions and 540 deletions.
diff --git a/run_test.sh b/run_test.sh
@@ -106,10 +106,10 @@ function run_sample {
   python3 -m stats.main --input_dir=sample/input --output_dir=sample/output --freeze_time
 
   echo "Writing tables to CSVs."
-  mkdir -p sample/output/debug
-  sqlite3 -header -csv sample/output/datacommons.db "select * from observations;" > sample/output/debug/observations.csv
-  sqlite3 -header -csv sample/output/datacommons.db "select * from triples;" > sample/output/debug/triples.csv
-  sqlite3 -header -csv sample/output/datacommons.db "select * from imports;" > sample/output/debug/imports.csv
+  mkdir -p sample/output/tables
+  sqlite3 -header -csv sample/output/datacommons.db "select * from observations;" > sample/output/tables/observations.csv
+  sqlite3 -header -csv sample/output/datacommons.db "select * from triples;" > sample/output/tables/triples.csv
+  sqlite3 -header -csv sample/output/datacommons.db "select * from imports;" > sample/output/tables/imports.csv
 
   deactivate
 }

diff --git a/simple/sample/input/config.json b/simple/sample/input/config.json
@@ -1,29 +1,43 @@
 {
   "inputFiles": {
     "countries.csv": {
+      "importType": "observations",
       "entityType": "Country",
       "provenance": "Provenance1 Name"
     },
     "geoids.csv": {
+      "importType": "observations",
       "entityType": "",
       "ignoreColumns": ["ignore1", "ignore2"],
       "provenance": "Provenance1 Name"
     },
     "latlng.csv": {
+      "importType": "observations",
       "entityType": "Country",
       "provenance": "Provenance1 Name"
     },
     "powerplants.csv": {
+      "importType": "observations",
       "entityType": "PowerPlant",
       "provenance": "Provenance2 Name"
     },
     "s2cells.csv": {
+      "importType": "observations",
       "entityType": "S2CellLevel10",
       "provenance": "Provenance2 Name"
     },
     "wikidataids.csv": {
+      "importType": "observations",
       "entityType": "Country",
       "provenance": "Provenance1 Name"
+    },
+    "latlng_events.csv": {
+      "importType": "events",
+      "eventType": "CrimeEvent",
+      "entityType": "CensusZipCodeTabulationArea",
+      "provenance": "Provenance1 Name",
+      "idColumn": "CASE",
+      "computedVariables": ["Crime Count"]
     }
   },
   "variables": {
@@ -42,6 +56,13 @@
     },
     "Variable 1": {
       "group": "Parent Group"
+    },
+    "Crime Count": {
+      "description": "Number of crimes",
+      "aggregation": {
+        "period": "month",
+        "method": "count"
+      }
     }
   },
   "sources": {

diff --git a/simple/sample/input/latlng_events.csv b/simple/sample/input/latlng_events.csv
@@ -0,0 +1,51 @@
+lat#lng,DATE  OF OCCURRENCE,CASE,PRIMARY DESCRIPTION
+41.927407329#-87.70729439,11/08/2023 8:50:00 PM,JG497095,THEFT
+41.896671699#-87.628635323,11/08/2023 3:14:00 PM,JG496991,ASSAULT
+41.808525157#-87.672792896,11/08/2023 10:55:00 PM,JG497145,ASSAULT
+41.979505088#-87.693158103,11/08/2023 4:39:00 PM,JG496701,OTHER OFFENSE
+41.771890947#-87.638705659,10/28/2023 7:30:00 PM,JG484195,THEFT
+41.985611859#-87.713834343,10/28/2023 5:00:00 PM,JG483131,CRIMINAL DAMAGE
+41.733053891#-87.568330657,11/08/2023 8:25:00 AM,JG498494,ASSAULT
+41.949586612#-87.664085689,11/08/2023 2:38:00 PM,JG496575,THEFT
+41.704388397#-87.626879123,09/17/2023 3:00:00 AM,JG427641,THEFT
+41.881944424#-87.634195294,08/02/2023 9:25:00 AM,JG365961,ASSAULT
+41.755481563#-87.649019949,11/08/2023 9:00:00 AM,JG496115,THEFT
+41.970433391#-87.763029002,11/08/2023 7:45:00 PM,JG496955,ROBBERY
+41.802269632#-87.605372566,11/08/2023 3:00:00 PM,JG501047,BURGLARY
+41.721303358#-87.655873595,11/08/2023 1:00:00 PM,JG496779,ASSAULT
+41.884497529#-87.625838595,11/08/2023 11:00:00 AM,JG496296,THEFT
+41.778436411#-87.589657198,11/05/2023 6:00:00 PM,JG504330,OFFENSE INVOLVING CHILDREN
+41.838219696#-87.704850674,11/08/2023 1:49:00 PM,JG496568,ASSAULT
+41.70319162#-87.651369057,11/08/2023 11:30:00 AM,JG496295,ROBBERY
+41.883969722#-87.644191276,10/28/2023 9:00:00 AM,JG488191,MOTOR VEHICLE THEFT
+41.884276844#-87.622098929,10/28/2023 12:10:00 PM,JG482122,ROBBERY
+41.87493626#-87.748170814,10/28/2023 1:30:00 AM,JG481621,CRIMINAL DAMAGE
+41.95417672#-87.677232056,11/08/2023 6:30:00 PM,JG499040,CRIMINAL TRESPASS
+41.948044095#-87.664039332,11/08/2023 9:01:00 PM,JG497052,BURGLARY
+41.875625633#-87.629450396,12/14/2022 7:00:00 PM,JF511492,THEFT
+41.976489992#-87.788483018,11/08/2023 10:40:00 PM,JG498785,THEFT
+41.828080528#-87.686233684,11/08/2023 12:02:00 PM,JG496372,BURGLARY
+41.993043969#-87.660360363,10/28/2023 9:15:00 AM,JG481891,THEFT
+41.858444489#-87.716414102,10/28/2023 2:29:00 PM,JG482300,OTHER OFFENSE
+41.879874073#-87.769750673,10/28/2023 2:00:00 AM,JG482034,THEFT
+41.771296232#-87.729149311,08/31/2023 7:00:00 PM,JG406115,BATTERY
+41.875679322#-87.62657476,09/08/2023 8:00:00 AM,JG416492,DECEPTIVE PRACTICE
+41.82539977#-87.637026874,08/31/2023 10:52:00 AM,JG405111,BATTERY
+41.863196881#-87.614817819,12/07/2022 1:39:00 PM,JF501686,CRIMINAL TRESPASS
+41.93743245#-87.649180491,10/28/2023 7:41:00 AM,JG484000,THEFT
+41.680799541#-87.669942159,10/28/2023 10:40:00 PM,JG482660,ASSAULT
+41.891874434#-87.647617474,10/15/2023 2:30:00 AM,JG464444,ROBBERY
+41.888993854#-87.626934833,12/08/2022 3:38:00 PM,JF254640,HOMICIDE
+41.902821551#-87.775389625,10/28/2023 4:00:00 PM,JG483104,THEFT
+41.724654303#-87.622283278,08/31/2023 9:13:00 PM,JG412467,MOTOR VEHICLE THEFT
+41.773780824#-87.645848665,09/29/2023 12:00:00 AM,JG453780,THEFT
+41.841289747#-87.628142362,09/30/2023 11:45:00 PM,JG445684,BATTERY
+41.776150283#-87.615522623,11/08/2023 3:00:00 AM,JG495860,CRIMINAL DAMAGE
+41.836069707#-87.613033345,11/08/2023 3:00:00 PM,JG497647,MOTOR VEHICLE THEFT
+41.74974473#-87.652507329,09/30/2023 9:30:00 PM,JG445669,ASSAULT
+41.946653043#-87.700875462,09/30/2023 10:01:00 AM,JG445052,THEFT
+41.890400093#-87.628021143,10/28/2023 3:34:00 AM,JG481737,BATTERY
+41.793842185#-87.620286919,11/13/2023 9:30:00 AM,JG503521,BURGLARY
+41.720900408#-87.554599376,10/15/2023 3:00:00 AM,JG468840,CRIMINAL DAMAGE
+41.742267488#-87.702192623,10/15/2023 8:00:00 PM,JG465660,THEFT
+41.766298978#-87.570076538,10/15/2023 11:01:00 AM,JG464615,MOTOR VEHICLE THEFT
diff --git a/simple/sample/output/debug/imports.csv b/simple/sample/output/debug/imports.csv
diff --git a/simple/sample/output/debug/triples.csv b/simple/sample/output/debug/triples.csv
diff --git a/simple/sample/output/nl/sentences.csv b/simple/sample/output/nl/sentences.csv
@@ -3,3 +3,4 @@ var1,Good var1 name;Good var1 description;Natural language sentence 1;Natural la
 var2,Good var2 name
 Variable_1,Variable 1
 Variable_2,Variable 2
+Crime_Count,Crime Count;Number of crimes
diff --git a/simple/sample/output/process/debug_resolve_latlng_events.csv b/simple/sample/output/process/debug_resolve_latlng_events.csv
@@ -0,0 +1,51 @@
+input,dcid,link
+41.927407329#-87.70729439,zip/60647,https://datacommons.org/browser/zip/60647
+41.896671699#-87.628635323,zip/60654,https://datacommons.org/browser/zip/60654
+41.808525157#-87.672792896,zip/60609,https://datacommons.org/browser/zip/60609
+41.979505088#-87.693158103,zip/60625,https://datacommons.org/browser/zip/60625
+41.771890947#-87.638705659,zip/60621,https://datacommons.org/browser/zip/60621
+41.985611859#-87.713834343,zip/60659,https://datacommons.org/browser/zip/60659
+41.733053891#-87.568330657,zip/60617,https://datacommons.org/browser/zip/60617
+41.949586612#-87.664085689,zip/60613,https://datacommons.org/browser/zip/60613
+41.704388397#-87.626879123,zip/60628,https://datacommons.org/browser/zip/60628
+41.881944424#-87.634195294,zip/60606,https://datacommons.org/browser/zip/60606
+41.755481563#-87.649019949,zip/60620,https://datacommons.org/browser/zip/60620
+41.970433391#-87.763029002,zip/60630,https://datacommons.org/browser/zip/60630
+41.802269632#-87.605372566,zip/60615,https://datacommons.org/browser/zip/60615
+41.721303358#-87.655873595,zip/60643,https://datacommons.org/browser/zip/60643
+41.884497529#-87.625838595,zip/60601,https://datacommons.org/browser/zip/60601
+41.778436411#-87.589657198,zip/60637,https://datacommons.org/browser/zip/60637
+41.838219696#-87.704850674,zip/60623,https://datacommons.org/browser/zip/60623
+41.70319162#-87.651369057,zip/60643,https://datacommons.org/browser/zip/60643
+41.883969722#-87.644191276,zip/60661,https://datacommons.org/browser/zip/60661
+41.884276844#-87.622098929,zip/60601,https://datacommons.org/browser/zip/60601
+41.87493626#-87.748170814,zip/60644,https://datacommons.org/browser/zip/60644
+41.95417672#-87.677232056,zip/60613,https://datacommons.org/browser/zip/60613
+41.948044095#-87.664039332,zip/60613,https://datacommons.org/browser/zip/60613
+41.875625633#-87.629450396,zip/60605,https://datacommons.org/browser/zip/60605
+41.976489992#-87.788483018,zip/60630,https://datacommons.org/browser/zip/60630
+41.828080528#-87.686233684,zip/60609,https://datacommons.org/browser/zip/60609
+41.993043969#-87.660360363,zip/60660,https://datacommons.org/browser/zip/60660
+41.858444489#-87.716414102,zip/60623,https://datacommons.org/browser/zip/60623
+41.879874073#-87.769750673,zip/60644,https://datacommons.org/browser/zip/60644
+41.771296232#-87.729149311,zip/60629,https://datacommons.org/browser/zip/60629
+41.875679322#-87.62657476,zip/60605,https://datacommons.org/browser/zip/60605
+41.82539977#-87.637026874,zip/60609,https://datacommons.org/browser/zip/60609
+41.863196881#-87.614817819,zip/60605,https://datacommons.org/browser/zip/60605
+41.93743245#-87.649180491,zip/60657,https://datacommons.org/browser/zip/60657
+41.680799541#-87.669942159,zip/60643,https://datacommons.org/browser/zip/60643
+41.891874434#-87.647617474,zip/60642,https://datacommons.org/browser/zip/60642
+41.888993854#-87.626934833,zip/60611,https://datacommons.org/browser/zip/60611
+41.902821551#-87.775389625,zip/60651,https://datacommons.org/browser/zip/60651
+41.724654303#-87.622283278,zip/60619,https://datacommons.org/browser/zip/60619
+41.773780824#-87.645848665,zip/60621,https://datacommons.org/browser/zip/60621
+41.841289747#-87.628142362,zip/60616,https://datacommons.org/browser/zip/60616
+41.776150283#-87.615522623,zip/60637,https://datacommons.org/browser/zip/60637
+41.836069707#-87.613033345,zip/60616,https://datacommons.org/browser/zip/60616
+41.74974473#-87.652507329,zip/60620,https://datacommons.org/browser/zip/60620
+41.946653043#-87.700875462,zip/60618,https://datacommons.org/browser/zip/60618
+41.890400093#-87.628021143,zip/60611,https://datacommons.org/browser/zip/60611
+41.793842185#-87.620286919,zip/60637,https://datacommons.org/browser/zip/60637
+41.720900408#-87.554599376,zip/60617,https://datacommons.org/browser/zip/60617
+41.742267488#-87.702192623,zip/60652,https://datacommons.org/browser/zip/60652
+41.766298978#-87.570076538,zip/60649,https://datacommons.org/browser/zip/60649
diff --git a/simple/sample/output/process/report.json b/simple/sample/output/process/report.json
@@ -18,6 +18,11 @@
       "startTime": "2023-01-01 00:00:00",
       "lastUpdate": "2023-01-01 00:00:00"
     },
+    "latlng_events.csv": {
+      "status": "SUCCESS",
+      "startTime": "2023-01-01 00:00:00",
+      "lastUpdate": "2023-01-01 00:00:00"
+    },
     "powerplants.csv": {
       "status": "SUCCESS",
       "startTime": "2023-01-01 00:00:00",

diff --git a/simple/sample/output/tables/imports.csv b/simple/sample/output/tables/imports.csv
@@ -0,0 +1,2 @@
+imported_at,status,metadata
+"2023-01-01 00:00:00",SUCCESS,"{""numVars"": 5, ""numObs"": 99}"
diff --git a/simple/sample/output/debug/observations.csv → simple/sample/output/tables/observations.csv b/simple/sample/output/debug/observations.csv → simple/sample/output/tables/observations.csv
@@ -29,6 +29,47 @@ country/USA,var1,2021,555,c/p/1
 country/IND,var1,2022,321,c/p/1
 country/USA,var2,2021,666,c/p/1
 country/IND,var2,2022,123,c/p/1
+zip/60647,Crime_Count,2023-11,1,c/p/1
+zip/60654,Crime_Count,2023-11,1,c/p/1
+zip/60609,Crime_Count,2023-11,2,c/p/1
+zip/60625,Crime_Count,2023-11,1,c/p/1
+zip/60621,Crime_Count,2023-10,1,c/p/1
+zip/60659,Crime_Count,2023-10,1,c/p/1
+zip/60617,Crime_Count,2023-11,1,c/p/1
+zip/60613,Crime_Count,2023-11,3,c/p/1
+zip/60628,Crime_Count,2023-09,1,c/p/1
+zip/60606,Crime_Count,2023-08,1,c/p/1
+zip/60620,Crime_Count,2023-11,1,c/p/1
+zip/60630,Crime_Count,2023-11,2,c/p/1
+zip/60615,Crime_Count,2023-11,1,c/p/1
+zip/60643,Crime_Count,2023-11,2,c/p/1
+zip/60601,Crime_Count,2023-11,1,c/p/1
+zip/60637,Crime_Count,2023-11,3,c/p/1
+zip/60623,Crime_Count,2023-11,1,c/p/1
+zip/60661,Crime_Count,2023-10,1,c/p/1
+zip/60601,Crime_Count,2023-10,1,c/p/1
+zip/60644,Crime_Count,2023-10,2,c/p/1
+zip/60605,Crime_Count,2022-12,2,c/p/1
+zip/60660,Crime_Count,2023-10,1,c/p/1
+zip/60623,Crime_Count,2023-10,1,c/p/1
+zip/60629,Crime_Count,2023-08,1,c/p/1
+zip/60605,Crime_Count,2023-09,1,c/p/1
+zip/60609,Crime_Count,2023-08,1,c/p/1
+zip/60657,Crime_Count,2023-10,1,c/p/1
+zip/60643,Crime_Count,2023-10,1,c/p/1
+zip/60642,Crime_Count,2023-10,1,c/p/1
+zip/60611,Crime_Count,2022-12,1,c/p/1
+zip/60651,Crime_Count,2023-10,1,c/p/1
+zip/60619,Crime_Count,2023-08,1,c/p/1
+zip/60621,Crime_Count,2023-09,1,c/p/1
+zip/60616,Crime_Count,2023-09,1,c/p/1
+zip/60616,Crime_Count,2023-11,1,c/p/1
+zip/60620,Crime_Count,2023-09,1,c/p/1
+zip/60618,Crime_Count,2023-09,1,c/p/1
+zip/60611,Crime_Count,2023-10,1,c/p/1
+zip/60617,Crime_Count,2023-10,1,c/p/1
+zip/60652,Crime_Count,2023-10,1,c/p/1
+zip/60649,Crime_Count,2023-10,1,c/p/1
 dc/000qxlm93vn93,var1,2023,0.19,c/p/2
 dc/5c7tz3lbln3p,var1,2023,0.21,c/p/2
 dc/8zmh7ctlkbsc4,var1,2023,0.29,c/p/2
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		imported_at,status,metadata
		"2023-01-01 00:00:00",SUCCESS,"{""numVars"": 5, ""numObs"": 99}"