Merge pull request #12 from NigelHambly/main

Completed DR3 set up with all available tables
wfau · Feb 13, 2023 · e4ed86d · e4ed86d
2 parents ec0310d + 94ca8bd
commit e4ed86d
Show file tree

Hide file tree

Showing 3 changed files with 397 additions and 124 deletions.
diff --git a/gaiadmpsetup/gaiadmpsetup.py b/gaiadmpsetup/gaiadmpsetup.py
@@ -45,7 +45,8 @@ def tablesExist(expected_tables, database):
             for table_key in edr3.table_dict.keys():
                 folder_path = edr3.table_dict[table_key][1]
                 schemas = edr3.table_dict[table_key][0]
-                reattachParquetFileResourceToSparkContext(table_key, data_store + folder_path, schemas)
+                pk = edr3.table_dict[table_key][2]
+                reattachParquetFileResourceToSparkContext(table_key, data_store + folder_path, schemas, cluster_key = pk, sort_key = pk)
 
         # check DR3
         database = "gaiadr3"
@@ -59,7 +60,8 @@ def tablesExist(expected_tables, database):
             for table_key in dr3.table_dict.keys():
                 folder_path = dr3.table_dict[table_key][1]
                 schemas = dr3.table_dict[table_key][0]
-                reattachParquetFileResourceToSparkContext(table_key, data_store + folder_path, schemas)
+                pk = dr3.table_dict[table_key][2]
+                reattachParquetFileResourceToSparkContext(table_key, data_store + folder_path, schemas, cluster_key = pk, sort_key = pk)
 
         # finally always leave the PySpark SQL context in the most recent Gaia DR3 database
         spark.sql("use gaiadr3")