incremental migrations for apogee spectra

sdss · Nov 13, 2024 · 22a0e4b · 22a0e4b
1 parent a000b3c
commit 22a0e4b
Show file tree

Hide file tree

Showing 9 changed files with 810 additions and 166 deletions.
diff --git a/bin/new_astra b/bin/new_astra
@@ -178,8 +178,12 @@ def migrate(
         migrate_from_spall_file,
         migrate_specfull_metadata_from_image_headers
     )
-    from astra.migrations.apogee import (
-        migrate_apvisit_metadata_from_image_headers,
+    #from astra.migrations.apogee import (
+    #    migrate_apvisit_metadata_from_image_headers,
+    #)
+    from astra.migrations.new_apogee import (
+        migrate_apogee_spectra_from_sdss5_apogee_drpdb,
+        migrate_dithered_metadata
     )
     from astra.migrations.catalog import (
         migrate_healpix,
@@ -239,7 +243,7 @@ def migrate(
                         from astra.migrations.apogee import migrate_sdss4_dr17_apogee_spectra_from_sdss5_catalogdb 
                         ptq.append(process_task(migrate_sdss4_dr17_apogee_spectra_from_sdss5_catalogdb, description="Ingesting APOGEE dr17 spectra"))
                     else:
-                        raise a
+                        ptq.append(process_task(migrate_apogee_spectra_from_sdss5_apogee_drpdb, apred, description=f"Ingesting APOGEE {apred} spectra"))
                 if run2d is not None:
                     ptq.append(process_task(migrate_from_spall_file, run2d, description=f"Ingesting BOSS {run2d} spectra"))
 
@@ -270,7 +274,8 @@ def migrate(
                 process_task(migrate_specfull_metadata_from_image_headers, description="Ingesting specFull metadata"),
 
 
-                process_task(migrate_apvisit_metadata_from_image_headers, description="Ingesting apVisit metadata"),                
+                process_task(migrate_dithered_metadata, description="Ingesting APOGEE dithered metadata"),
+                #process_task(migrate_apvisit_metadata_from_image_headers, description="Ingesting apVisit metadata"),                
                 process_task(migrate_healpix, description="Ingesting HEALPix values"),
                 process_task(migrate_tic_v8_identifier, description="Ingesting TIC v8 identifiers"),
                 process_task(update_galactic_coordinates, description="Computing Galactic coordinates"),

diff --git a/src/astra/migrations/boss.py b/src/astra/migrations/boss.py
@@ -7,10 +7,6 @@
 import concurrent.futures
 
 from astra.utils import log, expand_path, dict_to_iterable
-from astra.models.base import database
-from astra.models.boss import BossVisitSpectrum
-from astra.models.source import Source
-from astra.migrations.utils import enumerate_new_spectrum_pks, upsert_many, NoQueue
 
 from peewee import (
     chunked,
@@ -27,6 +23,11 @@ def migrate_from_spall_file(run2d, queue, gzip=True, limit=1_000_000, batch_size
     by the SDSS-V BOSS data reduction pipeline.
     """
 
+    from astra.models.base import database
+    from astra.models.boss import BossVisitSpectrum
+    from astra.models.source import Source
+    from astra.migrations.utils import enumerate_new_spectrum_pks, upsert_many, NoQueue
+
     #path = expand_path(f"$BOSS_SPECTRO_REDUX/{run2d}/spAll-{run2d}.fits")
     #if gzip:
     #    path += ".gz"
@@ -358,6 +359,11 @@ def migrate_specfull_metadata_from_image_headers(
     batch_size: Optional[int] = 100,
     queue = None
 ):
+    from astra.models.base import database
+    from astra.models.boss import BossVisitSpectrum
+    from astra.models.source import Source
+    from astra.migrations.utils import enumerate_new_spectrum_pks, upsert_many, NoQueue
+
     if queue is None:
         queue = NoQueue()