diff --git a/.github/workflows/build_container.yaml b/.github/workflows/build_container.yaml
index 6c2f7cc28..44fa815cb 100644
--- a/.github/workflows/build_container.yaml
+++ b/.github/workflows/build_container.yaml
@@ -13,7 +13,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- - uses: lsst-sqre/build-and-push-to-ghcr@v1
+ - uses: lsst-sqre/build-and-push-to-ghcr@tickets/DM-41857
id: build
with:
image: ${{ github.repository }}
diff --git a/Dockerfile b/Dockerfile
index 236c152b1..b2bbd1ff8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,20 +3,27 @@
# Base container
FROM mambaorg/micromamba:1.5.9
-# Add rubin-sim from conda-forge
+# Copy current directory
+COPY --chown=$MAMBA_USER:$MAMBA_USER . /home/${MAMBA_USER}/rubin_sim
-RUN micromamba install -y -n base -c conda-forge rubin-sim
-RUN micromamba install -y -n base -c conda-forge \
- jinja2 \
- tornado
+# Install container requirements from conda-forge
+# Note that open-orb dependencies are omitted
+RUN micromamba install -y -n base -f /home/${MAMBA_USER}/rubin_sim/container_environment.yaml
RUN micromamba clean --all --yes
+
ARG MAMBA_DOCKERFILE_ACTIVATE=1
+# Install current version of rubin-sim
+RUN python -m pip install /home/$MAMBA_USER/rubin_sim --no-deps
+
# Container execution
# Mount fbs simulation outputs expected at /data/fbs_sims
+# Mount rubin_sim_data (if needed) at /data/rubin_sim_data
+
+EXPOSE 80
+ENV PORT=80
-EXPOSE 8080
-ENV PORT 8080
+ENV RUBIN_SIM_DATA_DIR=/data/rubin_sim_data
-# Start up show_maf on port 8080
+# Start up show_maf on port 80
CMD cd /data/fbs_sims && show_maf -p 80 --no_browser
diff --git a/container_environment.yaml b/container_environment.yaml
new file mode 100644
index 000000000..5b981d1a3
--- /dev/null
+++ b/container_environment.yaml
@@ -0,0 +1,28 @@
+name: base
+channels:
+ - conda-forge
+dependencies:
+ - numpy <2
+ - matplotlib-base
+ - healpy
+ - pandas
+ - pyarrow
+ - numexpr
+ - scipy
+ - sqlalchemy
+ - astropy
+ - pytables
+ - h5py
+ - astroplan
+ - git
+ - colorcet
+ - cycler
+ - george
+ - scikit-learn
+ - shapely
+ - skyproj
+ - tqdm
+ - jinja2
+ - tornado
+ - rubin-scheduler
+
diff --git a/pyproject.toml b/pyproject.toml
index b1a20f2c9..cc0ce860f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,6 +39,7 @@ dependencies = [
"skyfield",
"skyproj",
"pyoorb",
+ "tqdm",
"rubin-scheduler",
"pyarrow",
]
@@ -70,6 +71,7 @@ run_moving_fractions = "rubin_sim.maf.run_moving_fractions:run_moving_fractions"
run_moving_join = "rubin_sim.maf.run_moving_join:run_moving_join"
scimaf_dir = "rubin_sim.maf.scimaf_dir:scimaf_dir"
run_selfcal_metric = "rubin_sim.maf.run_selfcal_metric:run_selfcal_metric"
+make_fbs_tracking_db = "rubin_sim.maf.make_fbs_tracking_db:make_fbs_tracking_db"
show_maf = "rubin_sim.maf.show_maf:show_maf"
make_lsst_obs = "rubin_sim.moving_objects.make_lsst_obs:make_lsst_obs"
diff --git a/requirements.txt b/requirements.txt
index 3fcc1b1b4..4a9cbe02c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
setuptools_scm
setuptools_scm_git_archive
-numpy<2
+numpy
matplotlib
healpy
pandas
@@ -21,5 +21,5 @@ scikit-learn
shapely
skyfield
skyproj
-tdqm
+tqdm
rubin-scheduler
diff --git a/rubin_sim/maf/db/tracking_db.py b/rubin_sim/maf/db/tracking_db.py
index c4f27dd59..e8a13ee18 100644
--- a/rubin_sim/maf/db/tracking_db.py
+++ b/rubin_sim/maf/db/tracking_db.py
@@ -70,9 +70,11 @@ def __init__(self, database=None, trackingDbverbose=False):
# connecting to non-existent database creates it automatically
if database is None:
# Default is a file in the current directory.
- self.database = os.path.join(os.getcwd(), "trackingDb_sqlite.db")
+ self.database = "trackingDb_sqlite.db"
+ self.tracking_db_dir = "."
else:
self.database = database
+ self.tracking_db_dir = os.path.dirname(database)
# only sqlite
dbAddress = url.URL.create(drivername=self.driver, database=self.database)
engine = create_engine(dbAddress, echo=self.verbose)
@@ -127,6 +129,7 @@ def add_run(
Set the date the MAF analysis was run.
maf_dir : `str`, optional
The relative path to the MAF directory.
+ Will be converted to a relative path if absolute.
db_file : `str`, optional
The relative path to the Opsim SQLite database file.
maf_run_id : `int`, optional
@@ -158,6 +161,8 @@ def add_run(
maf_date = "NULL"
if maf_dir is None:
maf_dir = "NULL"
+ if maf_dir is not None:
+ maf_dir = os.path.relpath(maf_dir, start=self.tracking_db_dir)
if db_file is None:
db_file = "NULL"
# Test if maf_dir already exists in database.
@@ -275,12 +280,12 @@ def add_run_to_database(
db_file : `str`, optional
Relative path + name of the opsim database file.
"""
- maf_dir = os.path.abspath(maf_dir)
+ trackingDb = TrackingDb(database=tracking_db_file)
+
+ maf_dir = os.path.relpath(maf_dir, start=os.path.dirname(trackingDb.tracking_db_dir))
if not os.path.isdir(maf_dir):
raise ValueError("There is no directory containing MAF outputs at %s." % (maf_dir))
- trackingDb = TrackingDb(database=tracking_db_file)
-
# Connect to resultsDb for additional information if available
if os.path.isfile(os.path.join(maf_dir, "resultsDb_sqlite.db")) and not skip_extras:
resdb = ResultsDb(maf_dir)
diff --git a/rubin_sim/maf/make_fbs_tracking_db.py b/rubin_sim/maf/make_fbs_tracking_db.py
new file mode 100644
index 000000000..13b3d8055
--- /dev/null
+++ b/rubin_sim/maf/make_fbs_tracking_db.py
@@ -0,0 +1,90 @@
+__all__ = ("make_fbs_tracking_db",)
+
+import argparse
+import os
+import sqlite3
+
+import pandas as pd
+
+from rubin_sim.maf.db import ResultsDb, TrackingDb, VersionRow
+
+
+def make_fbs_tracking_db():
+ """Make a database to track lots of opsim outputs,
+ adding in comments and identifiers based on the metric subsets.
+ """
+ parser = argparse.ArgumentParser(
+ description="Create a tracking database for many opsim maf outputs,"
+ "with comments and dates. "
+ "Assumes metrics are in a subdirectory called 'maf'."
+ )
+ parser.add_argument(
+ "--opsim_list",
+ type=str,
+ default="opsim_list",
+ help="File containing a list of all of the opsim runs to add.",
+ )
+ args = parser.parse_args()
+
+ batches = {
+ "meta": "General Info Metrics",
+ "glance": "Quick Look Metrics",
+ "sci": "Science Metrics",
+ "ss": "Solar System Metrics",
+ "ddf": "Deep Drilling Metrics",
+ }
+
+ tracking_db = TrackingDb()
+ print(f"Tracking database in directory {tracking_db.tracking_db_dir}")
+
+ with open(args.opsim_list, "r") as runlist:
+ for run in runlist:
+ db_file = run.replace("\n", "")
+ vals = db_file.split("/")
+ family = vals[-2]
+ db_name = vals[-1]
+ run_name = db_name.replace(".db", "")
+ run_version = run_name.split("_10yrs")[0][-4:]
+ run_group = run_version + family
+
+ # Try to build a comment on the run based on the run_name
+ run_comment = run_name.replace("_10yrs", "")[0:-4]
+ run_comment = run_comment.replace("_", "")
+
+ print(run_name, db_file)
+ conn = sqlite3.connect(db_file)
+ query = "select Value from info where Parameter == 'Date, ymd'"
+ result = pd.read_sql(query, conn)
+ sched_date = result.iloc[0, 0]
+ query = "select Value from info where Parameter like 'rubin_%.__version__'"
+ result = pd.read_sql(query, conn)
+ sched_version = result.iloc[0, 0]
+
+ # Look for metrics in any of the above sets
+ for k in batches:
+ maf_dir = os.path.join("maf", run_name + "_" + k)
+ maf_comment = batches[k]
+ # Get maf run date and version
+ if os.path.isfile(os.path.join(maf_dir, "resultsDb_sqlite.db")):
+ resdb = ResultsDb(maf_dir)
+ resdb.open()
+ query = resdb.session.query(VersionRow).all()
+ for v in query:
+ maf_version = v.version
+ maf_date = v.run_date
+ resdb.close()
+
+ maf_dir = os.path.relpath(maf_dir, start=os.path.dirname(tracking_db.tracking_db_dir))
+ runId = tracking_db.add_run(
+ run_group=run_group,
+ run_name=run_name,
+ run_comment=run_comment,
+ run_version=sched_version,
+ run_date=sched_date,
+ maf_comment=maf_comment,
+ maf_version=maf_version,
+ maf_date=maf_date,
+ maf_dir=maf_dir,
+ db_file=db_file,
+ )
+ print("Used MAF RunID %d" % (runId))
diff --git a/rubin_sim/maf/utils/sn_n_sn_utils.py b/rubin_sim/maf/utils/sn_n_sn_utils.py
index 18c5affb1..7d362fb0c 100644
--- a/rubin_sim/maf/utils/sn_n_sn_utils.py
+++ b/rubin_sim/maf/utils/sn_n_sn_utils.py
@@ -16,6 +16,7 @@
from astropy.table import Table
from rubin_scheduler.data import get_data_dir
from scipy.interpolate import RegularGridInterpolator
+
from rubin_sim.maf.utils import m52snr
warnings.filterwarnings("ignore", category=RuntimeWarning)
diff --git a/rubin_sim/maf/web/maf_tracking.py b/rubin_sim/maf/web/maf_tracking.py
index 24b12ae81..af5dd3b92 100644
--- a/rubin_sim/maf/web/maf_tracking.py
+++ b/rubin_sim/maf/web/maf_tracking.py
@@ -25,8 +25,9 @@ class MafTracking:
def __init__(self, database=None):
if database is None:
database = os.path.join(os.getcwd(), "trackingDb_sqlite.db")
+ self.tracking_db = database
- # Read in the results database.
+ # Read in the tracking database.
cols = [
"maf_run_id",
"run_name",
@@ -40,7 +41,7 @@ def __init__(self, database=None):
"maf_version",
"maf_date",
]
- self.runs = get_sim_data(database, "", cols, table_name="runs")
+ self.runs = get_sim_data(self.tracking_db, "", cols, table_name="runs")
self.runs = self.sort_runs(self.runs, order=["maf_run_id", "run_name", "maf_comment"])
self.runs_page = {}
@@ -59,16 +60,17 @@ def run_info(self, run):
Ordered dict version of the numpy structured array.
"""
runInfo = OrderedDict()
+ maf_dir = os.path.relpath(run["maf_dir"], start=os.path.dirname(self.tracking_db))
runInfo["Run Name"] = run["run_name"]
runInfo["Group"] = run["run_group"]
runInfo["Maf Comment"] = run["maf_comment"]
runInfo["Run Comment"] = run["run_comment"]
- runInfo["SQLite File"] = [
+ runInfo["RunDb File"] = [
os.path.relpath(run["db_file"]),
os.path.split(run["db_file"])[1],
]
- runInfo["ResultsDb"] = os.path.relpath(os.path.join(run["maf_dir"], "resultsDb_sqlite.db"))
- runInfo["maf_dir"] = run["maf_dir"]
+ runInfo["ResultsDb"] = os.path.join(maf_dir, "resultsDb_sqlite.db")
+ runInfo["maf_dir"] = maf_dir
runInfo["sched_version"] = run["run_version"]
runInfo["sched_date"] = run["run_date"]
runInfo["maf_version"] = run["maf_version"]
diff --git a/rubin_sim/maf/web/templates/runselect.html b/rubin_sim/maf/web/templates/runselect.html
index 91db60ae5..6ae406ecd 100644
--- a/rubin_sim/maf/web/templates/runselect.html
+++ b/rubin_sim/maf/web/templates/runselect.html
@@ -41,7 +41,7 @@
{% for key in runInfo %}
{% if loop.index == 1 %}
| {{runInfo[key]|escape }} |
- {% elif key == 'SQLite File' %}
+ {% elif key == 'RunDb File' %}
{{runInfo[key][1]|escape}} |
{% elif key == 'ResultsDb' %}
ResultsDb |