From 84ff33e3733f8acaf6560e8afaa5faf6d1b29550 Mon Sep 17 00:00:00 2001 From: Lynne Jones Date: Thu, 12 Sep 2024 16:47:09 -0700 Subject: [PATCH 1/2] run on any ref --- .github/workflows/build_container.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build_container.yaml b/.github/workflows/build_container.yaml index d696a738..6c2f7cc2 100644 --- a/.github/workflows/build_container.yaml +++ b/.github/workflows/build_container.yaml @@ -9,10 +9,6 @@ jobs: build: runs-on: ubuntu-latest - # (optional) only build on tags or ticket branches - if: > - startsWith(github.ref, 'refs/tags/') - || startsWith(github.head_ref, 'tickets/') steps: - uses: actions/checkout@v4 From 0240f79e659a639b4490366a7483dd0fc48f9033 Mon Sep 17 00:00:00 2001 From: Lynne Jones Date: Thu, 12 Sep 2024 18:50:13 -0700 Subject: [PATCH 2/2] Docker build script, updates to maf trackingDb, container workflow --- .github/workflows/build_container.yaml | 2 +- Dockerfile | 23 ++++-- container_environment.yaml | 28 +++++++ pyproject.toml | 2 + requirements.txt | 4 +- rubin_sim/maf/db/tracking_db.py | 13 +++- rubin_sim/maf/make_fbs_tracking_db.py | 90 ++++++++++++++++++++++ rubin_sim/maf/utils/sn_n_sn_utils.py | 1 + rubin_sim/maf/web/maf_tracking.py | 12 +-- rubin_sim/maf/web/templates/runselect.html | 2 +- 10 files changed, 156 insertions(+), 21 deletions(-) create mode 100644 container_environment.yaml create mode 100644 rubin_sim/maf/make_fbs_tracking_db.py diff --git a/.github/workflows/build_container.yaml b/.github/workflows/build_container.yaml index 6c2f7cc2..44fa815c 100644 --- a/.github/workflows/build_container.yaml +++ b/.github/workflows/build_container.yaml @@ -13,7 +13,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: lsst-sqre/build-and-push-to-ghcr@v1 + - uses: lsst-sqre/build-and-push-to-ghcr@tickets/DM-41857 id: build with: image: ${{ github.repository }} diff --git a/Dockerfile b/Dockerfile index 236c152b..b2bbd1ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,20 +3,27 @@ # Base container FROM mambaorg/micromamba:1.5.9 -# Add rubin-sim from conda-forge +# Copy current directory +COPY --chown=$MAMBA_USER:$MAMBA_USER . /home/${MAMBA_USER}/rubin_sim -RUN micromamba install -y -n base -c conda-forge rubin-sim -RUN micromamba install -y -n base -c conda-forge \ - jinja2 \ - tornado +# Install container requirements from conda-forge +# Note that open-orb dependencies are omitted +RUN micromamba install -y -n base -f /home/${MAMBA_USER}/rubin_sim/container_environment.yaml RUN micromamba clean --all --yes + ARG MAMBA_DOCKERFILE_ACTIVATE=1 +# Install current version of rubin-sim +RUN python -m pip install /home/$MAMBA_USER/rubin_sim --no-deps + # Container execution # Mount fbs simulation outputs expected at /data/fbs_sims +# Mount rubin_sim_data (if needed) at /data/rubin_sim_data + +EXPOSE 80 +ENV PORT=80 -EXPOSE 8080 -ENV PORT 8080 +ENV RUBIN_SIM_DATA_DIR=/data/rubin_sim_data -# Start up show_maf on port 8080 +# Start up show_maf on port 80 CMD cd /data/fbs_sims && show_maf -p 80 --no_browser diff --git a/container_environment.yaml b/container_environment.yaml new file mode 100644 index 00000000..5b981d1a --- /dev/null +++ b/container_environment.yaml @@ -0,0 +1,28 @@ +name: base +channels: + - conda-forge +dependencies: + - numpy <2 + - matplotlib-base + - healpy + - pandas + - pyarrow + - numexpr + - scipy + - sqlalchemy + - astropy + - pytables + - h5py + - astroplan + - git + - colorcet + - cycler + - george + - scikit-learn + - shapely + - skyproj + - tqdm + - jinja2 + - tornado + - rubin-scheduler + diff --git a/pyproject.toml b/pyproject.toml index b1a20f2c..cc0ce860 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "skyfield", "skyproj", "pyoorb", + "tqdm", "rubin-scheduler", "pyarrow", ] @@ -70,6 +71,7 @@ run_moving_fractions = "rubin_sim.maf.run_moving_fractions:run_moving_fractions" run_moving_join = "rubin_sim.maf.run_moving_join:run_moving_join" scimaf_dir = "rubin_sim.maf.scimaf_dir:scimaf_dir" run_selfcal_metric = "rubin_sim.maf.run_selfcal_metric:run_selfcal_metric" +make_fbs_tracking_db = "rubin_sim.maf.make_fbs_tracking_db:make_fbs_tracking_db" show_maf = "rubin_sim.maf.show_maf:show_maf" make_lsst_obs = "rubin_sim.moving_objects.make_lsst_obs:make_lsst_obs" diff --git a/requirements.txt b/requirements.txt index 3fcc1b1b..4a9cbe02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ setuptools_scm setuptools_scm_git_archive -numpy<2 +numpy matplotlib healpy pandas @@ -21,5 +21,5 @@ scikit-learn shapely skyfield skyproj -tdqm +tqdm rubin-scheduler diff --git a/rubin_sim/maf/db/tracking_db.py b/rubin_sim/maf/db/tracking_db.py index c4f27dd5..e8a13ee1 100644 --- a/rubin_sim/maf/db/tracking_db.py +++ b/rubin_sim/maf/db/tracking_db.py @@ -70,9 +70,11 @@ def __init__(self, database=None, trackingDbverbose=False): # connecting to non-existent database creates it automatically if database is None: # Default is a file in the current directory. - self.database = os.path.join(os.getcwd(), "trackingDb_sqlite.db") + self.database = "trackingDb_sqlite.db" + self.tracking_db_dir = "." else: self.database = database + self.tracking_db_dir = os.path.dirname(database) # only sqlite dbAddress = url.URL.create(drivername=self.driver, database=self.database) engine = create_engine(dbAddress, echo=self.verbose) @@ -127,6 +129,7 @@ def add_run( Set the date the MAF analysis was run. maf_dir : `str`, optional The relative path to the MAF directory. + Will be converted to a relative path if absolute. db_file : `str`, optional The relative path to the Opsim SQLite database file. maf_run_id : `int`, optional @@ -158,6 +161,8 @@ def add_run( maf_date = "NULL" if maf_dir is None: maf_dir = "NULL" + if maf_dir is not None: + maf_dir = os.path.relpath(maf_dir, start=self.tracking_db_dir) if db_file is None: db_file = "NULL" # Test if maf_dir already exists in database. @@ -275,12 +280,12 @@ def add_run_to_database( db_file : `str`, optional Relative path + name of the opsim database file. """ - maf_dir = os.path.abspath(maf_dir) + trackingDb = TrackingDb(database=tracking_db_file) + + maf_dir = os.path.relpath(maf_dir, start=os.path.dirname(trackingDb.tracking_db_dir)) if not os.path.isdir(maf_dir): raise ValueError("There is no directory containing MAF outputs at %s." % (maf_dir)) - trackingDb = TrackingDb(database=tracking_db_file) - # Connect to resultsDb for additional information if available if os.path.isfile(os.path.join(maf_dir, "resultsDb_sqlite.db")) and not skip_extras: resdb = ResultsDb(maf_dir) diff --git a/rubin_sim/maf/make_fbs_tracking_db.py b/rubin_sim/maf/make_fbs_tracking_db.py new file mode 100644 index 00000000..13b3d805 --- /dev/null +++ b/rubin_sim/maf/make_fbs_tracking_db.py @@ -0,0 +1,90 @@ +__all__ = ("make_fbs_tracking_db",) + +import argparse +import os +import sqlite3 + +import pandas as pd + +from rubin_sim.maf.db import ResultsDb, TrackingDb, VersionRow + + +def make_fbs_tracking_db(): + """Make a database to track lots of opsim outputs, + adding in comments and identifiers based on the metric subsets. + """ + parser = argparse.ArgumentParser( + description="Create a tracking database for many opsim maf outputs," + "with comments and dates. " + "Assumes metrics are in a subdirectory called 'maf'." + ) + parser.add_argument( + "--opsim_list", + type=str, + default="opsim_list", + help="File containing a list of all of the opsim runs to add.", + ) + args = parser.parse_args() + + batches = { + "meta": "General Info Metrics", + "glance": "Quick Look Metrics", + "sci": "Science Metrics", + "ss": "Solar System Metrics", + "ddf": "Deep Drilling Metrics", + } + + tracking_db = TrackingDb() + print(f"Tracking database in directory {tracking_db.tracking_db_dir}") + + with open(args.opsim_list, "r") as runlist: + for run in runlist: + db_file = run.replace("\n", "") + vals = db_file.split("/") + family = vals[-2] + db_name = vals[-1] + run_name = db_name.replace(".db", "") + run_version = run_name.split("_10yrs")[0][-4:] + run_group = run_version + family + + # Try to build a comment on the run based on the run_name + run_comment = run_name.replace("_10yrs", "")[0:-4] + run_comment = run_comment.replace("_", "") + + print(run_name, db_file) + conn = sqlite3.connect(db_file) + query = "select Value from info where Parameter == 'Date, ymd'" + result = pd.read_sql(query, conn) + sched_date = result.iloc[0, 0] + query = "select Value from info where Parameter like 'rubin_%.__version__'" + result = pd.read_sql(query, conn) + sched_version = result.iloc[0, 0] + + # Look for metrics in any of the above sets + for k in batches: + maf_dir = os.path.join("maf", run_name + "_" + k) + maf_comment = batches[k] + # Get maf run date and version + if os.path.isfile(os.path.join(maf_dir, "resultsDb_sqlite.db")): + resdb = ResultsDb(maf_dir) + resdb.open() + query = resdb.session.query(VersionRow).all() + for v in query: + maf_version = v.version + maf_date = v.run_date + resdb.close() + + maf_dir = os.path.relpath(maf_dir, start=os.path.dirname(tracking_db.tracking_db_dir)) + runId = tracking_db.add_run( + run_group=run_group, + run_name=run_name, + run_comment=run_comment, + run_version=sched_version, + run_date=sched_date, + maf_comment=maf_comment, + maf_version=maf_version, + maf_date=maf_date, + maf_dir=maf_dir, + db_file=db_file, + ) + print("Used MAF RunID %d" % (runId)) diff --git a/rubin_sim/maf/utils/sn_n_sn_utils.py b/rubin_sim/maf/utils/sn_n_sn_utils.py index 18c5affb..7d362fb0 100644 --- a/rubin_sim/maf/utils/sn_n_sn_utils.py +++ b/rubin_sim/maf/utils/sn_n_sn_utils.py @@ -16,6 +16,7 @@ from astropy.table import Table from rubin_scheduler.data import get_data_dir from scipy.interpolate import RegularGridInterpolator + from rubin_sim.maf.utils import m52snr warnings.filterwarnings("ignore", category=RuntimeWarning) diff --git a/rubin_sim/maf/web/maf_tracking.py b/rubin_sim/maf/web/maf_tracking.py index 24b12ae8..af5dd3b9 100644 --- a/rubin_sim/maf/web/maf_tracking.py +++ b/rubin_sim/maf/web/maf_tracking.py @@ -25,8 +25,9 @@ class MafTracking: def __init__(self, database=None): if database is None: database = os.path.join(os.getcwd(), "trackingDb_sqlite.db") + self.tracking_db = database - # Read in the results database. + # Read in the tracking database. cols = [ "maf_run_id", "run_name", @@ -40,7 +41,7 @@ def __init__(self, database=None): "maf_version", "maf_date", ] - self.runs = get_sim_data(database, "", cols, table_name="runs") + self.runs = get_sim_data(self.tracking_db, "", cols, table_name="runs") self.runs = self.sort_runs(self.runs, order=["maf_run_id", "run_name", "maf_comment"]) self.runs_page = {} @@ -59,16 +60,17 @@ def run_info(self, run): Ordered dict version of the numpy structured array. """ runInfo = OrderedDict() + maf_dir = os.path.relpath(run["maf_dir"], start=os.path.dirname(self.tracking_db)) runInfo["Run Name"] = run["run_name"] runInfo["Group"] = run["run_group"] runInfo["Maf Comment"] = run["maf_comment"] runInfo["Run Comment"] = run["run_comment"] - runInfo["SQLite File"] = [ + runInfo["RunDb File"] = [ os.path.relpath(run["db_file"]), os.path.split(run["db_file"])[1], ] - runInfo["ResultsDb"] = os.path.relpath(os.path.join(run["maf_dir"], "resultsDb_sqlite.db")) - runInfo["maf_dir"] = run["maf_dir"] + runInfo["ResultsDb"] = os.path.join(maf_dir, "resultsDb_sqlite.db") + runInfo["maf_dir"] = maf_dir runInfo["sched_version"] = run["run_version"] runInfo["sched_date"] = run["run_date"] runInfo["maf_version"] = run["maf_version"] diff --git a/rubin_sim/maf/web/templates/runselect.html b/rubin_sim/maf/web/templates/runselect.html index 91db60ae..6ae406ec 100644 --- a/rubin_sim/maf/web/templates/runselect.html +++ b/rubin_sim/maf/web/templates/runselect.html @@ -41,7 +41,7 @@ {% for key in runInfo %} {% if loop.index == 1 %} {{runInfo[key]|escape }} - {% elif key == 'SQLite File' %} + {% elif key == 'RunDb File' %} {{runInfo[key][1]|escape}} {% elif key == 'ResultsDb' %} ResultsDb