Skip to content

Commit

Permalink
Docker build script, updates to maf trackingDb, container workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
rhiannonlynne committed Sep 13, 2024
1 parent 84ff33e commit 0240f79
Show file tree
Hide file tree
Showing 10 changed files with 156 additions and 21 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build_container.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
steps:
- uses: actions/checkout@v4

- uses: lsst-sqre/build-and-push-to-ghcr@v1
- uses: lsst-sqre/build-and-push-to-ghcr@tickets/DM-41857
id: build
with:
image: ${{ github.repository }}
Expand Down
23 changes: 15 additions & 8 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,27 @@
# Base container
FROM mambaorg/micromamba:1.5.9

# Add rubin-sim from conda-forge
# Copy current directory
COPY --chown=$MAMBA_USER:$MAMBA_USER . /home/${MAMBA_USER}/rubin_sim

RUN micromamba install -y -n base -c conda-forge rubin-sim
RUN micromamba install -y -n base -c conda-forge \
jinja2 \
tornado
# Install container requirements from conda-forge
# Note that open-orb dependencies are omitted
RUN micromamba install -y -n base -f /home/${MAMBA_USER}/rubin_sim/container_environment.yaml
RUN micromamba clean --all --yes

ARG MAMBA_DOCKERFILE_ACTIVATE=1

# Install current version of rubin-sim
RUN python -m pip install /home/$MAMBA_USER/rubin_sim --no-deps

# Container execution
# Mount fbs simulation outputs expected at /data/fbs_sims
# Mount rubin_sim_data (if needed) at /data/rubin_sim_data

EXPOSE 80
ENV PORT=80

EXPOSE 8080
ENV PORT 8080
ENV RUBIN_SIM_DATA_DIR=/data/rubin_sim_data

# Start up show_maf on port 8080
# Start up show_maf on port 80
CMD cd /data/fbs_sims && show_maf -p 80 --no_browser
28 changes: 28 additions & 0 deletions container_environment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: base
channels:
- conda-forge
dependencies:
- numpy <2
- matplotlib-base
- healpy
- pandas
- pyarrow
- numexpr
- scipy
- sqlalchemy
- astropy
- pytables
- h5py
- astroplan
- git
- colorcet
- cycler
- george
- scikit-learn
- shapely
- skyproj
- tqdm
- jinja2
- tornado
- rubin-scheduler

2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ dependencies = [
"skyfield",
"skyproj",
"pyoorb",
"tqdm",
"rubin-scheduler",
"pyarrow",
]
Expand Down Expand Up @@ -70,6 +71,7 @@ run_moving_fractions = "rubin_sim.maf.run_moving_fractions:run_moving_fractions"
run_moving_join = "rubin_sim.maf.run_moving_join:run_moving_join"
scimaf_dir = "rubin_sim.maf.scimaf_dir:scimaf_dir"
run_selfcal_metric = "rubin_sim.maf.run_selfcal_metric:run_selfcal_metric"
make_fbs_tracking_db = "rubin_sim.maf.make_fbs_tracking_db:make_fbs_tracking_db"
show_maf = "rubin_sim.maf.show_maf:show_maf"
make_lsst_obs = "rubin_sim.moving_objects.make_lsst_obs:make_lsst_obs"

Expand Down
4 changes: 2 additions & 2 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
setuptools_scm
setuptools_scm_git_archive
numpy<2
numpy
matplotlib
healpy
pandas
Expand All @@ -21,5 +21,5 @@ scikit-learn
shapely
skyfield
skyproj
tdqm
tqdm
rubin-scheduler
13 changes: 9 additions & 4 deletions rubin_sim/maf/db/tracking_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,11 @@ def __init__(self, database=None, trackingDbverbose=False):
# connecting to non-existent database creates it automatically
if database is None:
# Default is a file in the current directory.
self.database = os.path.join(os.getcwd(), "trackingDb_sqlite.db")
self.database = "trackingDb_sqlite.db"
self.tracking_db_dir = "."
else:
self.database = database
self.tracking_db_dir = os.path.dirname(database)
# only sqlite
dbAddress = url.URL.create(drivername=self.driver, database=self.database)
engine = create_engine(dbAddress, echo=self.verbose)
Expand Down Expand Up @@ -127,6 +129,7 @@ def add_run(
Set the date the MAF analysis was run.
maf_dir : `str`, optional
The relative path to the MAF directory.
Will be converted to a relative path if absolute.
db_file : `str`, optional
The relative path to the Opsim SQLite database file.
maf_run_id : `int`, optional
Expand Down Expand Up @@ -158,6 +161,8 @@ def add_run(
maf_date = "NULL"
if maf_dir is None:
maf_dir = "NULL"
if maf_dir is not None:
maf_dir = os.path.relpath(maf_dir, start=self.tracking_db_dir)
if db_file is None:
db_file = "NULL"
# Test if maf_dir already exists in database.
Expand Down Expand Up @@ -275,12 +280,12 @@ def add_run_to_database(
db_file : `str`, optional
Relative path + name of the opsim database file.
"""
maf_dir = os.path.abspath(maf_dir)
trackingDb = TrackingDb(database=tracking_db_file)

maf_dir = os.path.relpath(maf_dir, start=os.path.dirname(trackingDb.tracking_db_dir))
if not os.path.isdir(maf_dir):
raise ValueError("There is no directory containing MAF outputs at %s." % (maf_dir))

trackingDb = TrackingDb(database=tracking_db_file)

# Connect to resultsDb for additional information if available
if os.path.isfile(os.path.join(maf_dir, "resultsDb_sqlite.db")) and not skip_extras:
resdb = ResultsDb(maf_dir)
Expand Down
90 changes: 90 additions & 0 deletions rubin_sim/maf/make_fbs_tracking_db.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
__all__ = ("make_fbs_tracking_db",)

import argparse
import os
import sqlite3

import pandas as pd

from rubin_sim.maf.db import ResultsDb, TrackingDb, VersionRow


def make_fbs_tracking_db():
"""Make a database to track lots of opsim outputs,
adding in comments and identifiers based on the metric subsets.
"""
parser = argparse.ArgumentParser(
description="Create a tracking database for many opsim maf outputs,"
"with comments and dates. "
"Assumes metrics are in a subdirectory called 'maf'."
)
parser.add_argument(
"--opsim_list",
type=str,
default="opsim_list",
help="File containing a list of all of the opsim runs to add.",
)
args = parser.parse_args()

batches = {
"meta": "General Info Metrics",
"glance": "Quick Look Metrics",
"sci": "Science Metrics",
"ss": "Solar System Metrics",
"ddf": "Deep Drilling Metrics",
}

tracking_db = TrackingDb()
print(f"Tracking database in directory {tracking_db.tracking_db_dir}")

with open(args.opsim_list, "r") as runlist:
for run in runlist:
db_file = run.replace("\n", "")
vals = db_file.split("/")
family = vals[-2]
db_name = vals[-1]
run_name = db_name.replace(".db", "")
run_version = run_name.split("_10yrs")[0][-4:]
run_group = run_version + family

# Try to build a comment on the run based on the run_name
run_comment = run_name.replace("_10yrs", "")[0:-4]
run_comment = run_comment.replace("_", "")

print(run_name, db_file)
conn = sqlite3.connect(db_file)
query = "select Value from info where Parameter == 'Date, ymd'"
result = pd.read_sql(query, conn)
sched_date = result.iloc[0, 0]
query = "select Value from info where Parameter like 'rubin_%.__version__'"
result = pd.read_sql(query, conn)
sched_version = result.iloc[0, 0]

# Look for metrics in any of the above sets
for k in batches:
maf_dir = os.path.join("maf", run_name + "_" + k)
maf_comment = batches[k]
# Get maf run date and version
if os.path.isfile(os.path.join(maf_dir, "resultsDb_sqlite.db")):
resdb = ResultsDb(maf_dir)
resdb.open()
query = resdb.session.query(VersionRow).all()
for v in query:
maf_version = v.version
maf_date = v.run_date
resdb.close()

maf_dir = os.path.relpath(maf_dir, start=os.path.dirname(tracking_db.tracking_db_dir))
runId = tracking_db.add_run(
run_group=run_group,
run_name=run_name,
run_comment=run_comment,
run_version=sched_version,
run_date=sched_date,
maf_comment=maf_comment,
maf_version=maf_version,
maf_date=maf_date,
maf_dir=maf_dir,
db_file=db_file,
)
print("Used MAF RunID %d" % (runId))
1 change: 1 addition & 0 deletions rubin_sim/maf/utils/sn_n_sn_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from astropy.table import Table
from rubin_scheduler.data import get_data_dir
from scipy.interpolate import RegularGridInterpolator

from rubin_sim.maf.utils import m52snr

warnings.filterwarnings("ignore", category=RuntimeWarning)
Expand Down
12 changes: 7 additions & 5 deletions rubin_sim/maf/web/maf_tracking.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ class MafTracking:
def __init__(self, database=None):
if database is None:
database = os.path.join(os.getcwd(), "trackingDb_sqlite.db")
self.tracking_db = database

# Read in the results database.
# Read in the tracking database.
cols = [
"maf_run_id",
"run_name",
Expand All @@ -40,7 +41,7 @@ def __init__(self, database=None):
"maf_version",
"maf_date",
]
self.runs = get_sim_data(database, "", cols, table_name="runs")
self.runs = get_sim_data(self.tracking_db, "", cols, table_name="runs")
self.runs = self.sort_runs(self.runs, order=["maf_run_id", "run_name", "maf_comment"])
self.runs_page = {}

Expand All @@ -59,16 +60,17 @@ def run_info(self, run):
Ordered dict version of the numpy structured array.
"""
runInfo = OrderedDict()
maf_dir = os.path.relpath(run["maf_dir"], start=os.path.dirname(self.tracking_db))
runInfo["Run Name"] = run["run_name"]
runInfo["Group"] = run["run_group"]
runInfo["Maf Comment"] = run["maf_comment"]
runInfo["Run Comment"] = run["run_comment"]
runInfo["SQLite File"] = [
runInfo["RunDb File"] = [
os.path.relpath(run["db_file"]),
os.path.split(run["db_file"])[1],
]
runInfo["ResultsDb"] = os.path.relpath(os.path.join(run["maf_dir"], "resultsDb_sqlite.db"))
runInfo["maf_dir"] = run["maf_dir"]
runInfo["ResultsDb"] = os.path.join(maf_dir, "resultsDb_sqlite.db")
runInfo["maf_dir"] = maf_dir
runInfo["sched_version"] = run["run_version"]
runInfo["sched_date"] = run["run_date"]
runInfo["maf_version"] = run["maf_version"]
Expand Down
2 changes: 1 addition & 1 deletion rubin_sim/maf/web/templates/runselect.html
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
{% for key in runInfo %}
{% if loop.index == 1 %}
<td><a href="allMetricResults?runId={{run['maf_run_id']}}">{{runInfo[key]|escape }} </a> </td>
{% elif key == 'SQLite File' %}
{% elif key == 'RunDb File' %}
<td><a href="{{runInfo[key][0]}}" download>{{runInfo[key][1]|escape}}</a> </td>
{% elif key == 'ResultsDb' %}
<td><a href="{{runInfo[key]}}" download>ResultsDb</a> </td>
Expand Down

0 comments on commit 0240f79

Please sign in to comment.