From d31e22c3f35c05a3d5428f550e357bd295ab3922 Mon Sep 17 00:00:00 2001 From: Steven Murray Date: Thu, 4 Apr 2024 12:47:38 +0200 Subject: [PATCH] test: cover the lstbin makeflow better --- hera_opm/data/sample_config/lstbin.toml | 30 -- .../data/sample_config/lstbin_options.toml | 30 -- .../data/sample_config/lstbin_simple.toml | 33 -- hera_opm/data/sample_config/lstbin_v2.toml | 32 -- hera_opm/mf_tools.py | 310 +----------------- hera_opm/tests/test_lstbin_makeflow.py | 271 +++++++++++++++ hera_opm/tests/test_mf_tools.py | 201 ------------ setup.cfg | 2 +- 8 files changed, 280 insertions(+), 629 deletions(-) delete mode 100644 hera_opm/data/sample_config/lstbin.toml delete mode 100644 hera_opm/data/sample_config/lstbin_options.toml delete mode 100644 hera_opm/data/sample_config/lstbin_simple.toml delete mode 100644 hera_opm/data/sample_config/lstbin_v2.toml create mode 100644 hera_opm/tests/test_lstbin_makeflow.py diff --git a/hera_opm/data/sample_config/lstbin.toml b/hera_opm/data/sample_config/lstbin.toml deleted file mode 100644 index 317fa13b..00000000 --- a/hera_opm/data/sample_config/lstbin.toml +++ /dev/null @@ -1,30 +0,0 @@ -[Options] -makeflow_type = "lstbin" -path_to_do_scripts = "~/hera/hera_opm/hera_opm/data/sample_task_scripts" -source_script = "~/.bashrc" -conda_env = "hera" -base_mem = 10000 -base_cpu = 1 - -[LSTBIN_OPTS] -sig_clip = true -sigma = 5 -min_N = 5 -rephase = false -ntimes_per_file = 60 -dlst = "None" -lst_start = 0.0 -fixed_lst_start = false -vis_units = "Jy" -parallelize = true -file_ext = "grp1.of2.{}.{}.{:7.5f}.uvh5" -outdir = "../data" -parent_dir = "../data" -data_files = ["'zen.2458043.*.HH.uvh5'", - "'zen.2458044.*.HH.uvh5'", - "'zen.2458045.*.HH.uvh5'"] - -[LSTBIN] -args = ["sig_clip", "sigma", "min_N", "rephase", "ntimes_per_file", "lst_start", - "fixed_lst_start", "dlst", "vis_units", "output_file_select", "file_ext", - "outdir"] diff --git a/hera_opm/data/sample_config/lstbin_options.toml b/hera_opm/data/sample_config/lstbin_options.toml deleted file mode 100644 index 2f50e916..00000000 --- a/hera_opm/data/sample_config/lstbin_options.toml +++ /dev/null @@ -1,30 +0,0 @@ -[Options] -makeflow_type = "lstbin" -path_to_do_scripts = "~/hera/hera_opm/hera_opm/data/sample_task_scripts" -conda_env = "hera" -base_mem = 10000 -base_cpu = 1 -timeout = "24h" - -[LSTBIN_OPTS] -sig_clip = true -sigma = 5 -min_N = 5 -rephase = false -ntimes_per_file = 60 -dlst = "None" -lst_start = 0.0 -fixed_lst_start = false -vis_units = "Jy" -parallelize = true -file_ext = "grp1.of2.{}.{}.{:7.5f}.uvh5" -outdir = "./" -parent_dir = "./" -data_files = ["'zen.2458043.*.HH.uvh5'", - "'zen.2458044.*.HH.uvh5'", - "'zen.2458045.*.HH.uvh5'"] - -[LSTBIN] -args = ["sig_clip", "sigma", "min_N", "rephase", "ntimes_per_file", "lst_start", - "fixed_lst_start", "dlst", "vis_units", "output_file_select", "file_ext", - "outdir"] diff --git a/hera_opm/data/sample_config/lstbin_simple.toml b/hera_opm/data/sample_config/lstbin_simple.toml deleted file mode 100644 index dfe5526c..00000000 --- a/hera_opm/data/sample_config/lstbin_simple.toml +++ /dev/null @@ -1,33 +0,0 @@ -[Options] -makeflow_type = "lstbin" -path_to_do_scripts = "~/hera/hera_opm/hera_opm/data/sample_task_scripts" -source_script = "~/.bashrc" -conda_env = "hera" -base_mem = 10000 -base_cpu = 1 - -[LSTBIN_OPTS] -sig_clip = true -sigma = 5 -min_N = 5 -rephase = false -ntimes_per_file = 60 -dlst = "None" -lst_start = 0.0 -fixed_lst_start = false -vis_units = "Jy" -parallelize = true -file_ext = "grp1.of2.{}.{}.{:7.5f}.uvh5" -outdir = "../data" -parent_dir = "../data" - -datadir = "placeholder" -nightdirs = ["2458043", "2458044", "2458045"] -sd="HH" -label="" -extension="uvh5" - -[LSTBIN] -args = ["sig_clip", "sigma", "min_N", "rephase", "ntimes_per_file", "lst_start", - "fixed_lst_start", "dlst", "vis_units", "output_file_select", "file_ext", - "outdir"] diff --git a/hera_opm/data/sample_config/lstbin_v2.toml b/hera_opm/data/sample_config/lstbin_v2.toml deleted file mode 100644 index c73692e0..00000000 --- a/hera_opm/data/sample_config/lstbin_v2.toml +++ /dev/null @@ -1,32 +0,0 @@ -# IDR2 v2 examle config -[Options] -makeflow_type = "lstbin" -path_to_do_scripts = "~/hera/hera_opm/hera_opm/data/sample_task_scripts" -source_script = "~/.bashrc" -conda_env = "hera" -base_mem = 10000 -base_cpu = 1 - -[LSTBIN_OPTS] -sig_clip = true -Nbls_to_load = "None" -sigma = 5 -min_N = 5 -rephase = false -ntimes_per_file = 60 -dlst = "None" -lst_start = 0.0 -fixed_lst_start = false -vis_units = "Jy" -parallelize = true -file_ext = "grp1.of2.{type}.{time:7.5f}.uvcA" -outdir = "../data" -parent_dir = "../data" -data_files = ["'zen.2458043.40141.HH.uvh5'", - "'zen.2458043.40887.HH.uvh5'"] -calibration = "" - -[LSTBIN] -args = ["sig_clip", "sigma", "min_N", "rephase", "ntimes_per_file", "lst_start", - "fixed_lst_start", "dlst", "vis_units", "output_file_select", "file_ext", - "outdir", "Nbls_to_load", "calibration"] diff --git a/hera_opm/mf_tools.py b/hera_opm/mf_tools.py index 5cb1775a..f37931ff 100644 --- a/hera_opm/mf_tools.py +++ b/hera_opm/mf_tools.py @@ -11,10 +11,8 @@ import shutil import subprocess import warnings -import glob import toml from pathlib import Path -import math def get_jd(filename): @@ -680,7 +678,7 @@ def build_makeflow_from_config( "lstbin" type, and call the appropriate funciton below. """ - if isinstance(config_file, str): + if isinstance(config_file, (str, Path)): # read in config file config = toml.load(config_file) else: @@ -695,15 +693,10 @@ def build_makeflow_from_config( build_lstbin_makeflow_from_config( config_file, mf_name=mf_name, work_dir=work_dir, **kwargs ) - elif makeflow_type == "lstbin-notebook": - build_lstbin_notebook_makeflow_from_config( - config_file, mf_name=mf_name, work_dir=work_dir, **kwargs - ) else: raise ValueError( - "unknown makeflow_type {} specified; must be 'analysis' or 'lstbin'".format( - makeflow_type - ) + f"unknown makeflow_type '{makeflow_type}' specified; " + "must be 'analysis' or 'lstbin'" ) return @@ -1343,104 +1336,14 @@ def build_analysis_makeflow_from_config( return -def get_lstbin_datafiles(config, parent_dir): - """Determine the datafiles for use in LST-binning makeflow.""" - # get data files - datafiles = get_config_entry(config, "LSTBIN_OPTS", "data_files", required=False) - - if datafiles is None: - # These are only required if datafiles wasn't specified specifically. - datadir = get_config_entry(config, "LSTBIN_OPTS", "datadir", required=True) - nightdirs = get_config_entry(config, "LSTBIN_OPTS", "nightdirs", required=True) - extension = get_config_entry(config, "LSTBIN_OPTS", "extension", required=True) - label = get_config_entry(config, "LSTBIN_OPTS", "label", required=True) - sd = get_config_entry(config, "LSTBIN_OPTS", "sd", required=True) - jdglob = get_config_entry( - config, "LSTBIN_OPTS", "jdglob", required=False, default="*" - ) - - if label: - label += "." - - datafiles = [] - for nd in nightdirs: - datafiles.append(f"{datadir}/{nd}/zen.{jdglob}.{sd}.{label}{extension}") - - # encapsulate in double quotes - return [ - "'{}'".format('"{}"'.format(os.path.join(parent_dir, df.strip('"').strip("'")))) - for df in datafiles - ] - - -def _legacy_make_lstbin_config_file(config, outdir: Path, datafiles): - try: - from hera_cal.lst_stack import make_lst_bin_config_file - except ImportError: - from hera_cal.lstbin_simple import make_lst_bin_config_file - - # Get dlst. Updated version supports leaving dlst unspecified or set as null. - # To support older versions which required string 'None', set that to None here. - dlst = get_config_entry(config, "LSTBIN_OPTS", "dlst", default=None, required=False) - if isinstance(dlst, str) and dlst.lower() in ("none", "null", ""): - warnings.warn( - "dlst should not be set to (string) 'None', but rather left unspecified in your TOML.", - DeprecationWarning, - ) - dlst = None - - lstbin_config_file = Path(outdir) / "file-config.yaml" - - clobber = get_config_entry(config, "LSTBIN_OPTS", "overwrite", default=False) - atol = get_config_entry(config, "LSTBIN_OPTS", "atol", default=1e-10) - lst_start = get_config_entry( - config, "LSTBIN_OPTS", "lst_start", default=None, required=False - ) - lst_width = get_config_entry( - config, "LSTBIN_OPTS", "lst_width", default=2 * math.pi - ) - ntimes_per_file = get_config_entry( - config, "LSTBIN_OPTS", "ntimes_per_file", default=60 - ) - blts_are_rectangular = get_config_entry( - config, "LSTBIN_OPTS", "blts_are_rectangular", default=None, required=False - ) - time_axis_faster_than_bls = get_config_entry( - config, - "LSTBIN_OPTS", - "time_axis_faster_than_bls", - default=None, - required=False, - ) - jd_regex = get_config_entry( - config, "LSTBIN_OPTS", "jd_regex", default=r"zen\.(\d+\.\d+)\." - ) - - file_config = make_lst_bin_config_file( - config_file=lstbin_config_file, - data_files=datafiles, - clobber=clobber, - dlst=dlst, - atol=atol, - lst_start=lst_start, - lst_width=lst_width, - ntimes_per_file=ntimes_per_file, - blts_are_rectangular=blts_are_rectangular, - time_axis_faster_than_bls=time_axis_faster_than_bls, - jd_regex=jd_regex, - ) - print(f"Created lstbin config file at {lstbin_config_file}.") - - return len(file_config["matched_files"]) - - def make_lstbin_config_file(config, outdir: str) -> int: # This must be a TOML file that specifies how to construct the LSTbin file-config lstconfig = config["FILE_CFG"] - from hera_cal.lst_stack.config import LSTBinConfiguration + from hera_cal.lst_stack.config import LSTBinConfigurator - lstconfig = LSTBinConfiguration.from_toml(toml.dumps(lstconfig)) + print(lstconfig) + lstconfig = LSTBinConfigurator.from_toml(toml.dumps(lstconfig)) print(f"Found {len(lstconfig.data_files)} nights of data.") print("Each night has the following number of files:") for flist in lstconfig.data_files: @@ -1460,191 +1363,10 @@ def make_lstbin_config_file(config, outdir: str) -> int: def build_lstbin_makeflow_from_config( - config_file, mf_name=None, work_dir=None, **kwargs -): - """Construct an LST-binning makeflow file from input data and a config_file. - - Parameters - ---------- - config_file : str - Full path to config file containing options. - mf_name : str - The name of makeflow file. Defaults to ".mf" if not - specified. - - Returns - ------- - None - - - Notes - ----- - The major difference between this function and the one above is the use of - the `config_lst_bin_files` function from hera_cal, which is used to - determine the number of output files, which are parallelized over in the - makeflow. - - """ - # import hera_cal - - # read in config file - config = toml.load(config_file) - cf = os.path.basename(config_file) - - # get LSTBIN arguments - lstbin_args = get_config_entry(config, "LSTBIN", "args", required=False) - - # set output_file_select to None - config["LSTBIN_OPTS"]["output_file_select"] = str("None") - - # get general options - path_to_do_scripts = Path(get_config_entry(config, "Options", "path_to_do_scripts")) - conda_env = get_config_entry(config, "Options", "conda_env", required=False) - source_script = get_config_entry(config, "Options", "source_script", required=False) - batch_system = get_config_entry(config, "Options", "batch_system", required=False) - timeout = _get_timeout(config) - - # open file for writing - if mf_name is not None: - fn = mf_name - else: - base, ext = os.path.splitext(cf) - fn = "{0}.mf".format(base) - - # determine whether or not to parallelize - parallelize = get_config_entry(config, "LSTBIN_OPTS", "parallelize", required=True) - if "parent_dir" in kwargs: - parent_dir = Path(kwargs["parent_dir"]) - else: - parent_dir = Path( - get_config_entry(config, "LSTBIN_OPTS", "parent_dir", required=True) - ) - - work_dir = Path(work_dir or parent_dir) - - makeflowfile = work_dir / fn - - # define command - command = path_to_do_scripts / "do_LSTBIN.sh" - - # write makeflow file - with open(makeflowfile, "w") as f: - # add comment at top of file listing date of creation and config file name - dt = time.strftime("%H:%M:%S on %d %B %Y") - print("# makeflow file generated from config file {}".format(cf), file=f) - print("# created at {}".format(dt), file=f) - - # add resource information - base_mem = get_config_entry(config, "Options", "base_mem", required=True) - base_cpu = get_config_entry(config, "Options", "base_cpu", required=False) - mail_user = get_config_entry(config, "Options", "mail_user", required=False) - default_queue = get_config_entry( - config, "Options", "default_queue", required=False - ) - if default_queue is None: - default_queue = "hera" - batch_options = process_batch_options( - base_mem, base_cpu, mail_user, default_queue, batch_system - ) - print("export BATCH_OPTIONS = {}".format(batch_options), file=f) - - if "outdir" in kwargs: - outdir = Path(kwargs["outdir"]) - else: - outdir = Path(get_config_entry(config, "LSTBIN_OPTS", "outdir")) - - datafiles = get_lstbin_datafiles(config, parent_dir) - - print("Searching for files in the following globs: ") - for df in datafiles: - print(" " + df.strip("'").strip('"')) - - # pre-process files to determine the number of output files - _datafiles = [sorted(glob.glob(df.strip("'").strip('"'))) for df in datafiles] - _datafiles = [df for df in _datafiles if len(df) > 0] - - nfiles = _legacy_make_lstbin_config_file(config, outdir, _datafiles) - - if not parallelize: - nfiles = 1 - - # loop over output files - for output_file_index in range(nfiles): - # if parallize, update output_file_select - if parallelize: - config["LSTBIN_OPTS"]["output_file_select"] = str(output_file_index) - - # make outfile list - outfile = Path(f"lstbin_outfile_{output_file_index}.LSTBIN.out") - - # get args list for lst-binning step - args = [ - str(get_config_entry(config, "LSTBIN_OPTS", a, required=True)) - for a in lstbin_args - ] - # turn into string - args = " ".join(args) - - # make logfile name - # logfile will capture stdout and stderr - logfile = work_dir / outfile.with_suffix(".log").name - - # make a small wrapper script that will run the actual command - # can't embed if; then statements in makeflow script - wrapper_script = work_dir / f"wrapper_{outfile.with_suffix('.sh').name}" - with open(wrapper_script, "w") as f2: - print("#!/bin/bash", file=f2) - if source_script is not None: - print("source {}".format(source_script), file=f2) - if conda_env is not None: - print("conda activate {}".format(conda_env), file=f2) - print("date", file=f2) - print("cd {}".format(parent_dir), file=f2) - if timeout is not None: - print( - "timeout {0} {1} {2}".format(timeout, command, args), - file=f2, - ) - else: - print("{0} {1}".format(command, args), file=f2) - print("if [ $? -eq 0 ]; then", file=f2) - print(" cd {}".format(work_dir), file=f2) - print(" touch {}".format(outfile), file=f2) - print("else", file=f2) - print( - " mv {0} {1}".format( - logfile, logfile.parent / f"{logfile.name}.error" - ), - file=f2, - ) - print("fi", file=f2) - print("date", file=f2) - # make file executable - os.chmod(wrapper_script, 0o755) - - # first line lists target file to make (dummy output file), and requirements - # second line is "build rule", which runs the shell script and makes the output file - line1 = "{0}: {1}".format(outfile, command) - line2 = "\t{0} > {1} 2>&1\n".format(wrapper_script, logfile) - print(line1, file=f) - print(line2, file=f) - - # Write the toml config to the output directory. - shutil.copy2(config_file, outdir / "lstbin-config.toml") - - # Also write the conda_env export to the LSTbin dir - if conda_env is not None: - os.system( - f"conda env export -n {conda_env} --file {outdir}/environment.yaml" - ) - - return - - -def build_lstbin_notebook_makeflow_from_config( config_file: str | Path, mf_name: str | None = None, work_dir: str | Path | None = None, + outdir: str | Path | None = None, ) -> None: """Construct a notebook-based LST-binning makeflow file from input data and a config_file. @@ -1672,7 +1394,7 @@ def build_lstbin_notebook_makeflow_from_config( makeflowfile = work_dir / mf_name - outdir = Path(get_config_entry(config, "LSTBIN_OPTS", "outdir")) + outdir = Path(outdir or get_config_entry(config, "LSTBIN_OPTS", "outdir")) # Write the toml config to the output directory. if not outdir.exists(): @@ -1699,16 +1421,6 @@ def build_lstbin_notebook_makeflow_from_config( config["LSTBIN_OPTS"]["lstavg_toml_file"] = str(lstavg_config.absolute()) config["LSTBIN_OPTS"]["kernel"] = conda_env - if "make_plots" not in config["LSTBIN_OPTS"]: - if "plot_every" in config["LSTBIN_OPTS"]: - plot_every = int( - get_config_entry(config, "LSTBIN_OPTS", "plot_every", required=False) - ) - else: - plot_every = 1 - else: - plot_every = int(bool(config["LSTBIN_OPTS"]["make_plots"])) - # determine whether or not to parallelize parallelize = get_config_entry(config, "LSTBIN_OPTS", "parallelize", required=True) @@ -1778,12 +1490,6 @@ def build_lstbin_notebook_makeflow_from_config( # if parallize, update output_file_select if parallelize: config["LSTBIN_OPTS"]["output_file_select"] = str(output_file_index) - if plot_every > 0: - config["LSTBIN_OPTS"]["make_plots"] = str( - output_file_index % plot_every == 0 - ) - else: - config["LSTBIN_OPTS"]["make_plots"] = "False" # make outfile list outfile = Path(f"{output_file_index:04}.LSTBIN.out") diff --git a/hera_opm/tests/test_lstbin_makeflow.py b/hera_opm/tests/test_lstbin_makeflow.py new file mode 100644 index 00000000..85e73350 --- /dev/null +++ b/hera_opm/tests/test_lstbin_makeflow.py @@ -0,0 +1,271 @@ +from pathlib import Path +import toml +from ..data import DATA_PATH +import pytest +from hera_opm import mf_tools as mt +import shutil + +hera_cal = pytest.importorskip("hera_cal") + + +def make_lstbin_config_file( + fl: Path, + datafiles: dict | list, + notebook: bool = False, + options=None, + lstbin_opts=None, + file_cfg=None, + lstavg_opts=None, +): + """Make a lstbin config file.""" + options = { + **{ + "makeflow_type": "lstbin", + "path_to_do_scripts": "/an/unused/path/for/tests", + "source_script": "~/.bashrc", + "conda_env": "hera", + "base_mem": 10000, + "base_cpu": 1, + }, + **(options or {}), + } + + lstbin_opts = { + **{ + "parallelize": True, + "outdir": str(fl.parent), + "parent_dir": str(fl.parent), + }, + **(lstbin_opts or {}), + } + + file_cfg = { + **{ + "nlsts_per_file": 60, + "lst_start": 0.0, + "datafiles": datafiles, + }, + **(file_cfg or {}), + } + + if isinstance(datafiles, list): + file_cfg["datadir"] = str(DATA_PATH) + + lstavg_opts = { + **{ + "outdir": "../data", + "bl_chunk_size": 5000, + "fname_format": "{inpaint_mode}/zen.{kind}.{lst:7.5f}.sum.uvh5", + "overwrite": True, + "write_med_mad": True, + "rephase": False, + }, + **(lstavg_opts or {}), + } + + if notebook: + action = "PER_OUTFILE_LSTSTACK_METRICS_NOTEBOOK" + args = [ + "outdir", + "lstconf", + "lstavg_toml_file", + "output_file_select", + "kernel", + ] + else: + action = "LSTBIN" + args = [ + "lstconf", + "lstavg_toml_file", + "output_file_select", + ] + + tomldict = { + "Options": options, + "LSTBIN_OPTS": lstbin_opts, + "FILE_CFG": file_cfg, + "LSTAVG_OPTS": lstavg_opts, + "WorkFlow": {"actions": [action]}, + action: {"args": args}, + } + + with open(fl, "w") as fl: + toml.dump(tomldict, fl) + + +@pytest.fixture(scope="module") +def lsttoml_direct_datafiles(tmp_path_factory) -> Path: + """Make a direct lstbin config file.""" + fl = tmp_path_factory.mktemp("data") / "lstbin_direct.toml" + make_lstbin_config_file( + fl, datafiles=["zen.2458043.40141.HH.uvh5", "zen.2458043.40887.HH.uvh5"] + ) + return fl + + +@pytest.fixture(scope="module") +def lsttoml_direct_datafiles_glob(tmp_path_factory) -> Path: + """Make a direct lstbin config file.""" + fl = tmp_path_factory.mktemp("data") / "lstbin_direct.toml" + make_lstbin_config_file( + fl, + datafiles=[ + "zen.2458043.*.HH.uvh5", + "zen.2458044.*.HH.uvh5", + "zen.2458045.*.HH.uvh5", + ], + ) + return fl + + +@pytest.fixture(scope="module") +def datafiles_in_nightly_folders(tmp_path_factory) -> Path: + + topdir = tmp_path_factory.mktemp("nightly-data") + + # Also, put our input files into nightly folders + for night in ["2458043", "2458044", "2458045"]: + ndir = topdir / night + ndir.mkdir() + + for fl in Path(DATA_PATH).glob("zen.*.uvh5"): + if f"{night}." in fl.name: + shutil.copy(fl, ndir / fl.name) + + return topdir + + +@pytest.fixture(scope="module") +def lsttoml_notebook_datafiles( + tmp_path_factory, datafiles_in_nightly_folders: Path +) -> Path: + """Make a notebook lstbin config file.""" + fl = tmp_path_factory.mktemp("data") / "lstbin_notebook.toml" + make_lstbin_config_file( + fl, + datafiles={ + "datadir": str(datafiles_in_nightly_folders), + "nightdirs": [fl.name for fl in datafiles_in_nightly_folders.glob("*")], + "sum_or_diff": "HH", + "extension": "uvh5", + }, + notebook=True, + ) + return fl + + +# @pytest.mark.filterwarnings("ignore:The default for the `center` keyword has changed") +# @pytest.mark.filterwarnings("ignore: A value for the") +# @pytest.mark.filterwarnings("ignore:dlst should not be set to (string) 'None'") +@pytest.mark.parametrize( + "config_file", + [ + "lsttoml_direct_datafiles", + "lsttoml_direct_datafiles_glob", + "lsttoml_notebook_datafiles", + ], +) +@pytest.mark.parametrize("give_mf_name", [True, False]) +def test_build_makeflow_from_config_lstbin_options( + config_file, + tmp_path_factory, + request, + give_mf_name, +): + """Test building a makeflow from a lstbin config file. + + In particular, this function calls the build_makeflow_from_config directly, + which dispatches to the build_lstbin_makeflow_from_config function. + """ + config_file = request.getfixturevalue(config_file) + + # test lstbin version with options + obsids = None + work_dir = tmp_path_factory.mktemp("test_output") + outfile = work_dir / config_file.name.replace(".toml", ".mf") + + mt.build_makeflow_from_config( + obsids, + config_file, + mf_name=outfile.name if give_mf_name else None, + work_dir=work_dir, + outdir=work_dir, # pass directly so that we can check the output + ) + + # make sure the output files we expected appeared + assert outfile.exists() + + +# @hc_skip +# @pytest.mark.filterwarnings("ignore:The default for the `center` keyword has changed") +# @pytest.mark.filterwarnings("ignore:dlst should not be set to (string) 'None'") +# @pytest.mark.parametrize( +# "provide_outfile,v2", [(False, False), (True, False), (False, True)] +# ) +# def test_build_lstbin_makeflow_from_config( +# config_options, tmp_path_factory, provide_outfile: bool, v2: bool +# ): +# # define load in config +# config_file = config_options["config_file_lstbin"] + +# if v2: +# config_file = config_file.replace("lstbin", "lstbin_v2") + +# # setup vars +# work_dir = tmp_path_factory.mktemp("test_output") +# if provide_outfile: +# mf_output = "output.mf" +# else: +# mf_output = os.path.splitext(os.path.basename(config_file))[0] + ".mf" +# outfile = work_dir / mf_output + +# kwargs = {"work_dir": str(work_dir), "parent_dir": DATA_PATH, "outdir": work_dir} +# if provide_outfile: +# kwargs["mf_name"] = outfile + +# mt.build_lstbin_makeflow_from_config(config_file, **kwargs) + +# # make sure the output files we expected appeared +# assert outfile.exists() + +# # check that the wrapper scripts have the right lines in them +# wrapper_scripts = [ +# f for f in sorted(os.listdir(work_dir)) if f.startswith("wrapper_") +# ] +# with open(work_dir / wrapper_scripts[0]) as infile: +# lines = infile.readlines() +# assert lines[0].strip() == "#!/bin/bash" +# assert lines[1].strip() == "source ~/.bashrc" +# assert lines[2].strip() == "conda activate hera" +# assert lines[3].strip() == "date" + + +# @hc_skip +# @pytest.mark.filterwarnings("ignore:The default for the `center` keyword has changed") +# @pytest.mark.filterwarnings("ignore: A value for the") +# @pytest.mark.parametrize("provide_outfile", [True, False]) +# def test_build_lstbin_makeflow_from_config_options( +# config_options, tmp_path_factory, provide_outfile +# ): +# # define load in config +# config_file = config_options["config_file_lstbin_options"] + +# # setup vars +# work_dir = tmp_path_factory.mktemp("test_output") +# if provide_outfile: +# mf_output = "output.mf" +# else: +# mf_output = os.path.splitext(os.path.basename(config_file))[0] + ".mf" + +# outfile = work_dir / mf_output + +# mt.build_lstbin_makeflow_from_config( +# config_file, +# mf_name=outfile, +# work_dir=str(work_dir), +# parent_dir=DATA_PATH, +# outdir=work_dir, +# ) + +# # make sure the output files we expected appeared +# assert outfile.exists() diff --git a/hera_opm/tests/test_mf_tools.py b/hera_opm/tests/test_mf_tools.py index 49e8c1e6..9568ffdd 100644 --- a/hera_opm/tests/test_mf_tools.py +++ b/hera_opm/tests/test_mf_tools.py @@ -7,23 +7,11 @@ import shutil import gzip import toml -from pathlib import Path from . import BAD_CONFIG_PATH from ..data import DATA_PATH from .. import mf_tools as mt -# define a pytest marker for skipping lstbin tests -try: - import hera_cal # noqa - - hc_installed = True -except ImportError: - hc_installed = False -hc_skip = pytest.mark.skipif( - not hc_installed, reason="hera_cal must be installed for this test" -) - @pytest.fixture(scope="module") def config_options(): @@ -44,12 +32,6 @@ def config_options(): config_dict["config_file_nopol"] = os.path.join( DATA_PATH, "sample_config", "nrao_rtp_nopol.toml" ) - config_dict["config_file_lstbin"] = os.path.join( - DATA_PATH, "sample_config", "lstbin.toml" - ) - config_dict["config_file_lstbin_options"] = os.path.join( - DATA_PATH, "sample_config", "lstbin_options.toml" - ) config_dict["config_file_setup_teardown"] = os.path.join( DATA_PATH, "sample_config", "nrao_rtp_setup_teardown.toml" ) @@ -800,164 +782,6 @@ def test_setup_teardown_errors(config_options): return -@hc_skip -@pytest.mark.filterwarnings("ignore:The default for the `center` keyword has changed") -@pytest.mark.filterwarnings("ignore:dlst should not be set to (string) 'None'") -@pytest.mark.parametrize( - "provide_outfile,v2", [(False, False), (True, False), (False, True)] -) -def test_build_lstbin_makeflow_from_config( - config_options, tmp_path_factory, provide_outfile: bool, v2: bool -): - # define load in config - config_file = config_options["config_file_lstbin"] - - if v2: - config_file = config_file.replace("lstbin", "lstbin_v2") - - # setup vars - work_dir = tmp_path_factory.mktemp("test_output") - if provide_outfile: - mf_output = "output.mf" - else: - mf_output = os.path.splitext(os.path.basename(config_file))[0] + ".mf" - outfile = work_dir / mf_output - - kwargs = {"work_dir": str(work_dir), "parent_dir": DATA_PATH, "outdir": work_dir} - if provide_outfile: - kwargs["mf_name"] = outfile - - mt.build_lstbin_makeflow_from_config(config_file, **kwargs) - - # make sure the output files we expected appeared - assert outfile.exists() - - # check that the wrapper scripts have the right lines in them - wrapper_scripts = [ - f for f in sorted(os.listdir(work_dir)) if f.startswith("wrapper_") - ] - with open(work_dir / wrapper_scripts[0]) as infile: - lines = infile.readlines() - assert lines[0].strip() == "#!/bin/bash" - assert lines[1].strip() == "source ~/.bashrc" - assert lines[2].strip() == "conda activate hera" - assert lines[3].strip() == "date" - - -@hc_skip -@pytest.mark.filterwarnings("ignore:The default for the `center` keyword has changed") -@pytest.mark.filterwarnings("ignore: A value for the") -@pytest.mark.parametrize("provide_outfile", [True, False]) -def test_build_lstbin_makeflow_from_config_options( - config_options, tmp_path_factory, provide_outfile -): - # define load in config - config_file = config_options["config_file_lstbin_options"] - - # setup vars - work_dir = tmp_path_factory.mktemp("test_output") - if provide_outfile: - mf_output = "output.mf" - else: - mf_output = os.path.splitext(os.path.basename(config_file))[0] + ".mf" - - outfile = work_dir / mf_output - - mt.build_lstbin_makeflow_from_config( - config_file, - mf_name=outfile, - work_dir=str(work_dir), - parent_dir=DATA_PATH, - outdir=work_dir, - ) - - # make sure the output files we expected appeared - assert outfile.exists() - - -@hc_skip -@pytest.mark.filterwarnings("ignore:The default for the `center` keyword has changed") -@pytest.mark.filterwarnings("ignore: A value for the") -def test_build_lstbin_makeflow_direct_options(config_options, tmp_path_factory): - # Get the config template - config_file = config_options["config_file_lstbin_options"] - # setup vars - work_dir = tmp_path_factory.mktemp("test_output") - mf_output = "output.mf" - outfile = work_dir / mf_output - - # Make new config with dynamic variables in it... - config = work_dir / "inputconf.toml" - with open(config_file, "r") as fl: - _cfg = toml.load(fl) - - _cfg["LSTBIN_OPTS"]["outdir"] = str(work_dir) - _cfg["LSTBIN_OPTS"]["parent_dir"] = DATA_PATH - print("DP: ", DATA_PATH) - with open(config, "w") as fl: - toml.dump(_cfg, fl) - - mt.build_lstbin_makeflow_from_config( - config, - mf_name=outfile, - work_dir=str(work_dir), - ) - - # make sure the output files we expected appeared - assert outfile.exists() - - -@hc_skip -@pytest.mark.filterwarnings("ignore:The default for the `center` keyword has changed") -@pytest.mark.filterwarnings("ignore: A value for the") -@pytest.mark.parametrize("label", ["", "label."]) -def test_build_lstbin_makeflow_simple(config_options, tmp_path_factory, label): - # Get the config template - config_file = config_options["config_file_lstbin"].replace( - "lstbin.", "lstbin_simple." - ) - - # setup vars - work_dir = tmp_path_factory.mktemp("test_output") - mf_output = "output.mf" - outfile = work_dir / mf_output - - # Make new config with dynamic variables in it... - config = work_dir / "inputconf.toml" - with open(config_file, "r") as fl: - _cfg = toml.load(fl) - - _cfg["LSTBIN_OPTS"]["outdir"] = str(work_dir) - _cfg["LSTBIN_OPTS"]["parent_dir"] = DATA_PATH - _cfg["LSTBIN_OPTS"]["datadir"] = str(work_dir) - _cfg["LSTBIN_OPTS"]["label"] = str(label)[:-1] - - with open(config, "w") as fl: - toml.dump(_cfg, fl) - - # Also, put our input files into nightly folders - (work_dir / "2458043").mkdir() - (work_dir / "2458044").mkdir() - (work_dir / "2458045").mkdir() - for fl in Path(DATA_PATH).glob("zen.*.uvh5"): - newfl = fl.with_suffix(f".{label}uvh5").name - if "2458043." in fl.name: - shutil.copy(fl, work_dir / "2458043" / newfl) - elif "2458044." in fl.name: - shutil.copy(fl, work_dir / "2458044" / newfl) - elif "2458045." in fl.name: - shutil.copy(fl, work_dir / "2458045" / newfl) - - mt.build_lstbin_makeflow_from_config( - config, - mf_name=outfile, - work_dir=str(work_dir), - ) - - # make sure the output files we expected appeared - assert outfile.exists() - - def test_build_makeflow_from_config(config_options): # define args obsids = config_options["obsids"][:1] @@ -985,31 +809,6 @@ def test_build_makeflow_from_config(config_options): return -@hc_skip -@pytest.mark.filterwarnings("ignore:The default for the `center` keyword has changed") -@pytest.mark.filterwarnings("ignore: A value for the") -@pytest.mark.filterwarnings("ignore:dlst should not be set to (string) 'None'") -def test_build_makeflow_from_config_lstbin_options(config_options, tmp_path_factory): - # test lstbin version with options - obsids = config_options["obsids"][:1] - config_file = config_options["config_file_lstbin_options"] - work_dir = tmp_path_factory.mktemp("test_output") - mf_output = os.path.splitext(os.path.basename(config_file))[0] + ".mf" - outfile = work_dir / mf_output - - mt.build_makeflow_from_config( - obsids, - config_file, - mf_name=str(outfile), - work_dir=str(work_dir), - parent_dir=DATA_PATH, - outdir=work_dir, - ) - - # make sure the output files we expected appeared - assert outfile.exists() - - def test_clean_wrapper_scripts(): # define args work_dir = os.path.join(DATA_PATH, "test_output") diff --git a/setup.cfg b/setup.cfg index d7670da6..8709ede6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ long_description_content_type = text/markdown [options.extras_require] test = - hera-calibration>=3.2.3 + hera-calibration>=3.3.0 pytest pytest-cov package =