From 08349f015d8c41c4f63f99d8f2ecfdcb68b46c3d Mon Sep 17 00:00:00 2001 From: Olivia Lynn Date: Fri, 17 May 2024 06:56:39 -0700 Subject: [PATCH 1/6] Read in proper env vars; curl data; run first command successfully --- tests/test_suite.sh | 50 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_suite.sh diff --git a/tests/test_suite.sh b/tests/test_suite.sh new file mode 100644 index 00000000..a8e88879 --- /dev/null +++ b/tests/test_suite.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +export OMP_NUM_THREADS='30' +tmpdir=$(python -c "import lephare as lp; print(f'{lp.LEPHAREDIR}')" | tail -n 1) # use tail to ignore the on-import print statement +tmpwork=$(python -c "import lephare as lp; print(f'{lp.dm.LEPHAREWORK}')" | tail -n 1) +export LEPHAREDIR=$tmpdir +export LEPHAREWORK=$tmpwork + +echo "LEPHAREDIR:" $LEPHAREDIR # TODO remove these at the end +echo "LEPHAREWORK:" $LEPHAREWORK + +# TODO check the paths in the files themselves - as in, the list files and such + +# Get the data +#curl -s -o ${LEPHAREDIR}/COSMOS.in https://raw.githubusercontent.com/lephare-photoz/lephare-data/main/examples/COSMOS.in +echo "STARTING DOWNLOAD" +if [ -d "$LEPHAREDIR" ]; then + curl -o ${LEPHAREDIR}/examples/COSMOS.in https://raw.githubusercontent.com/lephare-photoz/lephare-data/main/examples/COSMOS.in + if [ $? -eq 0 ]; then + echo "Download successful." + else + echo "Download failed." + fi +else + echo "Directory $LEPHAREDIR does not exist." +fi + +# Variables +export CONFIG_FILE="$LEPHAREDIR/examples/COSMOS.para" + +# Test suite commands +filter -c $CONFIG_FILE + +# sedtolib -c ./COSMOS.para -t S --STAR_SED $LEPHAREDIR/examples/STAR_MOD_ALL.list --LIB_ASCII YES +# mag_gal -c ./COSMOS.para -t S --LIB_ASCII YES --STAR_LIB_OUT ALLSTAR_COSMOS + +# sedtolib -c $LEPHAREDIR/examples/COSMOS.para -t Q --QSO_SED $LEPHAREDIR/sed/QSO/SALVATO09/AGN_MOD.list +# mag_gal -c $LEPHAREDIR/examples/COSMOS.para -t Q --MOD_EXTINC 0,1000 --EB_V 0.,0.1,0.2,0.3 --EXTINC_LAW SB_calzetti.dat --LIB_ASCII NO --Z_STEP 0.04,0,6 --LIB_ASCII YES + +# sedtolib -c $LEPHAREDIR/examples/COSMOS.para -t G --GAL_SED $LEPHAREDIR/examples/COSMOS_MOD.list --GAL_LIB LIB_VISTA +# mag_gal -c $LEPHAREDIR/examples/COSMOS.para -t G --GAL_LIB_IN LIB_VISTA --GAL_LIB_OUT VISTA_COSMOS_FREE --MOD_EXTINC 18,26,26,33,26,33,26,33 --EXTINC_LAW SMC_prevot.dat,SB_calzetti.dat,SB_calzetti_bump1.dat,SB_calzetti_bump2.dat --EM_LINES EMP_UV --EM_DISPERSION 0.5,0.75,1.,1.5,2. --Z_STEP 0.04,0,6 --LIB_ASCII YES + +# cat_out=zphot_short.out + +# zphota -c $LEPHAREDIR/examples/COSMOS.para --CAT_IN $LEPHAREDIR/examples/COSMOS.in --CAT_OUT $cat_out --ZPHOTLIB VISTA_COSMOS_FREE,ALLSTAR_COSMOS,QSO_COSMOS --ADD_EMLINES 0,100 --AUTO_ADAPT YES --Z_STEP 0.04,0,6 --CAT_LINES 1,100 --SPEC_OUT YES --PARA_OUT $LEPHAREDIR/examples/output.para --VERBOSE NO --ZFIX NO --PDZ_OUT $LEPHAREWORK/zphota/ + +# python $LEPHAREDIR/examples/figuresLPZ.py $cat_out +# python $LEPHAREDIR/examples/spec.py *.spec -d pdf -o $LEPHAREWORK/zphota/spec + +# mv $cat_out Id*.spec figuresLPZ.pdf $LEPHAREWORK/zphota/ \ No newline at end of file From 8e0ad675892ec4b7a9dd727d1428ae39f1129ad5 Mon Sep 17 00:00:00 2001 From: Olivia Lynn Date: Fri, 17 May 2024 07:41:47 -0700 Subject: [PATCH 2/6] All commands run; add some extra organization --- tests/test_suite.sh | 70 ++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/tests/test_suite.sh b/tests/test_suite.sh index a8e88879..879b2a92 100644 --- a/tests/test_suite.sh +++ b/tests/test_suite.sh @@ -1,50 +1,48 @@ #!/usr/bin/env bash +# Set the number of threads for OpenMP export OMP_NUM_THREADS='30' -tmpdir=$(python -c "import lephare as lp; print(f'{lp.LEPHAREDIR}')" | tail -n 1) # use tail to ignore the on-import print statement -tmpwork=$(python -c "import lephare as lp; print(f'{lp.dm.LEPHAREWORK}')" | tail -n 1) -export LEPHAREDIR=$tmpdir -export LEPHAREWORK=$tmpwork -echo "LEPHAREDIR:" $LEPHAREDIR # TODO remove these at the end -echo "LEPHAREWORK:" $LEPHAREWORK +# Set the LEPHAREDIR and LEPHAREWORK variables +export LEPHAREDIR=$(python -c "import lephare as lp; print(f'{lp.LEPHAREDIR}')" | tail -n 1) # use tail to ignore the on-import print statement +export LEPHAREWORK=$(python -c "import lephare as lp; print(f'{lp.dm.LEPHAREWORK}')" | tail -n 1) # use tail to ignore the on-import print statement -# TODO check the paths in the files themselves - as in, the list files and such +# Set catalog and config file names +CAT_IN="$LEPHAREDIR/examples/COSMOS.in" +CAT_OUT="zphot_short.out" +CONFIG_FILE="$LEPHAREDIR/examples/COSMOS.para" + +# For debugging: +# echo "LEPHAREDIR:" $LEPHAREDIR +# echo "LEPHAREWORK:" $LEPHAREWORK +# echo "CONFIG_FILE:" $CONFIG_FILE # Get the data -#curl -s -o ${LEPHAREDIR}/COSMOS.in https://raw.githubusercontent.com/lephare-photoz/lephare-data/main/examples/COSMOS.in -echo "STARTING DOWNLOAD" -if [ -d "$LEPHAREDIR" ]; then - curl -o ${LEPHAREDIR}/examples/COSMOS.in https://raw.githubusercontent.com/lephare-photoz/lephare-data/main/examples/COSMOS.in - if [ $? -eq 0 ]; then - echo "Download successful." - else - echo "Download failed." - fi -else - echo "Directory $LEPHAREDIR does not exist." -fi - -# Variables -export CONFIG_FILE="$LEPHAREDIR/examples/COSMOS.para" - -# Test suite commands -filter -c $CONFIG_FILE +echo "Downloading to $CAT_IN..." +curl -s -o $CAT_IN https://raw.githubusercontent.com/lephare-photoz/lephare-data/main/examples/COSMOS.in -# sedtolib -c ./COSMOS.para -t S --STAR_SED $LEPHAREDIR/examples/STAR_MOD_ALL.list --LIB_ASCII YES -# mag_gal -c ./COSMOS.para -t S --LIB_ASCII YES --STAR_LIB_OUT ALLSTAR_COSMOS +# Run commands +echo "Running filter..." +filter -c $CONFIG_FILE -# sedtolib -c $LEPHAREDIR/examples/COSMOS.para -t Q --QSO_SED $LEPHAREDIR/sed/QSO/SALVATO09/AGN_MOD.list -# mag_gal -c $LEPHAREDIR/examples/COSMOS.para -t Q --MOD_EXTINC 0,1000 --EB_V 0.,0.1,0.2,0.3 --EXTINC_LAW SB_calzetti.dat --LIB_ASCII NO --Z_STEP 0.04,0,6 --LIB_ASCII YES +echo "Running sedtolib and mag_gal for stars..." +sedtolib -c $CONFIG_FILE -t S --STAR_SED $LEPHAREDIR/examples/STAR_MOD_ALL.list --LIB_ASCII YES +mag_gal -c $CONFIG_FILE -t S --LIB_ASCII YES --STAR_LIB_OUT ALLSTAR_COSMOS -# sedtolib -c $LEPHAREDIR/examples/COSMOS.para -t G --GAL_SED $LEPHAREDIR/examples/COSMOS_MOD.list --GAL_LIB LIB_VISTA -# mag_gal -c $LEPHAREDIR/examples/COSMOS.para -t G --GAL_LIB_IN LIB_VISTA --GAL_LIB_OUT VISTA_COSMOS_FREE --MOD_EXTINC 18,26,26,33,26,33,26,33 --EXTINC_LAW SMC_prevot.dat,SB_calzetti.dat,SB_calzetti_bump1.dat,SB_calzetti_bump2.dat --EM_LINES EMP_UV --EM_DISPERSION 0.5,0.75,1.,1.5,2. --Z_STEP 0.04,0,6 --LIB_ASCII YES +echo "Running sedtolib and mag_gal for QSOs..." +sedtolib -c $CONFIG_FILE -t Q --QSO_SED $LEPHAREDIR/sed/QSO/SALVATO09/AGN_MOD.list +mag_gal -c $CONFIG_FILE -t Q --MOD_EXTINC 0,1000 --EB_V 0.,0.1,0.2,0.3 --EXTINC_LAW SB_calzetti.dat --LIB_ASCII NO --Z_STEP 0.04,0,6 --LIB_ASCII YES -# cat_out=zphot_short.out +echo "Running sedtolib and mag_gal for galaxies..." +sedtolib -c $CONFIG_FILE -t G --GAL_SED $LEPHAREDIR/examples/COSMOS_MOD.list --GAL_LIB LIB_VISTA +mag_gal -c $CONFIG_FILE -t G --GAL_LIB_IN LIB_VISTA --GAL_LIB_OUT VISTA_COSMOS_FREE --MOD_EXTINC 18,26,26,33,26,33,26,33 --EXTINC_LAW SMC_prevot.dat,SB_calzetti.dat,SB_calzetti_bump1.dat,SB_calzetti_bump2.dat --EM_LINES EMP_UV --EM_DISPERSION 0.5,0.75,1.,1.5,2. --Z_STEP 0.04,0,6 --LIB_ASCII YES -# zphota -c $LEPHAREDIR/examples/COSMOS.para --CAT_IN $LEPHAREDIR/examples/COSMOS.in --CAT_OUT $cat_out --ZPHOTLIB VISTA_COSMOS_FREE,ALLSTAR_COSMOS,QSO_COSMOS --ADD_EMLINES 0,100 --AUTO_ADAPT YES --Z_STEP 0.04,0,6 --CAT_LINES 1,100 --SPEC_OUT YES --PARA_OUT $LEPHAREDIR/examples/output.para --VERBOSE NO --ZFIX NO --PDZ_OUT $LEPHAREWORK/zphota/ +echo "Running zphota..." +zphota -c $CONFIG_FILE --CAT_IN $CAT_IN --CAT_OUT $CAT_OUT --ZPHOTLIB VISTA_COSMOS_FREE,ALLSTAR_COSMOS,QSO_COSMOS --ADD_EMLINES 0,100 --AUTO_ADAPT YES --Z_STEP 0.04,0,6 --CAT_LINES 1,100 --SPEC_OUT YES --PARA_OUT $LEPHAREDIR/examples/output.para --VERBOSE NO --ZFIX NO --PDZ_OUT $LEPHAREWORK/zphota/ -# python $LEPHAREDIR/examples/figuresLPZ.py $cat_out -# python $LEPHAREDIR/examples/spec.py *.spec -d pdf -o $LEPHAREWORK/zphota/spec +echo "Generating figures and specs..." +python $LEPHAREDIR/examples/figuresLPZ.py $CAT_OUT +python $LEPHAREDIR/examples/spec.py *.spec -d pdf -o $LEPHAREWORK/zphota/spec -# mv $cat_out Id*.spec figuresLPZ.pdf $LEPHAREWORK/zphota/ \ No newline at end of file +echo "Moving output files..." +mv $CAT_OUT Id*.spec figuresLPZ.pdf $LEPHAREWORK/zphota/ From 8c69e69928e966f091c842d313b386ff33f0222b Mon Sep 17 00:00:00 2001 From: Olivia Lynn Date: Fri, 17 May 2024 08:08:35 -0700 Subject: [PATCH 3/6] Add initial demo notebook for running test_suite.sh --- docs/notebooks/Command_line_demo.ipynb | 57 ++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 docs/notebooks/Command_line_demo.ipynb diff --git a/docs/notebooks/Command_line_demo.ipynb b/docs/notebooks/Command_line_demo.ipynb new file mode 100644 index 00000000..3267eab6 --- /dev/null +++ b/docs/notebooks/Command_line_demo.ipynb @@ -0,0 +1,57 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Command line demo\n", + "\n", + "For the time being, we assume you have \"test_suite.sh\" at the top level of your LEPHAREDIR dir.\n", + "\n", + "To discuss where we want to store this in the long term." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import lephare as lp\n", + "\n", + "test_suite_file = os.path.join(os.environ[\"LEPHAREDIR\"], \"test_suite.sh\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!bash $test_suite_file" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (lph)", + "language": "python", + "name": "lph" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From febf74a0799283ac0c80024b63a38f7ec99485d1 Mon Sep 17 00:00:00 2001 From: Raphael Shirley Date: Fri, 17 May 2024 21:09:49 +0200 Subject: [PATCH 4/6] Renamed folders and updated example notebook to download data first --- .../historical_examples}/color.py | 8 +++- .../historical_examples}/figuresLPP.py | 3 ++ .../historical_examples}/figuresLPZ.py | 3 ++ .../historical_examples}/spec.py | 3 ++ .../historical_examples}/test_suite.sh | 0 docs/notebooks/Command_line_demo.ipynb | 44 ++++++++++++++----- 6 files changed, 48 insertions(+), 13 deletions(-) rename {examples => docs/historical_examples}/color.py (99%) rename {examples => docs/historical_examples}/figuresLPP.py (99%) rename {examples => docs/historical_examples}/figuresLPZ.py (99%) rename {examples => docs/historical_examples}/spec.py (99%) rename {tests => docs/historical_examples}/test_suite.sh (100%) diff --git a/examples/color.py b/docs/historical_examples/color.py similarity index 99% rename from examples/color.py rename to docs/historical_examples/color.py index 1b4b4b07..3b5165cf 100644 --- a/examples/color.py +++ b/docs/historical_examples/color.py @@ -1,8 +1,12 @@ -from sys import * -from matplotlib.pyplot import * +# ruff: noqa +# These historical examples are note maintained or covered by tests or linting! + from math import * +from sys import * + import numpy as np from matplotlib.backends.backend_pdf import PdfPages +from matplotlib.pyplot import * ############################ diff --git a/examples/figuresLPP.py b/docs/historical_examples/figuresLPP.py similarity index 99% rename from examples/figuresLPP.py rename to docs/historical_examples/figuresLPP.py index f6674411..167dd06b 100644 --- a/examples/figuresLPP.py +++ b/docs/historical_examples/figuresLPP.py @@ -1,3 +1,6 @@ +# ruff: noqa +# These historical examples are note maintained or covered by tests or linting! + from sys import * from matplotlib.pyplot import * diff --git a/examples/figuresLPZ.py b/docs/historical_examples/figuresLPZ.py similarity index 99% rename from examples/figuresLPZ.py rename to docs/historical_examples/figuresLPZ.py index 051a6f96..b04b9639 100644 --- a/examples/figuresLPZ.py +++ b/docs/historical_examples/figuresLPZ.py @@ -1,3 +1,6 @@ +# ruff: noqa +# These historical examples are note maintained or covered by tests or linting! + from sys import * from matplotlib.pyplot import * diff --git a/examples/spec.py b/docs/historical_examples/spec.py similarity index 99% rename from examples/spec.py rename to docs/historical_examples/spec.py index bc1634b2..8720bd86 100644 --- a/examples/spec.py +++ b/docs/historical_examples/spec.py @@ -1,3 +1,6 @@ +# ruff: noqa +# These historical examples are note maintained or covered by tests or linting! + # Plot (file or window) the observed magnitudes # of the objects, along with best-fit templates, # reading the info form .zsp and .pdz output diff --git a/tests/test_suite.sh b/docs/historical_examples/test_suite.sh similarity index 100% rename from tests/test_suite.sh rename to docs/historical_examples/test_suite.sh diff --git a/docs/notebooks/Command_line_demo.ipynb b/docs/notebooks/Command_line_demo.ipynb index 3267eab6..f0c6e7da 100644 --- a/docs/notebooks/Command_line_demo.ipynb +++ b/docs/notebooks/Command_line_demo.ipynb @@ -6,9 +6,7 @@ "source": [ "# Command line demo\n", "\n", - "For the time being, we assume you have \"test_suite.sh\" at the top level of your LEPHAREDIR dir.\n", - "\n", - "To discuss where we want to store this in the long term." + "In this very simple notebook we download the files required to run lephare and demonstrate the command line example script." ] }, { @@ -18,9 +16,33 @@ "outputs": [], "source": [ "import os\n", - "import lephare as lp\n", - "\n", - "test_suite_file = os.path.join(os.environ[\"LEPHAREDIR\"], \"test_suite.sh\")" + "import lephare as lp" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Download the data required in the example config file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = lp.read_config(\"./data/COSMOS.para\")\n", + "lp.data_retrieval.get_auxiliary_data(\n", + " keymap=config, additional_files=[\"examples/COSMOS.in\", \"examples/output.para\", \"examples/COSMOS.para\"]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run the script. This will take around ten minutes." ] }, { @@ -29,15 +51,15 @@ "metadata": {}, "outputs": [], "source": [ - "!bash $test_suite_file" + "!bash ../historical_examples/test_suite.sh" ] } ], "metadata": { "kernelspec": { - "display_name": "Python (lph)", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "lph" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -49,9 +71,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.10.12" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } From 258e6d0413afb5666c5d299056cf0dbabd7b9e52 Mon Sep 17 00:00:00 2001 From: Raphael Shirley Date: Fri, 17 May 2024 21:11:44 +0200 Subject: [PATCH 5/6] Typo fix --- docs/historical_examples/color.py | 2 +- docs/historical_examples/figuresLPP.py | 2 +- docs/historical_examples/figuresLPZ.py | 2 +- docs/historical_examples/spec.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/historical_examples/color.py b/docs/historical_examples/color.py index 3b5165cf..3eeea802 100644 --- a/docs/historical_examples/color.py +++ b/docs/historical_examples/color.py @@ -1,5 +1,5 @@ # ruff: noqa -# These historical examples are note maintained or covered by tests or linting! +# These historical examples are not maintained or covered by tests or linting! from math import * from sys import * diff --git a/docs/historical_examples/figuresLPP.py b/docs/historical_examples/figuresLPP.py index 167dd06b..58a3dccb 100644 --- a/docs/historical_examples/figuresLPP.py +++ b/docs/historical_examples/figuresLPP.py @@ -1,5 +1,5 @@ # ruff: noqa -# These historical examples are note maintained or covered by tests or linting! +# These historical examples are not maintained or covered by tests or linting! from sys import * diff --git a/docs/historical_examples/figuresLPZ.py b/docs/historical_examples/figuresLPZ.py index b04b9639..1ff970dd 100644 --- a/docs/historical_examples/figuresLPZ.py +++ b/docs/historical_examples/figuresLPZ.py @@ -1,5 +1,5 @@ # ruff: noqa -# These historical examples are note maintained or covered by tests or linting! +# These historical examples are not maintained or covered by tests or linting! from sys import * diff --git a/docs/historical_examples/spec.py b/docs/historical_examples/spec.py index 8720bd86..7029b56f 100644 --- a/docs/historical_examples/spec.py +++ b/docs/historical_examples/spec.py @@ -1,5 +1,5 @@ # ruff: noqa -# These historical examples are note maintained or covered by tests or linting! +# These historical examples are not maintained or covered by tests or linting! # Plot (file or window) the observed magnitudes # of the objects, along with best-fit templates, From ac00039d3727af0b3a2df0289c8f73afd5e2dc00 Mon Sep 17 00:00:00 2001 From: Raphael Shirley Date: Sat, 18 May 2024 00:04:35 +0200 Subject: [PATCH 6/6] Updated to work with the default config --- docs/notebooks/Command_line_demo.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/notebooks/Command_line_demo.ipynb b/docs/notebooks/Command_line_demo.ipynb index f0c6e7da..c7df69b2 100644 --- a/docs/notebooks/Command_line_demo.ipynb +++ b/docs/notebooks/Command_line_demo.ipynb @@ -6,7 +6,9 @@ "source": [ "# Command line demo\n", "\n", - "In this very simple notebook we download the files required to run lephare and demonstrate the command line example script." + "In this very simple notebook we download the files required to run lephare and demonstrate the command line example script.\n", + "\n", + "This is not the reccomended way to run lephare but provides access to the legacy running commands." ] }, { @@ -32,7 +34,7 @@ "metadata": {}, "outputs": [], "source": [ - "config = lp.read_config(\"./data/COSMOS.para\")\n", + "config = lp.string_dict_to_keymap(lp.default_cosmos_config)\n", "lp.data_retrieval.get_auxiliary_data(\n", " keymap=config, additional_files=[\"examples/COSMOS.in\", \"examples/output.para\", \"examples/COSMOS.para\"]\n", ")"