Merge pull request #169 from Will-Cooper/updates

Updating requirements/environment

Procfile

@@ -1 +1 @@
-web: wget -O SIMPLE.db https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.sqlite && python -m simple_app.app_simple -p $PORT -i '0.0.0.0' -f 'SIMPLE.sqlite'
+web: wget -O SIMPLE.sqlite https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.sqlite && python -m simple_app.app_simple -p $PORT -i '0.0.0.0' -f 'SIMPLE.sqlite'

README.md

@@ -38,18 +38,13 @@ We also recommend keeping up to date with the repo changes, and most importantly
 the [astrodbkit2](https://github.com/dr-rodriguez/AstrodbKit2) package:
 ```bash
 git pull
-pip install git+https://github.com/dr-rodriguez/AstrodbKit2
+pip install -Ur requirements.txt
 ```
-You can also get the latest copy of the SQLite database binary file with:
+You can also get the latest copy of the SQLite database binary file again with:
 ```bash
 wget https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.db
 ```
 
-If updating requirements, update them with:
-```bash
-pip freeze > requirements.txt
-```
-
 ## Apache Config
 The major requirement for running this program on Apache
 is [`mod_wsgi`](https://flask.palletsprojects.com/en/2.1.x/deploying/mod_wsgi/)

environment.yml

@@ -1,138 +1,9 @@
 name: simple
 channels:
-  - http://ssb.stsci.edu/astroconda
   - defaults
 dependencies:
-  - _libgcc_mutex=0.1
-  - _openmp_mutex=5.1
-  - asttokens=2.0.5
-  - backcall=0.2.0
-  - blas=1.0
-  - ca-certificates=2022.10.11
-  - decorator=5.1.1
-  - executing=0.8.3
-  - freetype=2.12.1
-  - giflib=5.2.1
-  - greenlet=1.1.1
-  - intel-openmp=2021.4.0
-  - ipython_genutils=0.2.0
-  - jedi=0.18.1
-  - jpeg=9e
-  - lcms2=2.12
-  - ld_impl_linux-64=2.38
-  - lerc=3.0
-  - libdeflate=1.8
-  - libffi=3.4.2
-  - libgcc-ng=11.2.0
-  - libgomp=11.2.0
-  - libpng=1.6.37
-  - libstdcxx-ng=11.2.0
-  - libtiff=4.4.0
-  - libwebp=1.2.4
-  - libwebp-base=1.2.4
-  - lz4-c=1.9.3
-  - matplotlib-inline=0.1.6
-  - mkl=2021.4.0
-  - mkl-service=2.4.0
-  - mkl_fft=1.3.1
-  - mkl_random=1.2.2
-  - ncurses=6.3
-  - olefile=0.46
-  - openssl=1.1.1s
-  - parso=0.8.3
-  - pexpect=4.8.0
-  - pickleshare=0.7.5
-  - pluggy=1.0.0
-  - ptyprocess=0.7.0
-  - pure_eval=0.2.2
-  - py=1.11.0
-  - pygments=2.11.2
-  - pyparsing=3.0.9
-  - python=3.8.15
-  - python-dateutil=2.8.2
-  - pytz=2022.1
-  - readline=8.2
-  - six=1.16.0
-  - sqlite=3.40.0
-  - stack_data=0.2.0
-  - tk=8.6.12
-  - tomli=2.0.1
-  - tornado=6.2
-  - traitlets=5.1.1
-  - wcwidth=0.2.5
-  - xz=5.2.6
-  - yaml=0.2.5
-  - zlib=1.2.13
-  - zstd=1.5.2
+  - python=3.9.16
+  - pip
   - pip:
-    - asdf==2.10.1
-    - asdf-astropy==0.2.2
-    - asdf-coordinates-schemas==0.1.0
-    - asdf-standard==1.0.3
-    - asdf-transform-schemas==0.3.0
-    - asdf-wcs-schemas==0.1.1
-    - astrodbkit2==0.5
-    - astropy==5.2.1
-    - astroquery==0.4.6
-    - attrs==22.2.0
-    - beautifulsoup4==4.9.3
-    - bokeh==3.0.2
-    - certifi==2022.12.7
-    - cffi==1.14.5
-    - chardet==4.0.0
-    - click==8.1.3
-    - contourpy==1.0.6
-    - cryptography==40.0.1
-    - cycler==0.11.0
-    - exceptiongroup==1.1.1
-    - flask==2.2.3
-    - flask-cors==3.0.10
-    - flask-wtf==1.1.1
-    - fonttools==4.39.2
-    - gwcs==0.18.2
-    - html5lib==1.1
-    - idna==2.10
-    - importlib-metadata==5.1.0
-    - importlib-resources==5.10.0
-    - iniconfig==2.0.0
-    - ipython==8.11.0
-    - itsdangerous==2.1.2
-    - jeepney==0.6.0
-    - jinja2==3.1.2
-    - jmespath==1.0.1
-    - jsonschema==3.2.0
-    - keyring==23.0.1
-    - kiwisolver==1.4.4
-    - markdown2==2.4.0
-    - markupsafe==2.1.2
-    - matplotlib==3.7.1
-    - ndcube==2.0.3
-    - numpy==1.22.4
-    - packaging==23.0
-    - pandas==1.5.2
-    - pillow==9.3.0
-    - pip==23.0.1
-    - prompt-toolkit==3.0.38
-    - pycparser==2.20
-    - pyerfa==2.0.0.1
-    - pyrsistent==0.17.3
-    - pytest==7.2.2
-    - pyvo==1.4.1
-    - pyyaml==5.4.1
-    - requests==2.25.1
-    - scipy==1.6.2
-    - secretstorage==3.3.1
-    - semantic-version==2.8.5
-    - setuptools==67.2.0
-    - soupsieve==2.2.1
-    - specutils==1.9.1
-    - sqlalchemy==2.0.7
-    - tqdm==4.61.0
-    - typing-extensions==4.4.0
-    - urllib3==1.26.15
-    - webencodings==0.5.1
-    - werkzeug==2.2.3
-    - wheel==0.40.0
-    - wtforms==2.3.3
-    - xyzservices==2022.9.0
-    - zipp==3.4.1
+    - -r requirements.txt
+

requirements.txt

@@ -1,97 +1,18 @@
-asdf>=2.10.1
-asdf-astropy>=0.2.2
-asdf-coordinates-schemas>=0.1.0
-asdf-standard>=1.0.3
-asdf-transform-schemas>=0.3.0
-asdf-wcs-schemas>=0.1.1
-astrodbkit2>=0.5
-astropy>=5.2.1
-astroquery>=0.4.6
-asttokens>=2.0.5
-attrs>=22.2.0
-backcall>=0.2.0
-beautifulsoup4>=4.9.3
-bokeh>=3.0.2
-certifi>=2022.12.7
-cffi>=1.14.5
-chardet>=4.0.0
-click>=8.1.3
-contourpy>=1.0.6
-cryptography>=39.0.1
-decorator>=5.1.1
-executing>=0.8.3
-exceptiongroup>=1.1.1
-Flask>=2.2.3
-Flask-Cors>=3.0.10
-Flask-WTF>=1.1.1
-greenlet>=1.1.1
-gwcs>=0.18.2
-html5lib>=1.1
-idna>=2.10
-importlib-metadata>=5.1.0
-importlib-resources>=5.10.0
-iniconfig>=2.0.0
-ipython>=8.10.0
-ipython-genutils>=0.2.0
-itsdangerous>=2.1.2
-jedi>=0.18.1
-jeepney>=0.6.0
-Jinja2>=3.1.2
-jmespath>=1.0.1
-jsonschema>=3.2.0
-keyring>=23.0.1
-markdown2>=2.4.0
-MarkupSafe>=2.1.1
-matplotlib-inline>=0.1.6
-mkl-fft>=1.3.1
-mkl-random>=1.2.2
-mkl-service>=2.4.0
-ndcube>=2.0.3
-numpy>=1.22.4
-olefile>=0.46
-packaging>=23.0
-pandas>=1.5.2
-parso>=0.8.3
-pexpect>=4.8.0
-pickleshare>=0.7.5
-Pillow>=9.3.0
-pip>=22.2.2
-pluggy>=1.0.0,<2.0
-prompt-toolkit>=3.0.30,<3.1.0
-ptyprocess>=0.7.0
-pure-eval>=0.2.2
-py>=1.11.0
-pycparser>=2.20
-pyerfa>=2.0.0.1
-Pygments>=2.11.2
-pyparsing>=3.0.9
-pyrsistent>=0.17.3
-pytest>=7.2.2
-python-dateutil>=2.8.2
-pytz>=2022.1
-pyvo>=1.4.1
-PyYAML>=6.0
-requests>=2.25.1
-scipy>=1.6.2
-SecretStorage>=3.3.1
-semantic-version>=2.8.5
-setuptools>=65.5.1,<67.3.0
-six>=1.16.0
-soupsieve>=2.2.1
-specutils>=1.9.1
-SPLAT>=0.32
-SQLAlchemy>=2.0.7
-stack-data>=0.2.0
-tomli>=2.0.1
-tornado>=6.2
-tqdm>=4.61.0
-traitlets>=5.1.1
-typing_extensions>=4.4.0
-urllib3>=1.26.5
-wcwidth>=0.2.5
-webencodings>=0.5.1
-Werkzeug>=2.2.3
-wheel>=0.38.1
-WTForms>=2.3.3
-xyzservices>=2022.9.0
-zipp>=3.4.1
+astrodbkit2>=0.5.1
+astropy>=5.2.0,<6
+bokeh==3.0.2
+flask>=3.0,<4
+flask-cors>=4.0,<5
+flask-wtf>=1.2,<2
+markdown2>=2.4,<3
+multiprocess>=0.70,<1
+numpy>=1.24,<2
+pandas>=2.0,<2.1
+pysqlite3>=0.5,<1
+pytest>=8.0,<9
+requests>=2.31,<3
+specutils>=1.12,<2
+sqlalchemy>=2.0,<3
+tqdm>=4.66,<5
+werkzeug>=3.0,<4
+wtforms>=3.1,<4

simple_app/plots.py

@@ -549,7 +549,7 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu
         this_spectral_type = pd.DataFrame.from_dict(dict(spectral_type_code=[np.nan, ], adopted=[np.nan, ]))
 
     # use adopted spectral type if present
-    this_spectral_type['adopted'] = this_spectral_type.adopted.fillna(False)
+    this_spectral_type['adopted'] = this_spectral_type.adopted.astype(bool).fillna(False)
     if not this_spectral_type.adopted.any():
         this_spectral_type.loc[0, 'adopted'] = True
 
@@ -571,7 +571,7 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu
     else:
 
         # use adopted parallax, if present
-        this_parallaxes['adopted'] = this_parallaxes.adopted.fillna(False)
+        this_parallaxes['adopted'] = this_parallaxes.adopted.astype(bool).fillna(False)
         if not this_parallaxes.adopted.any():
             this_parallaxes.loc[0, 'adopted'] = True
         this_photometry['parallax'] = this_parallaxes.loc[this_parallaxes.adopted].parallax.iloc[0]

simple_app/simports.py

@@ -36,7 +36,7 @@
 from copy import deepcopy  # memory control
 from difflib import get_close_matches  # for redirecting bad file paths
 from io import StringIO, BytesIO, BufferedIOBase  # writing files without saving to disk
-import multiprocessing as mp  # multiprocessing for efficiency
+import multiprocess as mp  # multiprocessing for efficiency
 import os  # operating system
 import requests  # accessing internet
 from shutil import copy  # copying files

simple_app/utils.py

@@ -140,8 +140,20 @@ def list_concat(self, key: str, return_markdown: bool = True) -> Union[pd.DataFr
         """
         # construct dataframe for a given key corresponding to a table
         obj = self.results[key]
-        df: pd.DataFrame = pd.concat([pd.DataFrame(objrow, index=[i])
-                                      for i, objrow in enumerate(obj)], ignore_index=True)
+        df_list = []
+
+        # check each row for NaNs, only append to list if not wholly NaNs
+        for i, obj_row in enumerate(obj):
+            df_row = pd.DataFrame(obj_row, index=[i]).dropna(axis=0, how='all')
+
+            if len(df_row):
+                df_list.append(df_row)
+
+        # create a concatenated dataframe of all the rows in a given table
+        if len(df_list):
+            df: pd.DataFrame = pd.concat(df_list, ignore_index=True)
+        else:
+            df = pd.DataFrame(columns=list(obj[0].keys()))
 
         # switch whether to have a Markdown version of the table, or a normal DataFrame
         if return_markdown:
@@ -727,7 +739,7 @@ def absolute_magnitudes(df: pd.DataFrame, all_bands: np.ndarray) -> pd.DataFrame
         The output dataframe with absolute mags calculated
     """
 
-    def pogson_law(m: Union[float, np.ndarray]) -> Union[float, np.ndarray]:
+    def pogson_law(m: Union[float, pd.Series]) -> Union[float, np.ndarray]:
         """
         Distance modulus equation. Calculates the absolute magnitude only for sources with a positive parallax,
         otherwise returns a NaN
@@ -741,7 +753,10 @@ def pogson_law(m: Union[float, np.ndarray]) -> Union[float, np.ndarray]:
         _
             Absolute magnitude
         """
-        return np.where(df.parallax > 0, m + 5 * np.log10(df.parallax, where=df.parallax > 0) - 10, np.nan)
+        mask = df.parallax > 0
+        _abs_mag = np.full_like(m, fill_value=np.nan)
+        _abs_mag[mask] = m[mask] + 5 * np.log10(df.parallax[mask]) - 10
+        return _abs_mag
 
     # create absolute magnitude for each apparent magnitude
     d_magnitudes: Dict[str, np.ndarray] = {}