Merge pull request #176 from Will-Cooper/updates

Photometric Plot Simplification

simple_app/plots.py

@@ -76,6 +76,24 @@ def bokeh_formatter(p: figure) -> figure:
     return p
 
 
+def name_simplifier(s: str) -> str:
+    """
+    Simplifies name of magnitude into something nicer to read
+
+    Parameters
+    ----------
+    s
+        Input string
+
+    Returns
+    -------
+    s
+        Simplified input string
+    """
+    s = s.replace('-', ' - ').replace('GAIA3.', '').replace('2MASS.', '').replace('WISE.', '')
+    return s
+
+
 def spectra_plot(query: str, db_file: str, night_sky_theme: Theme,
                  js_callbacks: JSCallbacks) -> Tuple[Optional[str], Optional[str], Optional[int], Optional[str]]:
     """
@@ -126,7 +144,9 @@ def normalise() -> np.ndarray:
         else:
             fluxmed = np.nanmedian(flux)
 
-        return flux / fluxmed
+        if not np.isclose(fluxmed, 0, atol=1e-30):
+            return flux / fluxmed
+        return flux  # unable to normalise by first 0.01um
 
     # query the database for the spectra
     db = SimpleDB(db_file)  # open database
@@ -155,13 +175,15 @@ def normalise() -> np.ndarray:
     for spec in t_spectra:
         spectrum: Spectrum1D = spec['access_url']
 
-        # checking spectrum has good units and not only NaNs
+        # checking spectrum has good units and not only NaNs or 0s
         try:
             wave: np.ndarray = spectrum.spectral_axis.to(u.micron).value
             flux: np.ndarray = spectrum.flux.value
             nan_check: np.ndarray = ~np.isnan(flux) & ~np.isnan(wave)
-            wave = wave[nan_check]
-            flux = flux[nan_check]
+            zero_check: np.ndarray = ~np.isclose(flux, 0, atol=1e-30)
+            nanzero_check = nan_check & zero_check
+            wave = wave[nanzero_check]
+            flux = flux[nanzero_check]
             if not len(wave):
                 raise ValueError
 
@@ -387,7 +409,7 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
         _p_camd.x_range = Range1d(all_results_full[x_full_name].min(), all_results_full[x_full_name].max())
         _p_camd.y_range = Range1d(all_results_full[y_full_name].max(), all_results_full[y_full_name].min())
         _p_camd.xaxis.axis_label = x_shown_name
-        _p_camd.yaxis.axis_label = y_full_name
+        _p_camd.yaxis.axis_label = y_shown_name
         _p_camd = bokeh_formatter(_p_camd)
 
         # scatter plot for camd
@@ -405,7 +427,7 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
         tap_tool_mag.callback = OpenURL(url='/load_solo/@source')
         _button_mag_x_flip = Toggle(label='X Flip')
         _button_mag_x_flip.js_on_click(CustomJS(code=js_callbacks.button_flip, args={'ax_range': _p_camd.x_range}))
-        _button_mag_y_flip = Toggle(label='Y Flip')
+        _button_mag_y_flip = Toggle(label='Y Flip', active=True)
         _button_mag_y_flip.js_on_click(CustomJS(code=js_callbacks.button_flip, args={'ax_range': _p_camd.y_range}))
         _dropdown_mag_x = Select(options=dropdown_menu, value=x_full_name)  # x axis
         _dropdown_mag_x.js_on_change('value', CustomJS(code=js_callbacks.dropdown_x_js,
@@ -423,6 +445,8 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
     all_results_full = results_concat(all_results, all_photometry, all_parallaxes, all_spectral_types, all_bands)
     all_results_full.dropna(axis=1, how='all', inplace=True)
     all_bands = all_bands[np.isin(all_bands, all_results_full.columns)]
+    wanted_mags = ('GAIA3.G', 'GAIA3.Grp', '2MASS.J', '2MASS.H', '2MASS.Ks', 'WISE.W1', 'WISE.W2')
+    all_bands = np.array(list(set(wanted_mags).intersection(all_bands)))
 
     # bokeh tools initialisation
     full_cds = ColumnDataSource(all_results_full)
@@ -445,15 +469,17 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
     just_colours = all_results_full.loc[:, colour_bands].copy()
     x_full_name = just_colours.columns[0]
     y_full_name = just_colours.columns[1]
-    x_shown_name = x_full_name.replace('-', ' - ')
-    y_shown_name = y_full_name.replace('-', ' - ')
-    axis_names = [col.replace('-', ' - ') for col in just_colours.columns]
+    x_shown_name = name_simplifier(x_full_name)
+    y_shown_name = name_simplifier(y_full_name)
+    axis_names = [name_simplifier(col) for col in just_colours.columns]
     dropdown_menu = [*zip(just_colours.columns, axis_names), ]
 
     # colour-colour plot
     p_colour_colour, button_x_flip, button_y_flip, dropdown_x, dropdown_y = colour_colour_plot()
 
     # prepping the absolute magnitudes for camd
+    wanted_mags = ('GAIA3.G', '2MASS.J', 'WISE.W1')
+    all_bands = np.array(list(set(wanted_mags).intersection(all_bands)))
     just_mags: pd.DataFrame = all_results_full[all_bands]
     absmagnames = np.array(["M_" + col for col in just_mags.columns])
 
@@ -463,8 +489,10 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
             bad_cols.append(col)
 
     absmagnames = absmagnames[~np.isin(absmagnames, bad_cols)]
-    dropdown_menu_mag = [*zip(absmagnames, absmagnames)]
+    absmag_shown_name = [name_simplifier(mag) for mag in absmagnames]
+    dropdown_menu_mag = [*zip(absmagnames, absmag_shown_name)]
     y_full_name = absmagnames[0]
+    y_shown_name = absmag_shown_name[0]
 
     # camd plot
     p_camd, button_mag_x_flip, button_mag_y_flip, dropdown_mag_x, dropdown_mag_y = colour_absolute_magnitude_diagram()
@@ -602,9 +630,9 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu
 
     x_full_name = just_colours.columns[0]
     y_full_name = just_colours.columns[1]
-    x_shown_name = x_full_name.replace('-', ' - ')
-    y_shown_name = y_full_name.replace('-', ' - ')
-    axis_names = [col.replace('-', ' - ') for col in just_colours.columns]
+    x_shown_name = name_simplifier(x_full_name)
+    y_shown_name = name_simplifier(y_full_name)
+    axis_names = [name_simplifier(col) for col in just_colours.columns]
     dropdown_menu = [*zip(just_colours.columns, axis_names), ]
 
     # initialise plot

simple_app/templates/about.html

@@ -55,6 +55,11 @@ <h2 class="display-3">Holdings</h2>
         This database uses the <a href="https://docs.sqlalchemy.org/en/14/orm/index.html" target="_blank">
         SQLAlchemy ORM</a> and is designed to be interacted with via the
         <a href="https://pypi.org/project/astrodbkit2/" target="_blank"> astrodbkit2</a> package.
+        The full database can be downloaded from the
+        <a href="https://github.com/SIMPLE-AstroDB/SIMPLE-binary" target="_blank">binary repo</a>
+        and manipulated via local SQLite software.
+        The change log following the different versions of the database can be read
+        <a href="https://github.com/SIMPLE-AstroDB/SIMPLE-db/releases" target="_blank">here</a>.
         To see more details about how this project got started and our first discussions,
         check out the <a href="https://github.com/SIMPLE-AstroDB/SIMPLE-db/wiki/Original-Notes" target="_blank">
         archived running notes in the Wiki</a>.

simple_app/templates/solo_result.html

@@ -190,6 +190,23 @@ <h2 class="display-4">Companion Relationships</h2>
                 </p>
             {% endif %}
         </div>
+
+        <div class="row table-responsive">
+            {% if everything.modeledparameters %}
+                <h2 class="display-3"><a href="{{ url_for('create_file_for_download', key='ModeledParameters') }}">
+                    Modeled Parameters</a></h2>
+                <p class="display-6">
+                    {{ everything.modeledparameters|safe }}
+                </p>
+            {% else %}
+                <h2 class="display-4">Modeled Parameters</h2>
+                <p class="display-6">
+                    No Modeled Parameters for {{ query|safe }} in the SIMPLE Archive.
+                    If some exists and should be ingested, please open an
+                    <a href="https://github.com/SIMPLE-AstroDB/SIMPLE-db/issues/new?assignees=&labels=data+ingestion&projects=&template=missing_data.md&title=Data+ingest+request+from%3A+%3Cpublication%3E" target="_blank">issue</a>.
+                </p>
+            {% endif %}
+        </div>
     </div>
     <script>
         $(document).ready(function() {

simple_app/tests/test_utils.py

@@ -119,9 +119,10 @@ def test_results_concat(db, test_get_all_photometry, test_get_all_sources,
     all_results, all_results_full = test_get_all_sources
     all_parallaxes = test_get_all_parallaxes
     all_spectral_types = test_get_all_spectral_types
+    wanted_mags = {'GAIA3.G', '2MASS.J', 'WISE.W1'}
     all_results_concat = results_concat(all_results_full, all_photometry, all_parallaxes, all_spectral_types, all_bands)
     assert all([col in all_results_concat.columns for col in ('ra_projected', 'dec_projected')])
-    assert all([f'M_{band}' in all_results_concat.columns for band in all_bands])
+    assert all([f'M_{band}' in all_results_concat.columns for band in all_bands if band in wanted_mags])
     return
 
 

simple_app/utils.py

@@ -520,6 +520,10 @@ def band_validate(checking_band: str):
             raise KeyError(f'{checking_band_true} not yet a supported filter')
         return checking_band_true
 
+    wanted_mags = {'GAIA3.G', 'GAIA3.Grp', '2MASS.J', '2MASS.H', '2MASS.Ks', 'WISE.W1', 'WISE.W2'}
+    wanted_cols = {'GAIA3.G-GAIA3.Grp', 'GAIA3.G-2MASS.J', '2MASS.J-2MASS.Ks', '2MASS.H-2MASS.Ks', 'WISE.W1-WISE.W2'}
+    all_bands = np.array(list(wanted_mags.intersection(all_bands)))
+
     # looking at each band given in turn
     d_cols: Dict[str, np.ndarray] = {}
     for band in all_bands:
@@ -533,6 +537,9 @@ def band_validate(checking_band: str):
             # don't make a colour of same band
             if band == next_band:
                 continue
+            # only want certain colours defined above
+            elif f'{band}-{next_band}' not in wanted_cols:
+                continue
 
             # validate band
             next_band_true = band_validate(next_band)
@@ -625,33 +632,29 @@ def parse_photometry(photometry_df: pd.DataFrame, all_bands: np.ndarray, multi_s
     else:
         # initialise a DataFrame grouped by each target
         df_group_photometry = photometry_df.groupby('source')
-        d_new_photometry = {col: np.empty(len(df_group_photometry)) for col in all_bands}
-        d_new_photometry['target'] = np.empty(len(df_group_photometry), dtype=str)
-        df_new_photometry = pd.DataFrame(d_new_photometry)
 
-        # process the photometry of each source
-        p = mp.Pool(processes=mp.cpu_count() - 1 or 1)
-        sources = p.map(one_source_iter, [targetdf for (_, targetdf) in df_group_photometry])
+        sources_list: List[Dict[str, object]] = []  # initialize empty list of sources
 
-        # rearrange columns
-        for i, (target, targetdf) in tqdm(enumerate(df_group_photometry), total=len(df_group_photometry),
-                                          desc='Photometry'):
+        # process photometry of each source
+        with mp.Pool(processes=mp.cpu_count() - 1 or 1) as pool:
+            results = pool.map(one_source_iter, [df for _, df in df_group_photometry])
 
-            specificphoto = sources[i]
-            for key in df_new_photometry.columns:  # over all keys
+        for (target, _), data in tqdm(zip(df_group_photometry, results), total=len(df_group_photometry),
+                                      desc='Photometry'):
 
-                if key == 'target':
-                    df_new_photometry.loc[i, key] = target
+            row_data = {'target': target}
 
-                # for the magnitude columns
-                else:
+            for band in all_bands:
 
-                    try:
-                        df_new_photometry.loc[i, key] = specificphoto.loc['magnitude', key]
+                try:
+                    row_data[band] = data.loc['magnitude', band]
+                except KeyError:
+                    row_data[band] = None
+
+            sources_list.append(row_data)
+
+        df_new_photometry = pd.DataFrame(sources_list)
 
-                    # use None as filler value
-                    except KeyError:
-                        df_new_photometry.loc[i, key] = None
     return df_new_photometry
 
 
@@ -758,6 +761,9 @@ def pogson_law(m: Union[float, pd.Series]) -> Union[float, np.ndarray]:
         _abs_mag[mask] = m[mask] + 5 * np.log10(df.parallax[mask]) - 10
         return _abs_mag
 
+    wanted_mags = {'GAIA3.G', '2MASS.J', 'WISE.W1'}
+    all_bands = np.array(list(wanted_mags.intersection(all_bands)))
+
     # create absolute magnitude for each apparent magnitude
     d_magnitudes: Dict[str, np.ndarray] = {}
     for band in all_bands: