From a6700d7fa2705b81efc283749616abc02b413e36 Mon Sep 17 00:00:00 2001 From: Will-Cooper Date: Tue, 21 May 2024 17:35:32 +0100 Subject: [PATCH 1/6] photometry process bottleneck remove --- simple_app/utils.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/simple_app/utils.py b/simple_app/utils.py index 44237e3..a7772ae 100644 --- a/simple_app/utils.py +++ b/simple_app/utils.py @@ -625,33 +625,29 @@ def parse_photometry(photometry_df: pd.DataFrame, all_bands: np.ndarray, multi_s else: # initialise a DataFrame grouped by each target df_group_photometry = photometry_df.groupby('source') - d_new_photometry = {col: np.empty(len(df_group_photometry)) for col in all_bands} - d_new_photometry['target'] = np.empty(len(df_group_photometry), dtype=str) - df_new_photometry = pd.DataFrame(d_new_photometry) - # process the photometry of each source - p = mp.Pool(processes=mp.cpu_count() - 1 or 1) - sources = p.map(one_source_iter, [targetdf for (_, targetdf) in df_group_photometry]) + sources_list: List[Dict[str, object]] = [] # initialize empty list of sources - # rearrange columns - for i, (target, targetdf) in tqdm(enumerate(df_group_photometry), total=len(df_group_photometry), - desc='Photometry'): + # process photometry of each source + with mp.Pool(processes=mp.cpu_count() - 1 or 1) as pool: + results = pool.map(one_source_iter, [df for _, df in df_group_photometry]) - specificphoto = sources[i] - for key in df_new_photometry.columns: # over all keys + for (target, _), data in tqdm(zip(df_group_photometry, results), total=len(df_group_photometry), + desc='Photometry'): - if key == 'target': - df_new_photometry.loc[i, key] = target + row_data = {'target': target} - # for the magnitude columns - else: + for band in all_bands: + + try: + row_data[band] = data.loc['magnitude', band] + except KeyError: + row_data[band] = None + + sources_list.append(row_data) - try: - df_new_photometry.loc[i, key] = specificphoto.loc['magnitude', key] + df_new_photometry = pd.DataFrame(sources_list) - # use None as filler value - except KeyError: - df_new_photometry.loc[i, key] = None return df_new_photometry From 9c26d6e52066a51642aa2152b84c5ff9603f41a3 Mon Sep 17 00:00:00 2001 From: Will-Cooper Date: Tue, 21 May 2024 18:57:39 +0100 Subject: [PATCH 2/6] closes #102, only plotting select few colours and abs mags --- simple_app/plots.py | 42 +++++++++++++++++++++++++++++++++--------- simple_app/utils.py | 10 ++++++++++ 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/simple_app/plots.py b/simple_app/plots.py index 1c302a6..3bc8f41 100644 --- a/simple_app/plots.py +++ b/simple_app/plots.py @@ -76,6 +76,24 @@ def bokeh_formatter(p: figure) -> figure: return p +def name_simplifier(s: str) -> str: + """ + Simplifies name of magnitude into something nicer to read + + Parameters + ---------- + s + Input string + + Returns + ------- + s + Simplified input string + """ + s = s.replace('-', ' - ').replace('GAIA3.', '').replace('2MASS.', '').replace('WISE.', '') + return s + + def spectra_plot(query: str, db_file: str, night_sky_theme: Theme, js_callbacks: JSCallbacks) -> Tuple[Optional[str], Optional[str], Optional[int], Optional[str]]: """ @@ -387,7 +405,7 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, _p_camd.x_range = Range1d(all_results_full[x_full_name].min(), all_results_full[x_full_name].max()) _p_camd.y_range = Range1d(all_results_full[y_full_name].max(), all_results_full[y_full_name].min()) _p_camd.xaxis.axis_label = x_shown_name - _p_camd.yaxis.axis_label = y_full_name + _p_camd.yaxis.axis_label = y_shown_name _p_camd = bokeh_formatter(_p_camd) # scatter plot for camd @@ -405,7 +423,7 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, tap_tool_mag.callback = OpenURL(url='/load_solo/@source') _button_mag_x_flip = Toggle(label='X Flip') _button_mag_x_flip.js_on_click(CustomJS(code=js_callbacks.button_flip, args={'ax_range': _p_camd.x_range})) - _button_mag_y_flip = Toggle(label='Y Flip') + _button_mag_y_flip = Toggle(label='Y Flip', active=True) _button_mag_y_flip.js_on_click(CustomJS(code=js_callbacks.button_flip, args={'ax_range': _p_camd.y_range})) _dropdown_mag_x = Select(options=dropdown_menu, value=x_full_name) # x axis _dropdown_mag_x.js_on_change('value', CustomJS(code=js_callbacks.dropdown_x_js, @@ -423,6 +441,8 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, all_results_full = results_concat(all_results, all_photometry, all_parallaxes, all_spectral_types, all_bands) all_results_full.dropna(axis=1, how='all', inplace=True) all_bands = all_bands[np.isin(all_bands, all_results_full.columns)] + wanted_mags = ('GAIA3.G', 'GAIA3.Grp', '2MASS.J', '2MASS.H', '2MASS.Ks', 'WISE.W1', 'WISE.W2') + all_bands = np.array(list(set(wanted_mags).intersection(all_bands))) # bokeh tools initialisation full_cds = ColumnDataSource(all_results_full) @@ -445,15 +465,17 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, just_colours = all_results_full.loc[:, colour_bands].copy() x_full_name = just_colours.columns[0] y_full_name = just_colours.columns[1] - x_shown_name = x_full_name.replace('-', ' - ') - y_shown_name = y_full_name.replace('-', ' - ') - axis_names = [col.replace('-', ' - ') for col in just_colours.columns] + x_shown_name = name_simplifier(x_full_name) + y_shown_name = name_simplifier(y_full_name) + axis_names = [name_simplifier(col) for col in just_colours.columns] dropdown_menu = [*zip(just_colours.columns, axis_names), ] # colour-colour plot p_colour_colour, button_x_flip, button_y_flip, dropdown_x, dropdown_y = colour_colour_plot() # prepping the absolute magnitudes for camd + wanted_mags = ('GAIA3.G', '2MASS.J', 'WISE.W1') + all_bands = np.array(list(set(wanted_mags).intersection(all_bands))) just_mags: pd.DataFrame = all_results_full[all_bands] absmagnames = np.array(["M_" + col for col in just_mags.columns]) @@ -463,8 +485,10 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, bad_cols.append(col) absmagnames = absmagnames[~np.isin(absmagnames, bad_cols)] - dropdown_menu_mag = [*zip(absmagnames, absmagnames)] + absmag_shown_name = [name_simplifier(mag) for mag in absmagnames] + dropdown_menu_mag = [*zip(absmagnames, absmag_shown_name)] y_full_name = absmagnames[0] + y_shown_name = absmag_shown_name[0] # camd plot p_camd, button_mag_x_flip, button_mag_y_flip, dropdown_mag_x, dropdown_mag_y = colour_absolute_magnitude_diagram() @@ -602,9 +626,9 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu x_full_name = just_colours.columns[0] y_full_name = just_colours.columns[1] - x_shown_name = x_full_name.replace('-', ' - ') - y_shown_name = y_full_name.replace('-', ' - ') - axis_names = [col.replace('-', ' - ') for col in just_colours.columns] + x_shown_name = name_simplifier(x_full_name) + y_shown_name = name_simplifier(y_full_name) + axis_names = [name_simplifier(col) for col in just_colours.columns] dropdown_menu = [*zip(just_colours.columns, axis_names), ] # initialise plot diff --git a/simple_app/utils.py b/simple_app/utils.py index a7772ae..fa8a483 100644 --- a/simple_app/utils.py +++ b/simple_app/utils.py @@ -520,6 +520,10 @@ def band_validate(checking_band: str): raise KeyError(f'{checking_band_true} not yet a supported filter') return checking_band_true + wanted_mags = {'GAIA3.G', 'GAIA3.Grp', '2MASS.J', '2MASS.H', '2MASS.Ks', 'WISE.W1', 'WISE.W2'} + wanted_cols = {'GAIA3.G-GAIA3.Grp', 'GAIA3.G-2MASS.J', '2MASS.J-2MASS.Ks', '2MASS.H-2MASS.Ks', 'WISE.W1-WISE.W2'} + all_bands = np.array(list(wanted_mags.intersection(all_bands))) + # looking at each band given in turn d_cols: Dict[str, np.ndarray] = {} for band in all_bands: @@ -533,6 +537,9 @@ def band_validate(checking_band: str): # don't make a colour of same band if band == next_band: continue + # only want certain colours defined above + elif f'{band}-{next_band}' not in wanted_cols: + continue # validate band next_band_true = band_validate(next_band) @@ -754,6 +761,9 @@ def pogson_law(m: Union[float, pd.Series]) -> Union[float, np.ndarray]: _abs_mag[mask] = m[mask] + 5 * np.log10(df.parallax[mask]) - 10 return _abs_mag + wanted_mags = {'GAIA3.G', '2MASS.J', 'WISE.W1'} + all_bands = np.array(list(wanted_mags.intersection(all_bands))) + # create absolute magnitude for each apparent magnitude d_magnitudes: Dict[str, np.ndarray] = {} for band in all_bands: From a1e0483e55d2aed980bf4286b6e8f82ba5abfd01 Mon Sep 17 00:00:00 2001 From: Will-Cooper Date: Tue, 21 May 2024 19:33:11 +0100 Subject: [PATCH 3/6] fixes #173, trims out zeroes from flux array --- simple_app/plots.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/simple_app/plots.py b/simple_app/plots.py index 3bc8f41..6dbc96f 100644 --- a/simple_app/plots.py +++ b/simple_app/plots.py @@ -144,7 +144,9 @@ def normalise() -> np.ndarray: else: fluxmed = np.nanmedian(flux) - return flux / fluxmed + if not np.isclose(fluxmed, 0): + return flux / fluxmed + return flux # unable to normalise by first 0.01um # query the database for the spectra db = SimpleDB(db_file) # open database @@ -173,13 +175,15 @@ def normalise() -> np.ndarray: for spec in t_spectra: spectrum: Spectrum1D = spec['access_url'] - # checking spectrum has good units and not only NaNs + # checking spectrum has good units and not only NaNs or 0s try: wave: np.ndarray = spectrum.spectral_axis.to(u.micron).value flux: np.ndarray = spectrum.flux.value nan_check: np.ndarray = ~np.isnan(flux) & ~np.isnan(wave) - wave = wave[nan_check] - flux = flux[nan_check] + zero_check: np.ndarray = ~np.isclose(flux, 0) + nanzero_check = nan_check & zero_check + wave = wave[nanzero_check] + flux = flux[nanzero_check] if not len(wave): raise ValueError From a6a38316c7c10efa1bd4bef8f9e291b0a2d8bc79 Mon Sep 17 00:00:00 2001 From: Will-Cooper Date: Tue, 21 May 2024 19:38:06 +0100 Subject: [PATCH 4/6] fixes #166 --- simple_app/templates/solo_result.html | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/simple_app/templates/solo_result.html b/simple_app/templates/solo_result.html index 8ec8835..f39b96c 100644 --- a/simple_app/templates/solo_result.html +++ b/simple_app/templates/solo_result.html @@ -190,6 +190,23 @@

Companion Relationships

{% endif %} + +
+ {% if everything.modeledparameters %} +

+ Modeled Parameters

+

+ {{ everything.modeledparameters|safe }} +

+ {% else %} +

Modeled Parameters

+

+ No Modeled Parameters for {{ query|safe }} in the SIMPLE Archive. + If some exists and should be ingested, please open an + issue. +

+ {% endif %} +