Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Photometric Plot Simplification #176

Merged
merged 6 commits into from
May 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 41 additions & 13 deletions simple_app/plots.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,24 @@ def bokeh_formatter(p: figure) -> figure:
return p


def name_simplifier(s: str) -> str:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if you have a test for plot-related functions, but this one would be easy to write tests for.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will add this to #145

"""
Simplifies name of magnitude into something nicer to read

Parameters
----------
s
Input string

Returns
-------
s
Simplified input string
"""
s = s.replace('-', ' - ').replace('GAIA3.', '').replace('2MASS.', '').replace('WISE.', '')
return s


def spectra_plot(query: str, db_file: str, night_sky_theme: Theme,
js_callbacks: JSCallbacks) -> Tuple[Optional[str], Optional[str], Optional[int], Optional[str]]:
"""
Expand Down Expand Up @@ -126,7 +144,9 @@ def normalise() -> np.ndarray:
else:
fluxmed = np.nanmedian(flux)

return flux / fluxmed
if not np.isclose(fluxmed, 0, atol=1e-30):
return flux / fluxmed
return flux # unable to normalise by first 0.01um

# query the database for the spectra
db = SimpleDB(db_file) # open database
Expand Down Expand Up @@ -155,13 +175,15 @@ def normalise() -> np.ndarray:
for spec in t_spectra:
spectrum: Spectrum1D = spec['access_url']

# checking spectrum has good units and not only NaNs
# checking spectrum has good units and not only NaNs or 0s
try:
wave: np.ndarray = spectrum.spectral_axis.to(u.micron).value
flux: np.ndarray = spectrum.flux.value
nan_check: np.ndarray = ~np.isnan(flux) & ~np.isnan(wave)
wave = wave[nan_check]
flux = flux[nan_check]
zero_check: np.ndarray = ~np.isclose(flux, 0, atol=1e-30)
nanzero_check = nan_check & zero_check
wave = wave[nanzero_check]
flux = flux[nanzero_check]
if not len(wave):
raise ValueError

Expand Down Expand Up @@ -387,7 +409,7 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
_p_camd.x_range = Range1d(all_results_full[x_full_name].min(), all_results_full[x_full_name].max())
_p_camd.y_range = Range1d(all_results_full[y_full_name].max(), all_results_full[y_full_name].min())
_p_camd.xaxis.axis_label = x_shown_name
_p_camd.yaxis.axis_label = y_full_name
_p_camd.yaxis.axis_label = y_shown_name
_p_camd = bokeh_formatter(_p_camd)

# scatter plot for camd
Expand All @@ -405,7 +427,7 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
tap_tool_mag.callback = OpenURL(url='/load_solo/@source')
_button_mag_x_flip = Toggle(label='X Flip')
_button_mag_x_flip.js_on_click(CustomJS(code=js_callbacks.button_flip, args={'ax_range': _p_camd.x_range}))
_button_mag_y_flip = Toggle(label='Y Flip')
_button_mag_y_flip = Toggle(label='Y Flip', active=True)
_button_mag_y_flip.js_on_click(CustomJS(code=js_callbacks.button_flip, args={'ax_range': _p_camd.y_range}))
_dropdown_mag_x = Select(options=dropdown_menu, value=x_full_name) # x axis
_dropdown_mag_x.js_on_change('value', CustomJS(code=js_callbacks.dropdown_x_js,
Expand All @@ -423,6 +445,8 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
all_results_full = results_concat(all_results, all_photometry, all_parallaxes, all_spectral_types, all_bands)
all_results_full.dropna(axis=1, how='all', inplace=True)
all_bands = all_bands[np.isin(all_bands, all_results_full.columns)]
wanted_mags = ('GAIA3.G', 'GAIA3.Grp', '2MASS.J', '2MASS.H', '2MASS.Ks', 'WISE.W1', 'WISE.W2')
all_bands = np.array(list(set(wanted_mags).intersection(all_bands)))

# bokeh tools initialisation
full_cds = ColumnDataSource(all_results_full)
Expand All @@ -445,15 +469,17 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
just_colours = all_results_full.loc[:, colour_bands].copy()
x_full_name = just_colours.columns[0]
y_full_name = just_colours.columns[1]
x_shown_name = x_full_name.replace('-', ' - ')
y_shown_name = y_full_name.replace('-', ' - ')
axis_names = [col.replace('-', ' - ') for col in just_colours.columns]
x_shown_name = name_simplifier(x_full_name)
y_shown_name = name_simplifier(y_full_name)
axis_names = [name_simplifier(col) for col in just_colours.columns]
dropdown_menu = [*zip(just_colours.columns, axis_names), ]

# colour-colour plot
p_colour_colour, button_x_flip, button_y_flip, dropdown_x, dropdown_y = colour_colour_plot()

# prepping the absolute magnitudes for camd
wanted_mags = ('GAIA3.G', '2MASS.J', 'WISE.W1')
all_bands = np.array(list(set(wanted_mags).intersection(all_bands)))
just_mags: pd.DataFrame = all_results_full[all_bands]
absmagnames = np.array(["M_" + col for col in just_mags.columns])

Expand All @@ -463,8 +489,10 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
bad_cols.append(col)

absmagnames = absmagnames[~np.isin(absmagnames, bad_cols)]
dropdown_menu_mag = [*zip(absmagnames, absmagnames)]
absmag_shown_name = [name_simplifier(mag) for mag in absmagnames]
dropdown_menu_mag = [*zip(absmagnames, absmag_shown_name)]
y_full_name = absmagnames[0]
y_shown_name = absmag_shown_name[0]

# camd plot
p_camd, button_mag_x_flip, button_mag_y_flip, dropdown_mag_x, dropdown_mag_y = colour_absolute_magnitude_diagram()
Expand Down Expand Up @@ -602,9 +630,9 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu

x_full_name = just_colours.columns[0]
y_full_name = just_colours.columns[1]
x_shown_name = x_full_name.replace('-', ' - ')
y_shown_name = y_full_name.replace('-', ' - ')
axis_names = [col.replace('-', ' - ') for col in just_colours.columns]
x_shown_name = name_simplifier(x_full_name)
y_shown_name = name_simplifier(y_full_name)
axis_names = [name_simplifier(col) for col in just_colours.columns]
dropdown_menu = [*zip(just_colours.columns, axis_names), ]

# initialise plot
Expand Down
5 changes: 5 additions & 0 deletions simple_app/templates/about.html
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ <h2 class="display-3">Holdings</h2>
This database uses the <a href="https://docs.sqlalchemy.org/en/14/orm/index.html" target="_blank">
SQLAlchemy ORM</a> and is designed to be interacted with via the
<a href="https://pypi.org/project/astrodbkit2/" target="_blank"> astrodbkit2</a> package.
The full database can be downloaded from the
<a href="https://github.com/SIMPLE-AstroDB/SIMPLE-binary" target="_blank">binary repo</a>
and manipulated via local SQLite software.
The change log following the different versions of the database can be read
<a href="https://github.com/SIMPLE-AstroDB/SIMPLE-db/releases" target="_blank">here</a>.
To see more details about how this project got started and our first discussions,
check out the <a href="https://github.com/SIMPLE-AstroDB/SIMPLE-db/wiki/Original-Notes" target="_blank">
archived running notes in the Wiki</a>.
Expand Down
17 changes: 17 additions & 0 deletions simple_app/templates/solo_result.html
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,23 @@ <h2 class="display-4">Companion Relationships</h2>
</p>
{% endif %}
</div>

<div class="row table-responsive">
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Separate from PR topic, but I was able to confirm this part is working fine.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah apologies, this was for fixing #166

{% if everything.modeledparameters %}
<h2 class="display-3"><a href="{{ url_for('create_file_for_download', key='ModeledParameters') }}">
Modeled Parameters</a></h2>
<p class="display-6">
{{ everything.modeledparameters|safe }}
</p>
{% else %}
<h2 class="display-4">Modeled Parameters</h2>
<p class="display-6">
No Modeled Parameters for {{ query|safe }} in the SIMPLE Archive.
If some exists and should be ingested, please open an
<a href="https://github.com/SIMPLE-AstroDB/SIMPLE-db/issues/new?assignees=&labels=data+ingestion&projects=&template=missing_data.md&title=Data+ingest+request+from%3A+%3Cpublication%3E" target="_blank">issue</a>.
</p>
{% endif %}
</div>
</div>
<script>
$(document).ready(function() {
Expand Down
3 changes: 2 additions & 1 deletion simple_app/tests/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,10 @@ def test_results_concat(db, test_get_all_photometry, test_get_all_sources,
all_results, all_results_full = test_get_all_sources
all_parallaxes = test_get_all_parallaxes
all_spectral_types = test_get_all_spectral_types
wanted_mags = {'GAIA3.G', '2MASS.J', 'WISE.W1'}
all_results_concat = results_concat(all_results_full, all_photometry, all_parallaxes, all_spectral_types, all_bands)
assert all([col in all_results_concat.columns for col in ('ra_projected', 'dec_projected')])
assert all([f'M_{band}' in all_results_concat.columns for band in all_bands])
assert all([f'M_{band}' in all_results_concat.columns for band in all_bands if band in wanted_mags])
return


Expand Down
46 changes: 26 additions & 20 deletions simple_app/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,10 @@ def band_validate(checking_band: str):
raise KeyError(f'{checking_band_true} not yet a supported filter')
return checking_band_true

wanted_mags = {'GAIA3.G', 'GAIA3.Grp', '2MASS.J', '2MASS.H', '2MASS.Ks', 'WISE.W1', 'WISE.W2'}
wanted_cols = {'GAIA3.G-GAIA3.Grp', 'GAIA3.G-2MASS.J', '2MASS.J-2MASS.Ks', '2MASS.H-2MASS.Ks', 'WISE.W1-WISE.W2'}
all_bands = np.array(list(wanted_mags.intersection(all_bands)))

# looking at each band given in turn
d_cols: Dict[str, np.ndarray] = {}
for band in all_bands:
Expand All @@ -533,6 +537,9 @@ def band_validate(checking_band: str):
# don't make a colour of same band
if band == next_band:
continue
# only want certain colours defined above
elif f'{band}-{next_band}' not in wanted_cols:
continue

# validate band
next_band_true = band_validate(next_band)
Expand Down Expand Up @@ -625,33 +632,29 @@ def parse_photometry(photometry_df: pd.DataFrame, all_bands: np.ndarray, multi_s
else:
# initialise a DataFrame grouped by each target
df_group_photometry = photometry_df.groupby('source')
d_new_photometry = {col: np.empty(len(df_group_photometry)) for col in all_bands}
d_new_photometry['target'] = np.empty(len(df_group_photometry), dtype=str)
df_new_photometry = pd.DataFrame(d_new_photometry)

# process the photometry of each source
p = mp.Pool(processes=mp.cpu_count() - 1 or 1)
sources = p.map(one_source_iter, [targetdf for (_, targetdf) in df_group_photometry])
sources_list: List[Dict[str, object]] = [] # initialize empty list of sources

# rearrange columns
for i, (target, targetdf) in tqdm(enumerate(df_group_photometry), total=len(df_group_photometry),
desc='Photometry'):
# process photometry of each source
with mp.Pool(processes=mp.cpu_count() - 1 or 1) as pool:
results = pool.map(one_source_iter, [df for _, df in df_group_photometry])

specificphoto = sources[i]
for key in df_new_photometry.columns: # over all keys
for (target, _), data in tqdm(zip(df_group_photometry, results), total=len(df_group_photometry),
desc='Photometry'):

if key == 'target':
df_new_photometry.loc[i, key] = target
row_data = {'target': target}

# for the magnitude columns
else:
for band in all_bands:

try:
df_new_photometry.loc[i, key] = specificphoto.loc['magnitude', key]
try:
row_data[band] = data.loc['magnitude', band]
except KeyError:
row_data[band] = None

sources_list.append(row_data)

df_new_photometry = pd.DataFrame(sources_list)

# use None as filler value
except KeyError:
df_new_photometry.loc[i, key] = None
return df_new_photometry


Expand Down Expand Up @@ -758,6 +761,9 @@ def pogson_law(m: Union[float, pd.Series]) -> Union[float, np.ndarray]:
_abs_mag[mask] = m[mask] + 5 * np.log10(df.parallax[mask]) - 10
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason why pogson_law is defined inside another function? I know python allows for this, but for better testing, reuse, and clarity it could be moved outside of it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no real reason other than it's the only place it was needed, and I find it fun taking advantage of encapsulation. I do this in a few places IIRC

return _abs_mag

wanted_mags = {'GAIA3.G', '2MASS.J', 'WISE.W1'}
all_bands = np.array(list(wanted_mags.intersection(all_bands)))

# create absolute magnitude for each apparent magnitude
d_magnitudes: Dict[str, np.ndarray] = {}
for band in all_bands:
Expand Down
Loading