diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b00e304 --- /dev/null +++ b/.gitignore @@ -0,0 +1,193 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# eclipse +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +*~ + +local.properties +.settings/ +.project +.classpath +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Typically, this file would be tracked if it contains build/dependency configurations: +.project + +.DS* \ No newline at end of file diff --git a/usecases/README.md b/usecases/README.md index 7647f0f..56b465a 100644 --- a/usecases/README.md +++ b/usecases/README.md @@ -5,36 +5,37 @@ ### Connections between projects and use-cases +| Usecase vs Project | XMM | Chandra | Vizier | GLAST | LSST | GAVO | Gaia | Simbad | Other project | Mock data | +| -------------------| ------ | -----| ----- | ----- | ---- | ---- | ---- | ------ | ------------- | --------- | +| identity | C | C | C | C | C | C | C | P C | | | +| native_frames | | | P C | | | | | | | | +| simple_position | C | C | C | C | C | C | C | C | | | +| standard_properties| P C | C | C | C | C | C | C | C | | | +| precise_astrometry | | | C | | C | | C | | | P | +| column_grouping | | | P C | | | | | | | | +| combined_data | P C | C | C | C | C | C | | | | | +| complex-shaped-object| | | | P C | | | | | | | +| orbital-system | | | | P C | | | | | | | +| time-series | C | C | C | | C | P C | | | | | + * **P** Raw data provided * **C** Concerned by the use-case -| Usecase vs Project | XMM | Chandra | Vizier | GLAST | LSST | GAVO | Gaia | Simbad | Other project | Mock data | -| -------------------| ------ | ----- | ----- | ---- | --- | --- | ----- | ----- | ----- | ----- | -| identity | C | C | C | C | C | C |C | P C | | | -| native_frames | | | P C | | | | | | | | -| simple_position | C | C | C | C | C | C | C | C | | | -| standard_properties| P C | C | C | C | C | C | C | C | | | -| precise_astrometry | | | C | | C | | C | | | P | -| column_grouping | | | P C | | | | | | | | -| combined_data | P C | C | C | C | C | C | | | | | -| complex-shaped-object| | | | P C | | | | | | | -| orbital-system | | | | P C | | | | | | | -| time-series | C | C | C | | C | P C | | | | | ### Use case proposal coverage -| Usecase vs proposal | mango-proposal | md-proposal | -| ---------------------| -------------- | ----------- | -| identity | SI | | -| native_frames | SI | | -| simple_position | SI | | -| standard_properties | SI | | -| precise_astrometry | SI | | -| column_grouping | SI | | -| combined_data | SI | | -| complex-shaped-object| S | | -| orbital-system | S | | -| time-series | | SI | +| Usecase vs proposal | mango-proposal | md-proposal | mcd-implementation | modelinstanceinvot | +| ---------------------| -------------- | ----------- | ------------------ |--------------------| +| identity | S I | | | | +| native_frames | S I | | S I | | +| simple_position | S I | | | | +| standard_properties | S I | | | | +| precise_astrometry | S I | | | | +| column_grouping | S I | | | | +| combined_data | S I | | | | +| complex-shaped-object| S | | | | +| orbital-system | S | | | | +| time-series | | S I | S I | S I | * **S** supported * **I** implemented diff --git a/usecases/column_grouping/mcd-implementation/README.md b/usecases/column_grouping/mcd-implementation/README.md new file mode 100644 index 0000000..99e6436 --- /dev/null +++ b/usecases/column_grouping/mcd-implementation/README.md @@ -0,0 +1,78 @@ +# Implementation of 'Column Grouping' case + +# Overview + This case involves Vizier data with a variety of columns, some of which + are 'in some way' related to the Radial Velocity column. + + The primary goals of the case appear to be + * Illustrate that the model is able to associate these columns in a way that is easily resolved by a client. + + The data has been annotated using IVOA VO-DML Mapping syntax + * Note: The associated properties are basic types, and are (IMO) improperly modeled + as extensions of the Measure class. Since the content of the Parameter is not + relevant to the Association, I have not annotated the Associated Parameter content. + + Annotation was produced using the 'Jovial' modeling toolset (Java). Jovial + was written by Omar Laurino, and updated by me to the current data model content. + It provides utilities to help define and create instances of (annotations) + IVOA VO-DML compliant data models. + + Uses the 'rama' python package to parse annotated data file and instantiate + instances of VO Data Model Classes. The package also applies Adapters which + translate certain VO Data Model Classes to corresponding AstroPY types. + eg: meas:Point -> astropy:SkyCoord + eg: meas:Time -> astropy:Time + This package was developed by Omar Laurino, and updated by me to the current + data model content. + +## Resources Used +* Mapping Syntax + + Working Draft document: + https://volute.g-vo.org/svn/trunk/projects/dm/vo-dml-mapping/doc/VO-DML_mapping_WD.pdf + +* Jovial Library + + version used in this project: + https://github.com/mcdittmar/jovial + + master repository: + https://github.com/olaurino/jovial + +* Rama module + + version used in this project: + https://github.com/mcdittmar/rama + + master repository: + https://github.com/olaurino/rama + +## Details +* Load Annotated VOTable + ``` + doc = Reader( Votable(infile) ) + ``` + +* **Goal: Discover associated properties** + ``` + catalog = doc.find_instances(Source)[0] + sys.stdout.write("o Source:\n" ) + for param in ( catalog.parameter_dock ): + sys.stdout.write(" o Parameter: ucd='%s'\n"%(param.ucd) ) + if param.associated_parameters is None: + sys.stdout.write(" + no Associated Parameters\n" ) + else: + for item in ( param.associated_parameters ): + inst = item.referenced_instance + sys.stdout.write(" + Associated Parameter: ucd='%s'\n"%(inst.ucd) ) + ``` + Source: + + Parameter: ucd='pos.eq' + + no Associated Parameters + + Parameter: ucd='spect.dopplerVeloc' + + Associated Parameter: ucd='meta.code.qual' + + Associated Parameter: ucd='meta.number' + + Associated Parameter: ucd='meta.ref' + + Parameter: ucd='meta.code.qual' + + no Associated Parameters + + Parameter: ucd='meta.number' + + no Associated Parameters + + Parameter: ucd='meta.ref' + + no Associated Parameters + + diff --git a/usecases/column_grouping/mcd-implementation/vizier_grouped_col_annotated.vot b/usecases/column_grouping/mcd-implementation/vizier_grouped_col_annotated.vot new file mode 100644 index 0000000..52a7b29 --- /dev/null +++ b/usecases/column_grouping/mcd-implementation/vizier_grouped_col_annotated.vot @@ -0,0 +1,333 @@ + + + + mango + file:/Users/sao/Documents/IVOA/GitHub/ivoa-dm-examples/tmp/Mango-v1.0.vo-dml.xml + + + meas + https://www.ivoa.net/xml/Meas/20200908/Meas-v1.0.vo-dml.xml + + + coords + https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml + + + ivoa + https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + _coosys1 + + + + + + + + + + + + _prop03 + _prop04 + _prop05 + + + + + + + + + + _coosys2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + VizieR Astronomical Server vizier.u-strasbg.fr + Date: 2021-01-07T12:19:32 [V1.99+ (14-Oct-2013)] + Explanations and Statistics of UCDs: See LINK below + In case of problem, please report to: cds-question@unistra.fr + In this version, NULL integer columns are written as an empty string + <TD></TD>, explicitely possible from VOTable-1.3 + + + + + + +-oc.form=dec +-out.max=50 +-nav=cat:III/21&tab:{III/21/gcrv}&key:source=3021&HTTPPRM:& +-c.eq=J2000 +-c.r= 2 +-c.u=arcmin +-c.geom=r +-source=III/21/gcrv +-order=I +-out=recno +-out=GCRV +-out=GC +-out=HD +-out=RAB1950 +-out=DEB1950 +-out=mag +-out=n_mag +-out=pm +-out=SpT +-out=RV +-out=q_RV +-out=o_RV +-out=r_RV +Simbad=Simbad +-out=_RA.icrs +-out=_DE.icrs + + +* ex: catalogue III/21 (cf mail 12/juin) : https://vizier.u-strasbg.fr/viz-bin/VizieR?-source=3021 +on voudrait rassembler RV, q_RV (qualite), o_RV (nb observation), r_RV (reference) + + + + + + + General Catalogue of Stellar Radial Velocities (Wilson, 1953) + + + The General Catalogue of Stellar Radial Velocity + + + + + + Record number assigned by the VizieR team. Should Not be used for identification. + + + + [1,15106] GCRV (Wilson) running number + + + [1,33342]? Boss's General Catalog (I/113) [NULL integer written as an empty string] + + + + [1,292630]? Henry Draper Catalog (III/135) [NULL integer written as an empty string] + + + + + Right Ascension B1950 + + + Declination B1950 + + + Visual magnitude (or photographic, see n_mag) + + + [PV] Photographic (blue) magnitude, or Variable + + + ? Total proper motion [NULL integer written as an empty string] + + + + Spectral type (5) + + + ? Heliocentric radial velocity + + + + [A-E*] Quality of radial velocity (1) + + + Number of plates used for radial velocity + + + Abbreviation of observatory (3) + + + + Right ascension (ICRS) (computed by VizieR, not part of the original data) + + + + Declination (ICRS) (computed by VizieR, not part of the original data) + + + + + + +
11333422250010.0+15.597247A2-11.0B6S00.02339+16.1542
+matching records + + + +truncated result (maxtup=50) + + + diff --git a/usecases/time-series/modelinstanceinvot-cube-mapping/README.md b/usecases/time-series/modelinstanceinvot-cube-mapping/README.md new file mode 100644 index 0000000..9d4497c --- /dev/null +++ b/usecases/time-series/modelinstanceinvot-cube-mapping/README.md @@ -0,0 +1,196 @@ +## `cube_ts.xml` serialization + + +This proposal is derived from the MCD's [proposal](../mcd-implementation). + +It maps the same data on the same model (Cube) but by using another syntax, namely `ModelInstanceInVot` + +### Resources + +* The **ModelInstanceInVot** [repository](https://github.com/ivoa-std/ModelInstanceInVot) provides rationale, syntax rules, XSD schema and large data samples. + +* The [modelinstanceinvot-code](https://github.com/ivoa/modelinstanceinvot-code/) repository contains many Python tools operating different aspects of the data mapping. + +### Test tools + +The current proposal can be processed by [cube_ts.py](https://github.com/ivoa/modelinstanceinvot-code/modelinstanceinvot-code/python/client/demo/cube_ts.py) + +* In the frame of `ModelInstanceInVot` we are considering that providing JSON serializations of model instances is a good validation scheme as long as the code generating those JSONs has no dependence with any particular model or data set. + +* Such JSONs can be very easily processed with most of the languages (Py, JS, Java, Rust). Therefore it becomes easy for different softwares to share model instances. + +* The code snippets below show how to generate and process JSON serializations: + * Building the JSON Serialization + +```python +votable_path = os.path.join(data_dir, + "annotated_data", + "cube_ts.annot.xml" + ) +vodml_instance = VodmlInstance(votable_path) +vodml_instance.populate_templates() +vodml_instance.connect_join_iterators() + +instance = vodml_instance.get_root_element("cube:SparseCube") +``` + + * Retrieving the GLOBALS Types + +```python +coord_systems = instance_browser.get_globals_types() +print(coord_systems) + +['mango:stcextends.PhotometryCoordSys', 'coords:TimeSys', 'coords:SpaceSys'] +``` + + * Getting the time coordinate system + +``` +print(DictUtils.get_pretty_json(instance_browser.get_globals_by_type("coords:TimeSys"))) + +{ + "@ID": "TimeSys_BARYCENTER", + "@dmtype": "coords:TimeSys", + "coords:PhysicalCoordSys.frame": { + "@dmtype": "coords:TimeFrame", + "coords:TimeFrame.refPosition": { + "@dmtype": "coords:StdRefLocation", + "coords:StdRefLocation.position": { + "@dmtype": "ivoa:string", + "@value": "BARYCENTER" + } + }, + "coords:TimeFrame.timescale": { + "@dmtype": "ivoa:string", + "@value": "TCB" + } + } +} +``` + + * Getting the roles of the root object child + +```python +root_component_roles = instance_browser.get_root_component_roles() +print(root_component_roles) + +['cube:SparseCube.data', 'cube:SparseCube.dataset'] +``` + + * Getting the dataset instance + +```python +print(DictUtils.get_pretty_json(instance_browser.get_root_component_by_role('cube:SparseCube.dataset'))) + +{ + "@dmtype": "ds:experiment.ObsDataset", + "ds:dataset.Dataset.curation": { + "@dmtype": "ds:dataset.Curation", + "ds:dataset.Curation.publisher": { + "@dmtype": "ds:dataset.Publisher", + "ds:dataset.Publisher.publisherID": { + "@dmtype": "ivoa:anyURI", + "@value": "ivo://org.gavo.dc" + }, + "ds:party.Role.party": { + "@dmtype": "ds:party.Organization", + "ds:party.Party.name": { + "@dmtype": "ivoa:string", + "@value": "GAVO Data Center" + } + } + } + }, + "ds:dataset.Dataset.dataID": { + "@dmtype": "ds:dataset.DataID", + "ds:dataset.DataID.title": { + "@dmtype": "ivoa:string", + "@ref": "ndupgttbwtit", + "@value": "Gaia DR2 G photometry time series for star 1866721434011386240" + } + }, + "ds:dataset.Dataset.dataProductSubtype": { + "@dmtype": "ivoa:string", + "@value": "lightcurve" + }, + "ds:dataset.Dataset.dataProductType": { + "@dmtype": "ds:dataset.DataProductType", + "@value": "TIMESERIES" + }, + "ds:experiment.ObsDataset.calibLevel": { + "@dmtype": "ivoa:integer", + "@value": "1" + }, + "ds:experiment.ObsDataset.target": { + "@dmtype": "ds:experiment.Target", + "ds:experiment.BaseTarget.name": { + "@dmtype": "ivoa:string", + "@value": "1866721434011386240" + }, + "meas:Position.coord": { + "@dmtype": "meas:Position", + "ds:experiment.Target.position": { + "@dmtype": "coords:Point", + "coords:Coordinate.coordSys": { + "@dmref": "SpaceSys_ICRS" + }, + "coords:Point.axis1": { + "@dmtype": "ivoa:RealQuantity", + "@ref": "ndupgttbwuga", + "@value": 315.018457397759 + }, + "coords:Point.axis2": { + "@dmtype": "ivoa:RealQuantity", + "@ref": "ndupgttbwltt", + "@value": 35.3014389949649 + } + } + } + } +} +``` + + * Getting the column mapping. The syntax used for describing individual columns is not par of any standard. It is due the client implementation. + +```python +print(instance.get_flatten_data_head()) + +['meas:Time.coord(coords:JD.date) [col#0 obs_time]', 'meas:GenericMeasure.coord(coords:PhysicalCoordinate.cval) [col#1 phot]', 'meas:GenericMeasure.coord(coords:PhysicalCoordinate.cval) [col#2 flux]', 'meas:Error.statError(meas:Symmetrical.radius) [col#3 flux_error]'] +``` + + * Getting the mapped table data row per row. The elements of the data arrays returned by the iterator match the above column mapping on the base of their positions. + +```python + while True: + inst = instance._get_next_flatten_row() + if inst != None: + print(DictUtils.get_pretty_json(inst)) + else: + break +[ + 2456964.614751482, + 13.1433285545337, + 104238.6015625, + 784.7562866210938 +] +[ + 2456964.688753904, + 13.145307101711005, + 104048.8203125, + 203.0587615966797 +] +... +... +[ + 2457507.3690639553, + 13.019767042062135, + 116802.78125, + 137.4085693359375 +] +[ + 2457507.443075384, + 13.014477387452114, + 117373.2265625, + 457.4692077636719 +] +``` \ No newline at end of file diff --git a/usecases/time-series/modelinstanceinvot-cube-mapping/cube_ts.annot.xml b/usecases/time-series/modelinstanceinvot-cube-mapping/cube_ts.annot.xml new file mode 100644 index 0000000..74ae2c7 --- /dev/null +++ b/usecases/time-series/modelinstanceinvot-cube-mapping/cube_ts.annot.xml @@ -0,0 +1,512 @@ + + + + At the GAVO data Center, we hold several data collections part of + or derived from Gaia data release 2: + + * A version + of GDR2's gaia_source table with just enough columns + to allow basic science (but therefore a bit faster and simpler to + deal with + than the full gaia_source table). This table also + has Lindegren's RUWE measure for filtering out marginal solutions. + * The light + curves released with DR2 as both a TAP-queriable table + (light curves as arrays) and an SSA service. + * The gdr2dist.main table + with distances estimates computed by + Bailer-Jones et al (:bibcode:`2018AJ....156...58B`) that should be + used in preference to + simple parallax operations. + * The gdr2mock schema, which contains a virtual Gaia catalog + generated from a carefully built model + of the Galaxy. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + With data relase 2, the Gaia DPAC released lightcurves for objects + considered variable. This is such a + lightcurve (with invalid points + removed), for the GAIA/GAIA2.G band. + + Note that the magnitudes in phot are blindly converted to + flux using + the zero point given in the metadata. If flux_error/flux<0.1, you can + use 1.09*flux_error/flux as a good + estimate for the error; else the + distribution is so skewed that you should work with fluxes rather than + magnitude. + + The zero + point in the magnitude metadata has been taken from the SVO + filter profile service. + + + Observation time (JD in barycentric TCB). + + + Magnitude in GAIA/GAIA2.G, Vega system. + + + Integrated flux in GAIA/GAIA2.G. This is in CCD electrons per second. + + + Error in GAIA/GAIA2.G flux. + + + + + Gaia DR2 source_id of the object + + + Publisher-assigned title of the data set + + + Gaia DR2 RA of source object + + + Gaia DR2 Dec of source object + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
2456964.61475148213.1433285545337104238.602784.756287
2456964.68875390413.145307101711005104048.82203.058762
2456990.10505471113.051750456984891113412.211520.204834
2457019.341929545613.080361680003474110462.617115.218361
2457019.415932144513.085564601049686109934.539126.182106
2457065.89862806213.10800968029834107685.227389.942261
2457066.07478568113.104134378611676108070.27374.146286
2457066.148764073413.109822897622953107505.539188.289246
2457066.324947255713.10128410191652108354.35296.1394348
2457066.398952739313.100995597528158108383.148105.116081
2457081.582596307613.008353874924927118037.0781208.80115
2457081.656603452313.031208830929806115578.344195.553192
2457081.83276277213.001301277300179118806.3051245.4635
2457081.90677096413.031768797857135115518.75109.625305
2457130.393880981513.085899752457955109900.60982.0990906
2457130.570047330613.086879782098539109801.453183.367706
2457160.81267154312.990842870790523119956.242106.287735
2457188.056463165713.093987718627234109084.969501.87973
2457188.130474656413.101001387701364108382.57539.240479
2457230.261807570213.089965407523454109489.844929.107666
2457230.335817137313.100171884467892108465.406580.758545
2457242.016840520313.139704520994812104587.117144.969971
2457242.09084880413.13877482897491104676.71191.0609665
2457288.20028299213.24402393378045295005.8672103.232765
2457301.77766979313.23347162369042795933.7344256.566437
2457301.953819782513.2249412743107996690.429773.0098953
2457369.219609854313.25692456182893793883.6953141.182571
2457395.963780460413.159863472720843102663.156318.280396
2457446.199298490313.106650583650332107820.109111.751579
2457446.37545938513.103625026194251108120.984145.705811
2457446.449439002713.105192839011954107964.969185.77774
2457446.62562525813.099763974466097108506.164723.06897
2457463.633623213.030502392158379115653.57102.266785
2457463.707631608513.030362897946631115668.43124.907585
2457463.88379440513.02936852134825115774.414386.660461
2457463.95780412513.034002422426937115281.34498.1785889
2457507.369063955313.019767042062135116802.781137.408569
2457507.44307538413.014477387452114117373.227457.469208
+ +