From 3bdb6aa3d6d374bf9884b5cb385044cb54701fb8 Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 12 Dec 2017 17:26:06 -0500 Subject: [PATCH 01/29] WIP: schemas --- docs/collections/grants.rst | 2 +- regolith/schemas/schemas.py | 41 +++++++++++++++++++++++++++++++++++++ regolith/validators.py | 41 ++++++++++++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 regolith/schemas/schemas.py diff --git a/docs/collections/grants.rst b/docs/collections/grants.rst index 802b3e153..e64a3c783 100644 --- a/docs/collections/grants.rst +++ b/docs/collections/grants.rst @@ -21,7 +21,7 @@ The following lists key names mapped to its type and meaning for each entry. :begin_day: int, start day of the grant, optional :end_year: int, end year of the grant :end_month": str, end month of the grant -:end_day: str, end day of teh grant, optional +:end_day: str, end day of the grant, optional :team: list of dicts, information about the team members participating in the grant. These dicts have the following form: diff --git a/regolith/schemas/schemas.py b/regolith/schemas/schemas.py new file mode 100644 index 000000000..04df2356e --- /dev/null +++ b/regolith/schemas/schemas.py @@ -0,0 +1,41 @@ +schemas = {} +schemas['grants'] = { + '_id': {'description': 'short represntation, such as this-is-my-name', + 'type': (str, int, float)}, + 'amount': {'description': 'value of award', 'type': (int, float)}, + 'begin_day': { + 'description': 'start day of the grant, optional', + 'type': int}, + 'begin_month': {'description': 'start month of the grant', 'type': str}, + 'begin_year': {'description': 'start year of the grant', 'type': int}, + 'benefit_of_collaboration': {'description': 'optional, URL of document', + 'type': str}, + 'call_for_proposals': { + 'description': 'optional, URL to the call for proposals', + 'type': str}, + 'currency': {'description': "typically '$' or 'USD'", 'type': str}, + 'end_day': {'description': 'end day of teh grant, optional', 'type': str}, + 'end_month"': {'description': 'end month of the grant', 'type': str}, + 'end_year': {'description': 'end year of the grant', 'type': int}, + 'funder': {'description': 'the agency funding the work', 'type': str}, + 'grant_id': { + 'description': 'optional, the identfier for this work, eg #42024', + 'type': str}, + 'narrative': {'description': 'optional, URL of document', 'type': str}, + 'program': {'description': 'the program the work was funded under', + 'type': str}, + 'team': { + 'description': 'information about the team members participating in ' + 'the grant.', + 'type': [{"name": { + 'type': str, + 'description': "should match a person's name or AKA"}, + "position": { + 'type': str, + 'description': "PI, Co-PI, Co-I, Researcher, etc."}, + "institution": { + "type": str, + 'description': "The institution of this investigator"}, + "subaward_amount": {"type": (int, float)}, + "cv": {'type': str, 'description': "URL of document"}}]}, + 'title': {'description': 'actual title of proposal / grant', 'type': str}} diff --git a/regolith/validators.py b/regolith/validators.py index 64030873d..5a5a5e48f 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -1,10 +1,12 @@ """Validators and convertors for regolith input.""" +import collections import os import re from getpass import getpass from regolith.tools import string_types + def noop(x): """Does nothing, just returns the input.""" return x @@ -29,6 +31,7 @@ def is_bool(x): """Tests if something is a boolean""" return isinstance(x, bool) + def is_string(x): """Tests if something is a string""" return isinstance(x, string_types) @@ -54,6 +57,7 @@ def ensure_string(x): else: return str(x) + def ensure_database(db): db['name'] = ensure_string(db['name']) db['url'] = ensure_string(db['url']) @@ -109,4 +113,39 @@ def ensure_email(email): 'databases': (always_false, ensure_databases), 'stores': (always_false, ensure_stores), 'email': (always_false, ensure_email), - } +} + + +def validate_schema(record, schema): + if isinstance(record, dict): + total_keys = set(record.keys()) + total_keys.update(set(schema.keys())) + for k in total_keys: + if k not in schema: + pass + if k not in record and schema[k].get('required', False): + raise ValueError('{} is required'.format(k)) + elif k in record and k in schema: + print(k) + validate_schema(record[k], schema[k]) + elif isinstance(record, collections.Iterable) and not isinstance(record, + str): + for r in record: + validate_schema(r, schema) + else: + if not isinstance(record, schema['type']): + raise ValueError('Schema expected type: {}, ' + 'got type: {}'.format(type(record), + schema['type'])) + + +dict_schema = {'key': + {'type': str, 'required': True} + } +dict_schema2 = {'key': + {'key1': {'type': str, 'required': True, 'description': + 'key one schema'}, + 'key2': {'type': [int, float], 'required': False}, + 'key3': [{'type': [int, float], 'required': False}] + } + } From 4c6ce368ff4221596aac95d24cdba7142a64c5e0 Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 12 Dec 2017 20:01:43 -0500 Subject: [PATCH 02/29] TST: add test for validator --- .travis.yml | 5 ++++- regolith/schemas/schemas.py | 3 ++- regolith/tests/test_validators.py | 31 +++++++++++++++++++++++++++++ regolith/validators.py | 33 ++++++++++++------------------- requirements/run.txt | 1 + requirements/test.txt | 3 ++- 6 files changed, 53 insertions(+), 23 deletions(-) create mode 100644 regolith/tests/test_validators.py diff --git a/.travis.yml b/.travis.yml index 24926a997..b12aa9a10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,11 +20,14 @@ install: - conda update conda # Install dependencies - - conda install python + - conda install -n test --file requirements/run.txt + - source activate test - python setup.py install script: - set -e + - conda install --file requirements/test.txt + - pytest -vvs . - # Command to build your docs - pip install doctr - conda install --file requirements/doc.txt diff --git a/regolith/schemas/schemas.py b/regolith/schemas/schemas.py index 04df2356e..3cb3e040c 100644 --- a/regolith/schemas/schemas.py +++ b/regolith/schemas/schemas.py @@ -14,7 +14,8 @@ 'description': 'optional, URL to the call for proposals', 'type': str}, 'currency': {'description': "typically '$' or 'USD'", 'type': str}, - 'end_day': {'description': 'end day of teh grant, optional', 'type': str}, + 'end_day': {'description': 'end day of teh grant, optional', + 'type': (str, int)}, 'end_month"': {'description': 'end month of the grant', 'type': str}, 'end_year': {'description': 'end year of the grant', 'type': int}, 'funder': {'description': 'the agency funding the work', 'type': str}, diff --git a/regolith/tests/test_validators.py b/regolith/tests/test_validators.py new file mode 100644 index 000000000..a64341124 --- /dev/null +++ b/regolith/tests/test_validators.py @@ -0,0 +1,31 @@ +from regolith.validators import validate_schema +from regolith.schemas.schemas import schemas +import json +from io import StringIO + + +def test_grant_validation(): + raw_json = """{"_id": "SymPy-1.1", + "amount": 3000.0, + "begin_day": 1, + "begin_month": "May", + "begin_year": 2017, + "call_for_proposals": "https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ", + "end_day": 31, + "end_month": "December", + "end_year": 2017, + "funder": "NumFOCUS", + "narrative": "https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing", + "program": "Small Development Grants", + "team": [{"institution": "University of South Carolina", + "name": "Anthony Scopatz", + "position": "PI"}, + {"institution": "University of South Carolina", + "name": "Aaron Meurer", + "position": "researcher"} + ], + "title": "SymPy 1.1 Release Support"}""" + record = json.loads(raw_json) + validate_schema(record, schemas['grants']) + + diff --git a/regolith/validators.py b/regolith/validators.py index 5a5a5e48f..2a2bd70d1 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -116,36 +116,29 @@ def ensure_email(email): } -def validate_schema(record, schema): +def validate_schema(record, schema, key=None): if isinstance(record, dict): - total_keys = set(record.keys()) - total_keys.update(set(schema.keys())) + total_keys = set(schema.keys()) + remove_keys = ['required', 'type', 'description'] + for k in remove_keys: + if k in total_keys: + total_keys.remove(k) + total_keys.update(set(record.keys())) + for k in total_keys: if k not in schema: pass if k not in record and schema[k].get('required', False): raise ValueError('{} is required'.format(k)) elif k in record and k in schema: - print(k) - validate_schema(record[k], schema[k]) + validate_schema(record[k], schema[k], k) elif isinstance(record, collections.Iterable) and not isinstance(record, str): for r in record: - validate_schema(r, schema) + validate_schema(r, schema, key) else: if not isinstance(record, schema['type']): raise ValueError('Schema expected type: {}, ' - 'got type: {}'.format(type(record), - schema['type'])) - - -dict_schema = {'key': - {'type': str, 'required': True} - } -dict_schema2 = {'key': - {'key1': {'type': str, 'required': True, 'description': - 'key one schema'}, - 'key2': {'type': [int, float], 'required': False}, - 'key3': [{'type': [int, float], 'required': False}] - } - } + 'got type: {} in ' + '{{{}:{}}}'.format(type(record), + schema['type'], key, record)) diff --git a/requirements/run.txt b/requirements/run.txt index 4c0f633d2..27917a779 100644 --- a/requirements/run.txt +++ b/requirements/run.txt @@ -1 +1,2 @@ +python ruamel.yaml \ No newline at end of file diff --git a/requirements/test.txt b/requirements/test.txt index 4c0f633d2..24137e07a 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1 +1,2 @@ -ruamel.yaml \ No newline at end of file +ruamel.yaml +pytest \ No newline at end of file From 4c2e92da20d2a343a5dc78eb242efdb56b622c20 Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 12 Dec 2017 22:35:07 -0500 Subject: [PATCH 03/29] ENH: working schemas with examples --- docs/collections/abstracts.rst | 2 +- docs/collections/grants.rst | 2 +- docs/collections/people.rst | 12 +- regolith/schemas/exemplars.py | 280 +++++++++++++++++++ regolith/schemas/schemas.py | 437 +++++++++++++++++++++++++++--- regolith/tests/test_validators.py | 30 +- regolith/validators.py | 21 +- 7 files changed, 705 insertions(+), 79 deletions(-) create mode 100644 regolith/schemas/exemplars.py diff --git a/docs/collections/abstracts.rst b/docs/collections/abstracts.rst index a6523dd28..fef82ecf6 100644 --- a/docs/collections/abstracts.rst +++ b/docs/collections/abstracts.rst @@ -16,7 +16,7 @@ The following lists key names mapped to its type and meaning for each entry. :institution: str, name of the inistitution :references: str, HTML string of reference for the abstract itself, optional :text: str, HTML string of the abstract. -:timestamp: The time when the abstract was submitted. +:timestamp: float, The time when the abstract was submitted. YAML Example diff --git a/docs/collections/grants.rst b/docs/collections/grants.rst index e64a3c783..a0a44e3ff 100644 --- a/docs/collections/grants.rst +++ b/docs/collections/grants.rst @@ -30,7 +30,7 @@ The following lists key names mapped to its type and meaning for each entry. [{"name": str, # should match a person's name or AKA "position": str, # PI, Co-PI, Co-I, Researcher, etc. "institution": str, # The institution of this investigator - "subaward_amount", int or float, # optional + "subaward_amount": int or float, # optional "cv": str, # optional, URL of document }, ... diff --git a/docs/collections/people.rst b/docs/collections/people.rst index eca73904b..72d356408 100644 --- a/docs/collections/people.rst +++ b/docs/collections/people.rst @@ -54,7 +54,7 @@ The following lists key names mapped to its type and meaning for each entry. ] :funding: list of dicts, Funding and scholarship that the group member has - individually obtained in the past (optional). **WARNING:** this is not to be confused + individually obtained in the past, optional. **WARNING:** this is not to be confused with the **grants** collection, that may refee back to group members. These dicts have the following form: @@ -70,7 +70,7 @@ The following lists key names mapped to its type and meaning for each entry. ... ] -:service: list of dicts, Service that this group member has provided (optional). +:service: list of dicts, Service that this group member has provided, optional. These dicts have the following form: .. code-block:: python @@ -84,7 +84,7 @@ The following lists key names mapped to its type and meaning for each entry. ... ] -:honors: list of dicts, Honors that have been awarded to this group member (optional). +:honors: list of dicts, Honors that have been awarded to this group member, optional. These dicts have the following form: .. code-block:: python @@ -97,7 +97,7 @@ The following lists key names mapped to its type and meaning for each entry. ... ] -:teaching: list of dicts, Courses that this group member has taught, if any (optional). +:teaching: list of dicts, Courses that this group member has taught, if any, optional. These dicts have the following form: .. code-block:: python @@ -118,7 +118,7 @@ The following lists key names mapped to its type and meaning for each entry. ... ] -:membership: list of dicts, Profesional organizations this member is a part of (optional). +:membership: list of dicts, Profesional organizations this member is a part of, optional. These dicts have the following form: .. code-block:: python @@ -135,7 +135,7 @@ The following lists key names mapped to its type and meaning for each entry. ... ] -:skills: list of dicts, Skill the group member has (optional) +:skills: list of dicts, Skill the group member has, optional These dicts have the following form: .. code-block:: python diff --git a/regolith/schemas/exemplars.py b/regolith/schemas/exemplars.py new file mode 100644 index 000000000..1928698a9 --- /dev/null +++ b/regolith/schemas/exemplars.py @@ -0,0 +1,280 @@ +exemplars = {'abstracts': {'_id': 'Mouginot.Model', + 'coauthors': 'P.P.H. Wilson', + 'email': 'mouginot@wisc.edu', + 'firstname': 'Baptiste', + 'institution': 'University of Wisconsin-Madison', + 'lastname': 'Mouginot', + 'references': '[1] B. MOUGINOT, “cyCLASS: CLASS models for Cyclus,”, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 (2016).', + 'text': 'The CLASS team has developed high quality predictors based on pre-trained neural network...', + 'timestamp': '5/5/2017 13:15:59', + 'title': 'Model Performance Analysis'}, + 'assignments': {'_id': 'hw01-rx-power', + 'category': 'homework', + 'courses': ['EMCH-558-2016-S', + 'EMCH-758-2016-S'], + 'points': [1, 2, 3], + 'questions': ['1-9', '1-10', '1-12']}, + 'blog': {'_id': 'my-vision', + 'author': 'Anthony Scopatz', + 'day': 18, + 'month': 'September', + 'original': 'https://scopatz.com/my-vision/', + 'post': 'I would like see things move forward. Deep, I know!', + 'title': 'My Vision', + 'year': 2015}, + 'citations': {'_id': 'meurer2016sympy', + 'author': ['Meurer, Aaron', + 'Smith, Christopher P', + 'Paprocki, Mateusz', + "{\\v{C}}ert{\\'\\i}k, Ond{\\v{r}}ej", + 'Rocklin, Matthew', + 'Kumar, AMiT', + 'Ivanov, Sergiu', + 'Moore, Jason K', + 'Singh, Sartaj', + 'Rathnayake, Thilina', + 'Sean Vig', + 'Brian E Granger', + 'Richard P Muller', + 'Francesco Bonazzi', + 'Harsh Gupta', + 'Shivam Vats', + 'Fredrik Johansson', + 'Fabian Pedregosa', + 'Matthew J Curry', + 'Ashutosh Saboo', + 'Isuru Fernando', + 'Sumith Kulal', + 'Robert Cimrman', + 'Anthony Scopatz'], + 'entrytype': 'article', + 'journal': 'PeerJ Computer Science', + 'month': 'Jan', + 'pages': 'e103', + 'publisher': 'PeerJ Inc. San Francisco, USA', + 'title': 'SymPy: Symbolic computing in Python', + 'volume': '4', + 'year': '2017'}, + 'courses': {'_id': 'EMCH-552-2016-F', + 'active': False, + 'department': 'EMCH', + 'number': 552, + 'scale': [[0.875, 'A'], + [0.8125, 'B+'], + [0.75, 'B'], + [0.6875, 'C+'], + [0.625, 'C'], + [0.5625, 'D+'], + [0.5, 'D'], + [-1.0, 'F']], + 'season': 'F', + 'students': ['Human A. Person', 'Human B. Person'], + 'syllabus': 'emch552-2016-f-syllabus.pdf', + 'weights': {'class-notes': 0.15, + 'final': 0.3, + 'homework': 0.35, + 'midterm': 0.2}, + 'year': 2016}, + 'grades': { + '_id': 'Human A. Person-rx-power-hw02-EMCH-758-2017-S', + 'student': 'hap', + 'assignment': '2017-rx-power-hw02', + 'course': 'EMCH-758-2017-S', + 'scores': [1, 1.6, 3]}, + 'grants': {'_id': 'SymPy-1.1', + 'amount': 3000.0, + 'begin_day': 1, + 'begin_month': 'May', + 'begin_year': 2017, + 'call_for_proposals': 'https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ', + 'end_day': 31, + 'end_month': 'December', + 'end_year': 2017, + 'funder': 'NumFOCUS', + 'narrative': 'https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing', + 'program': 'Small Development Grants', + 'team': [ + {'institution': 'University of South Carolina', + 'name': 'Anthony Scopatz', + 'position': 'PI'}, + {'institution': 'University of South Carolina', + 'name': 'Aaron Meurer', + 'position': 'researcher'}], + 'title': 'SymPy 1.1 Release Support'}, + 'jobs': {'_id': '0004', + 'background_fields': ['Data Science', + 'Data Engineering', + 'Computer Engineering', + 'Computer Science', + 'Applied Mathematics', + 'Physics', + 'Nuclear Engineering', + 'Mechanical Engineering', + 'Or similar'], + 'compensation': [ + 'Salary and compensation will be based on prior work experience.'], + 'contact': 'Please send CV or resume to Prof. Scopatz at scopatzATcec.sc.edu.', + 'day': 1, + 'description': '

We are seeking a dedicated individual to help to aid in ...', + 'month': 'July', + 'open': False, + 'positions': ['Scientific Software Developer', + 'Programmer'], + 'start_date': 'ASAP', + 'title': 'Open Source Scientific Software Maintainer', + 'year': 2015}, + 'news': {'_id': '56b4eb6d421aa921504ef2a9', + 'author': 'Anthony Scopatz', + 'body': 'Dr. Robert Flanagan joined ERGS as a post-doctoral scholar.', + 'day': 1, + 'month': 'February', + 'year': 2016}, + 'people': {'_id': 'scopatz', + 'aka': ['Scopatz', + 'Scopatz, A', + 'Scopatz, A.', + 'Scopatz, A M', + 'Anthony Michael Scopatz'], + 'avatar': 'https://avatars1.githubusercontent.com/u/320553?v=3&s=200', + 'bio': 'Anthony Scopatz is currently an Assistant Professor', + 'education': [{'begin_year': 2008, + 'degree': 'Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program', + 'end_year': 2011, + 'institution': 'The University of Texas at Austin', + 'location': 'Austin, TX', + 'other': [ + 'Adviser: Erich A. Schneider', + 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis']}, + {'begin_year': 2006, + 'degree': 'M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program', + 'end_year': 2007, + 'institution': 'The University of Texas at Austin', + 'location': 'Austin, TX', + 'other': [ + 'Adviser: Erich A. Schneider', + 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership']}, + {'begin_year': 2002, + 'degree': 'B.S. Physics', + 'end_year': 2006, + 'institution': 'University of California, Santa Barbara', + 'location': 'Santa Barbara, CA', + 'other': [ + 'Graduated with a Major in Physics and a Minor in Mathematics']}], + 'email': 'scopatz@cec.sc.edu', + 'employment': [{'begin_year': 2015, + 'location': 'Columbia, SC', + 'organization': 'The University of South Carolina', + 'other': [ + 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.', + 'PyNE: The Nuclear Engineering Toolkit.', + 'Website: http://www.ergs.sc.edu/'], + 'position': 'Assistant Professor, Mechanical Engineering Department'}, + {'begin_year': 2013, + 'end_year': 2015, + 'location': 'Madison, WI', + 'organization': 'CNERG, The University of Wisconsin-Madison', + 'other': [ + 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.', + 'PyNE: The Nuclear Engineering Toolkit.', + 'Website: https://cnerg.github.io/'], + 'position': 'Associate Scientist, Engineering Physics Department'}, + {'begin_month': 'Nov', + 'begin_year': 2011, + 'end_month': 'May', + 'end_year': 2013, + 'location': 'Chicago, IL', + 'organization': 'The FLASH Center, The University of Chicago', + 'other': [ + 'NIF: Simulation of magnetic field generation from neutral plasmas using FLASH.', + 'CosmoB: Simulation of magnetic field generation from neutral plasmas using FLASH.', + 'FLASH4: High-energy density physics capabilities and utilities.', + 'Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from FLASH.', + 'OpacPlot: HDF5-based equation of state and opacity file format.', + 'Website: http://flash.uchicago.edu/site/'], + 'position': 'Research Scientist, Postdoctoral Scholar'}], + 'funding': [ + {'name': "Omega Laser User's Group Travel Award", + 'value': 1100, + 'year': 2013}, + { + 'name': "NIF User's Group Travel Award", + 'value': 1150, + 'year': 2013}], + 'membership': [{'begin_year': 2006, + 'organization': 'American Nuclear Society', + 'position': 'Member'}, + {'begin_year': 2013, + 'organization': 'Python Software Foundation', + 'position': 'Fellow'}], + 'name': 'Anthony Scopatz', + 'position': 'professor', + 'skills': [{'category': 'Programming Languages', + 'level': 'expert', + 'name': 'Python'}, + { + 'category': 'Programming Languages', + 'level': 'expert', + 'name': 'Cython'}], + 'teaching': [{ + 'course': 'EMCH 552: Intro to Nuclear Engineering', + 'description': 'This course is an introduction to nuclear physics.', + 'month': 'August', + 'organization': 'University of South Carolina', + 'position': 'Professor', + 'syllabus': 'https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc', + 'year': 2017}, + { + 'course': 'EMCH 558/758: Reactor Power Systems', + 'description': 'This course covers conventional reactors.', + 'month': 'January', + 'organization': 'University of South Carolina', + 'position': 'Professor', + 'syllabus': 'https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing', + 'year': 2017}], + 'title': 'Dr.'}, + 'projects': {'_id': 'Cyclus', + 'name': 'Cyclus', + 'description': 'Agent-Based Nuclear Fuel Cycle Simulator', + 'logo': 'http://fuelcycle.org/_static/big_c.png', + 'other': [ + 'Discrete facilities with discrete material transactions', + 'Low barrier to entry, rapid payback to adoption'], + 'repo': 'https://github.com/cyclus/cyclus/', + 'team': [{'begin_month': 'June', + 'begin_year': 2013, + 'end_month': 'July', + 'end_year': 2015, + 'name': 'Anthony Scopatz', + 'position': 'Project Lead'}], + 'website': 'http://fuelcycle.org/'}, + 'proposals': {'_id': 'mypropsal', + 'ammount': 1000000.0, + 'authors': ['Anthony Scopatz', + 'Robert Flanagan'], + 'currency': 'USD', + 'day': 18, + 'durration': 3, + 'full': { + 'benefit_of_collaboration': 'http://pdf.com/benefit_of_collaboration', + 'cv': [ + 'http://pdf.com/scopatz-cv', + 'http://pdf.com/flanagan-cv'], + 'narrative': 'http://some.com/pdf'}, + 'month': 'Aug', + 'pi': 'Anthony Scopatz', + 'pre': { + 'benefit_of_collaboration': 'http://pdf.com/benefit_of_collaboration', + 'cv': [ + 'http://pdf.com/scopatz-cv', + 'http://pdf.com/flanagan-cv'], + 'day': 2, + 'month': 'Aug', + 'narrative': 'http://some.com/pdf', + 'year': 1998}, + 'status': 'submitted', + 'title': 'A very fine proposal indeed', + 'year': 1999}, + 'students': {'_id': 'Human A. Person', + 'aka': ['H. A. Person'], + 'email': 'haperson@uni.edu', + 'university_id': 'HAP42'}} diff --git a/regolith/schemas/schemas.py b/regolith/schemas/schemas.py index 3cb3e040c..4b338ed82 100644 --- a/regolith/schemas/schemas.py +++ b/regolith/schemas/schemas.py @@ -1,42 +1,399 @@ -schemas = {} -schemas['grants'] = { - '_id': {'description': 'short represntation, such as this-is-my-name', - 'type': (str, int, float)}, - 'amount': {'description': 'value of award', 'type': (int, float)}, - 'begin_day': { - 'description': 'start day of the grant, optional', - 'type': int}, - 'begin_month': {'description': 'start month of the grant', 'type': str}, - 'begin_year': {'description': 'start year of the grant', 'type': int}, - 'benefit_of_collaboration': {'description': 'optional, URL of document', +schemas = { + 'grants': { + '_id': + {'description': 'short represntation, such as this-is-my-name', + 'required': True, + 'type': (str, int, float)}, + 'amount': {'description': 'value of award', + 'required': True, + 'type': (int, float)}, + 'begin_day': {'description': 'start day of the grant', + 'required': False, + 'type': int}, + 'begin_month': {'description': 'start month of the grant', + 'required': True, + 'type': str}, + 'begin_year': {'description': 'start year of the grant', + 'required': True, + 'type': int}, + 'benefit_of_collaboration': {'description': '', + 'required': False, + 'type': str}, + 'call_for_proposals': {'description': '', + 'required': False, + 'type': str}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': False, + 'type': str}, + 'end_day': {'description': 'end day of the grant', + 'required': False, + 'type': (str, int)}, + 'end_month"': {'description': 'end month of the grant', + 'required': False, + 'type': str}, + 'end_year': {'description': 'end year of the grant', + 'required': True, + 'type': int}, + 'funder': {'description': 'the agency funding the work', + 'required': True, + 'type': str}, + 'grant_id': {'description': 'the identfier for this work', + 'required': False, + 'type': str}, + 'narrative': { + 'description': '', + 'required': False, + 'type': str}, + 'program': {'description': 'the program the work was funded under', + 'required': True, + 'type': str}, + 'team': { + 'description': 'information about the team members participating ' + 'in the grant.', + 'required': True, + 'type': {'cv': {'required': False, 'type': str}, + 'institution': {'required': True, 'type': str}, + 'name': {'required': True, 'type': str}, + 'position': {'required': True, 'type': str}, + 'subaward_amount': { + 'required': False, + 'type': (int, float)}}}, + 'title': {'description': 'actual title of proposal / grant', + 'required': True, + 'type': str}}, + 'abstracts': { + '_id': { + 'description': 'Unique identifier for submission. This generally ' + 'includes the author name and part of the title.', + 'required': True, + 'type': str}, + 'coauthors': {'description': 'names of coauthors', + 'required': False, + 'type': str}, + 'email': {'description': 'contact email for the author.', + 'required': True, + 'type': str}, + 'firstname': {'description': 'first name of the author.', + 'required': True, + 'type': str}, + 'institution': {'description': 'name of the inistitution', + 'required': True, + 'type': str}, + 'lastname': {'description': 'last name of the author.', + 'required': True, + 'type': str}, + 'references': { + 'description': 'HTML string of reference for the abstract itself', + 'required': False, + 'type': str}, + 'text': {'description': 'HTML string of the abstract.', + 'required': True, + 'type': str}, + 'timestamp': { + 'description': 'The time when the abstract was submitted.', + 'required': True, + 'type': str}, + 'title': {'description': 'title of the presentation/paper.', + 'required': True, + 'type': str}}, + 'blog': { + '_id': {'description': 'short represntation, such as this-is-my-title', + 'required': True, + 'type': str}, + 'author': {'description': 'name or AKA of author', + 'required': True, + 'type': str}, + 'day': { + 'description': 'Publication day', + 'required': True, + 'type': int}, + 'month': { + 'description': 'Publication month', + 'required': True, + 'type': str}, + 'original': { + 'description': 'URL of original post, if this is a repost', + 'required': False, + 'type': str}, + 'post': {'description': 'actual contents of the post', + 'required': True, + 'type': str}, + 'title': {'description': 'full human readable title', + 'required': True, + 'type': str}, + 'year': { + 'description': 'Publication year', + 'required': True, + 'type': int}}, + 'grades': { + '_id': { + 'description': 'unique id, typically the ' + 'student-assignment-course', + 'required': True, + 'type': str}, + 'assignment': { + 'description': 'assignment id', + 'required': True, + 'type': str}, + 'course': {'description': 'course id', 'required': True, 'type': str}, + 'filename': {'description': 'path to file in store', + 'required': False, + 'type': str}, + 'scores': { + 'description': 'the number of points earned on each question', + 'required': True, + 'type': (int, float)}, + 'student': { + 'description': 'student id', + 'required': True, + 'type': str}}, + 'assignments': { + '_id': { + 'description': 'A unique id for the assignment, such a ' + 'HW01-EMCH-558-2016-S', + 'required': True, + 'type': str}, + 'category': {'description': "such as 'homework' or 'final'", + 'required': True, + 'type': str}, + 'courses': { + 'description': 'ids of the courses that have this assignment', + 'required': True, + 'type': str}, + 'file': {'description': 'path to assignment file in store', + 'required': False, + 'type': str}, + 'points': { + 'description': 'list of number of points possible for each ' + 'question. Length is the number of questions', + 'required': True, + 'type': (int, float)}, + 'question': { + 'description': 'titles for the questions on this assignment', + 'required': False, + 'type': str}, + 'solution': {'description': 'path to solution file in store', + 'required': False, + 'type': str}}, + 'projects': {'_id': {'description': 'Unique project identifier.', + 'required': True, + 'type': str}, + 'description': {'description': 'brief project description.', + 'required': True, 'type': str}, - 'call_for_proposals': { - 'description': 'optional, URL to the call for proposals', - 'type': str}, - 'currency': {'description': "typically '$' or 'USD'", 'type': str}, - 'end_day': {'description': 'end day of teh grant, optional', - 'type': (str, int)}, - 'end_month"': {'description': 'end month of the grant', 'type': str}, - 'end_year': {'description': 'end year of the grant', 'type': int}, - 'funder': {'description': 'the agency funding the work', 'type': str}, - 'grant_id': { - 'description': 'optional, the identfier for this work, eg #42024', - 'type': str}, - 'narrative': {'description': 'optional, URL of document', 'type': str}, - 'program': {'description': 'the program the work was funded under', + 'logo': {'description': 'URL to the project logo', + 'required': False, + 'type': str}, + 'name': {'description': 'name of the project.', + 'required': True, + 'type': str}, + 'other': { + 'description': 'other information about the project', + 'required': False, + 'type': str}, + 'repo': { + 'description': 'URL of the source code repo, if available', + 'required': False, + 'type': str}, + # XXX: FIXME + 'team': { + 'description': 'People who are/have been woking on this ' + 'project.', + 'required': True, + 'type': {'begin_month': {'required': False, 'type': str}, + 'begin_year': {'required': True, 'type': int}, + 'end_month': {'required': False, 'type': str}, + 'end_year': {'required': False, 'type': int}, + 'name': {'required': True, 'type': str}, + 'position': {'required': True, 'type': str}} + }, + 'website': {'description': 'URL of the website.', + 'required': True, + 'type': str}}, + 'proposals': { + '_id': {'description': 'short represntation, such as this-is-my-name', + 'required': True, + 'type': (str, int, float)}, + 'ammount': {'description': 'value of award', + 'required': True, + 'type': (int, float)}, + 'authors': {'description': 'other investigator names', + 'required': True, + 'type': str}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': True, + 'type': str}, + 'day': {'description': 'day that the proposal is due', + 'required': True, + 'type': int}, + 'durration': {'description': 'number of years', + 'required': True, + 'type': (int, float)}, + 'month': {'description': 'month that the proposal is due', + 'required': True, + 'type': str}, + 'pi': {'description': 'principal investigator name', + 'required': True, + 'type': str}, + # XXX: FIXME + 'pre': {'description': 'Information about the pre-proposal', + 'required': False, + 'type': dict}, + 'status': {'description': "e.g. 'submitted', 'accepted', 'rejected'", + 'required': True, + 'type': str}, + 'title': {'description': 'actual title of proposal', + 'required': True, + 'type': str}, + 'year': {'description': 'Year that the proposal is due', + 'required': True, + 'type': int}}, + 'students': { + '_id': {'description': 'short represntation, such as this-is-my-name', + 'required': True, 'type': str}, - 'team': { - 'description': 'information about the team members participating in ' - 'the grant.', - 'type': [{"name": { - 'type': str, - 'description': "should match a person's name or AKA"}, - "position": { - 'type': str, - 'description': "PI, Co-PI, Co-I, Researcher, etc."}, - "institution": { - "type": str, - 'description': "The institution of this investigator"}, - "subaward_amount": {"type": (int, float)}, - "cv": {'type': str, 'description': "URL of document"}}]}, - 'title': {'description': 'actual title of proposal / grant', 'type': str}} + 'aka': {'description': 'list of aliases', + 'required': False, + 'type': (list, str)}, + 'email': { + 'description': 'email address', + 'required': False, + 'type': str}, + 'university_id': { + 'description': 'The university identifier for the student', + 'required': False, + 'type': str}}, + 'people': {'_id': {'description': 'unique identifier for the group member', + 'required': True, + 'type': str}, + 'active': { + 'description': 'If the person is an active member, ' + 'default True.', + 'required': False, + 'type': bool}, + 'aka': { + 'description': 'list of aliases (also-known-as), useful ' + 'for identifying the group member in ' + 'citations or elsewhere.', + 'required': True, + 'type': str}, + 'avatar': { + 'description': 'URL to avatar', + 'required': True, + 'type': str}, + 'bio': {'description': 'short biographical text', + 'required': True, + 'type': str}, + 'collab': { + 'description': 'If the person is a collaborator, default ' + 'False.', + 'required': False, + 'type': bool}, + 'education': { + 'description': 'This contains the educational information ' + 'for the group member.', + 'required': True, + 'type': {'begin_month': {'required': False, 'type': str}, + 'begin_year': {'required': True, 'type': int}, + 'degree': {'required': True, 'type': str}, + 'end_month': {'required': False, 'type': str}, + 'end_year': {'required': True, 'type': int}, + 'gpa': {'required': False, 'type': (float, str)}, + 'institution': {'required': True, 'type': str}, + 'location': {'required': True, 'type': str}, + 'other': {'required': False, 'type': str}} + }, + 'email': {'description': 'email address of the group member', + 'required': False, + 'type': str}, + 'employment': { + 'description': 'Employment information, similar to ' + 'educational information.', + 'required': True, + 'type': {'begin_month': {'required': False, 'type': str}, + 'begin_year': {'required': True, 'type': int}, + 'end_month': {'required': False, 'type': str}, + 'end_year': {'required': False, 'type': int}, + 'location': {'required': True, 'type': str}, + 'organization': {'required': True, 'type': str}, + 'other': {'required': False, 'type': str}, + 'position': {'required': True, 'type': str}}}, + 'funding': { + 'description': 'Funding and scholarship that the group ' + 'member has individually obtained in the ' + 'past. **WARNING:** this is not to be ' + 'confused with the **grants** collection', + 'required': False, + 'type': {'currency': {'required': False, 'type': str}, + 'duration': {'required': False, 'type': str}, + 'month': {'required': False, 'type': str}, + 'name': {'required': True, 'type': str}, + 'value': {'required': True, 'type': (float, int)}, + 'year': {'required': True, 'type': int}} + }, + 'honors': { + 'description': 'Honors that have been awarded to this ' + 'group member', + 'required': False, + 'type': {'description': {'required': False, 'type': str}, + 'month': {'required': False, 'type': str}, + 'name': {'required': True, 'type': str}, + 'year': {'required': True, 'type': int}} + }, + 'membership': { + 'description': 'Profesional organizations this member is ' + 'a part of', + 'required': False, + 'type': {'begin_month': {'required': False, 'type': str}, + 'begin_year': {'required': True, 'type': int}, + 'description': {'required': False, 'type': str}, + 'end_month': {'required': False, 'type': str}, + 'end_year': {'required': False, 'type': int}, + 'organization': {'required': True, 'type': str}, + 'position': {'required': True, 'type': str}, + 'website': {'required': False, 'type': str}}}, + 'name': {'description': 'Full, canonical name for the person', + 'required': True, + 'type': str}, + 'position': { + 'description': 'such as professor, graduate student, ' + 'or scientist', + 'required': True, + 'type': str}, + 'service': { + 'description': 'Service that this group member has provided', + 'required': False, + 'type': {'description': {'required': False, 'type': str}, + 'duration': {'required': False, 'type': str}, + 'month': {'required': False, 'type': str}, + 'name': {'required': True, 'type': str}, + 'year': {'required': True, 'type': int}} + }, + 'skills': {'description': 'Skill the group member has', + 'required': False, + 'type': {'category': {'required': True, 'type': str}, + 'level': {'required': True, 'type': str}, + 'name': {'required': True, 'type': str}} + }, + 'teaching': { + 'description': 'Courses that this group member has ' + 'taught, if any', + 'required': False, + 'type': {'course': {'required': True, 'type': str}, + 'description': {'required': False, 'type': str}, + 'end_month': {'required': False, 'type': str}, + 'end_year': {'required': False, 'type': int}, + 'materials': {'required': False, 'type': str}, + 'month': {'required': False, 'type': str}, + 'organization': {'required': True, 'type': str}, + 'position': {'required': True, 'type': str}, + 'syllabus': {'required': False, 'type': str}, + 'video': {'required': False, 'type': str}, + 'website': {'required': False, 'type': str}, + 'year': {'required': True, 'type': int}} + }, + 'title': {'description': 'for example, Dr., etc.', + 'required': False, + 'type': str} + } +} diff --git a/regolith/tests/test_validators.py b/regolith/tests/test_validators.py index a64341124..a6da000a9 100644 --- a/regolith/tests/test_validators.py +++ b/regolith/tests/test_validators.py @@ -1,31 +1,11 @@ from regolith.validators import validate_schema from regolith.schemas.schemas import schemas -import json -from io import StringIO +from regolith.schemas.exemplars import exemplars +import pytest -def test_grant_validation(): - raw_json = """{"_id": "SymPy-1.1", - "amount": 3000.0, - "begin_day": 1, - "begin_month": "May", - "begin_year": 2017, - "call_for_proposals": "https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ", - "end_day": 31, - "end_month": "December", - "end_year": 2017, - "funder": "NumFOCUS", - "narrative": "https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing", - "program": "Small Development Grants", - "team": [{"institution": "University of South Carolina", - "name": "Anthony Scopatz", - "position": "PI"}, - {"institution": "University of South Carolina", - "name": "Aaron Meurer", - "position": "researcher"} - ], - "title": "SymPy 1.1 Release Support"}""" - record = json.loads(raw_json) - validate_schema(record, schemas['grants']) +@pytest.mark.parametrize('key', schemas.keys()) +def test_validation(key): + validate_schema(exemplars[key], schemas[key], (key, )) diff --git a/regolith/validators.py b/regolith/validators.py index 2a2bd70d1..ec5751ae8 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -116,7 +116,12 @@ def ensure_email(email): } -def validate_schema(record, schema, key=None): +# TODO: run this as a lint like thing rather than raising at the first error +def validate_schema(record, schema, keys=()): + print(keys) + print(schema) + print(record) + print('=========================================') if isinstance(record, dict): total_keys = set(schema.keys()) remove_keys = ['required', 'type', 'description'] @@ -129,16 +134,20 @@ def validate_schema(record, schema, key=None): if k not in schema: pass if k not in record and schema[k].get('required', False): - raise ValueError('{} is required'.format(k)) + raise ValueError('{} is required in {}'.format(k, keys)) elif k in record and k in schema: - validate_schema(record[k], schema[k], k) + validate_schema(record[k], schema[k], keys + (k, )) elif isinstance(record, collections.Iterable) and not isinstance(record, str): - for r in record: - validate_schema(r, schema, key) + if isinstance(record[0], dict): + for r in record: + validate_schema(r, schema['type'], keys) + else: + for r in record: + validate_schema(r, schema, keys) else: if not isinstance(record, schema['type']): raise ValueError('Schema expected type: {}, ' 'got type: {} in ' '{{{}:{}}}'.format(type(record), - schema['type'], key, record)) + schema['type'], keys, record)) From 5c2dd0f3f20e643b73266443d22676e21de35667 Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 12 Dec 2017 22:46:53 -0500 Subject: [PATCH 04/29] DOC: add docs --- regolith/schemas/exemplars.py | 1 + regolith/schemas/schemas.py | 1 + regolith/validators.py | 18 +++++++++++++----- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/regolith/schemas/exemplars.py b/regolith/schemas/exemplars.py index 1928698a9..6269f7b63 100644 --- a/regolith/schemas/exemplars.py +++ b/regolith/schemas/exemplars.py @@ -1,3 +1,4 @@ +"""Exemplars of the database schemas""" exemplars = {'abstracts': {'_id': 'Mouginot.Model', 'coauthors': 'P.P.H. Wilson', 'email': 'mouginot@wisc.edu', diff --git a/regolith/schemas/schemas.py b/regolith/schemas/schemas.py index 4b338ed82..cc80d8397 100644 --- a/regolith/schemas/schemas.py +++ b/regolith/schemas/schemas.py @@ -1,3 +1,4 @@ +"""Database schemas""" schemas = { 'grants': { '_id': diff --git a/regolith/validators.py b/regolith/validators.py index ec5751ae8..697c9e7cc 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -1,4 +1,4 @@ -"""Validators and convertors for regolith input.""" +"""Validators and converters for regolith input.""" import collections import os import re @@ -118,10 +118,18 @@ def ensure_email(email): # TODO: run this as a lint like thing rather than raising at the first error def validate_schema(record, schema, keys=()): - print(keys) - print(schema) - print(record) - print('=========================================') + """Validate a database record against a schema + + Parameters + ---------- + record : dict + The database record to be tracked + schema : dict + The schema to validate the record against + keys : tuple + The sequential keys for the data being accessed, used for + printing/debuging errors + """ if isinstance(record, dict): total_keys = set(schema.keys()) remove_keys = ['required', 'type', 'description'] From c684b868180d4d53d3f1a0560ea91814c144ef28 Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 12 Dec 2017 22:48:23 -0500 Subject: [PATCH 05/29] CI: create not install --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b12aa9a10..9d2238bbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ install: - conda update conda # Install dependencies - - conda install -n test --file requirements/run.txt + - conda create -n test --file requirements/run.txt - source activate test - python setup.py install From 4a498b3a0115642a25b5b4b576b995fbc06e5013 Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 12 Dec 2017 22:53:22 -0500 Subject: [PATCH 06/29] MTN: fix run and move the schemas out of folder --- regolith/{schemas => }/exemplars.py | 0 regolith/{schemas => }/schemas.py | 0 regolith/tests/test_validators.py | 4 ++-- requirements/run.txt | 3 ++- 4 files changed, 4 insertions(+), 3 deletions(-) rename regolith/{schemas => }/exemplars.py (100%) rename regolith/{schemas => }/schemas.py (100%) diff --git a/regolith/schemas/exemplars.py b/regolith/exemplars.py similarity index 100% rename from regolith/schemas/exemplars.py rename to regolith/exemplars.py diff --git a/regolith/schemas/schemas.py b/regolith/schemas.py similarity index 100% rename from regolith/schemas/schemas.py rename to regolith/schemas.py diff --git a/regolith/tests/test_validators.py b/regolith/tests/test_validators.py index a6da000a9..851f38add 100644 --- a/regolith/tests/test_validators.py +++ b/regolith/tests/test_validators.py @@ -1,6 +1,6 @@ from regolith.validators import validate_schema -from regolith.schemas.schemas import schemas -from regolith.schemas.exemplars import exemplars +from regolith.schemas import schemas +from regolith.exemplars import exemplars import pytest diff --git a/requirements/run.txt b/requirements/run.txt index 27917a779..49eb843d7 100644 --- a/requirements/run.txt +++ b/requirements/run.txt @@ -1,2 +1,3 @@ python -ruamel.yaml \ No newline at end of file +ruamel.yaml +jinja2 \ No newline at end of file From e2ae6506ac170a9b14af9825f911f0419a7e985d Mon Sep 17 00:00:00 2001 From: christopher Date: Thu, 14 Dec 2017 17:09:30 -0500 Subject: [PATCH 07/29] ENH: validator --- regolith/schemas.py | 348 +++++++++++++++++++++++------------------ regolith/validators.py | 60 ++----- requirements/run.txt | 3 +- 3 files changed, 213 insertions(+), 198 deletions(-) diff --git a/regolith/schemas.py b/regolith/schemas.py index cc80d8397..ad19d4fa7 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -2,399 +2,439 @@ schemas = { 'grants': { '_id': - {'description': 'short represntation, such as this-is-my-name', + {'description': 'short representation, such as this-is-my-name', 'required': True, - 'type': (str, int, float)}, + 'type': ('string', 'integer', 'float')}, 'amount': {'description': 'value of award', 'required': True, - 'type': (int, float)}, + 'type': ('integer', 'float')}, 'begin_day': {'description': 'start day of the grant', 'required': False, - 'type': int}, + 'type': 'integer'}, 'begin_month': {'description': 'start month of the grant', 'required': True, - 'type': str}, + 'type': 'string'}, 'begin_year': {'description': 'start year of the grant', 'required': True, - 'type': int}, + 'type': 'integer'}, 'benefit_of_collaboration': {'description': '', 'required': False, - 'type': str}, + 'type': 'string'}, 'call_for_proposals': {'description': '', 'required': False, - 'type': str}, + 'type': 'string'}, 'currency': {'description': "typically '$' or 'USD'", 'required': False, - 'type': str}, + 'type': 'string'}, 'end_day': {'description': 'end day of the grant', 'required': False, - 'type': (str, int)}, + 'type': ('string', 'integer')}, 'end_month"': {'description': 'end month of the grant', 'required': False, - 'type': str}, + 'type': 'string'}, 'end_year': {'description': 'end year of the grant', 'required': True, - 'type': int}, + 'type': 'integer'}, 'funder': {'description': 'the agency funding the work', 'required': True, - 'type': str}, + 'type': 'string'}, 'grant_id': {'description': 'the identfier for this work', 'required': False, - 'type': str}, + 'type': 'string'}, 'narrative': { 'description': '', 'required': False, - 'type': str}, + 'type': 'string'}, 'program': {'description': 'the program the work was funded under', 'required': True, - 'type': str}, + 'type': 'string'}, 'team': { 'description': 'information about the team members participating ' - 'in the grant.', + 'in the grant.', 'required': True, - 'type': {'cv': {'required': False, 'type': str}, - 'institution': {'required': True, 'type': str}, - 'name': {'required': True, 'type': str}, - 'position': {'required': True, 'type': str}, - 'subaward_amount': { - 'required': False, - 'type': (int, float)}}}, + 'type': 'list', + 'schema': {'type': 'dict', + 'schema': {'cv': {'required': False, 'type': 'string'}, + 'institution': { + 'required': True, + 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'position': { + 'required': True, + 'type': 'string'}, + 'subaward_amount': { + 'required': False, + 'type': ('integer', 'float')}}}}, 'title': {'description': 'actual title of proposal / grant', 'required': True, - 'type': str}}, + 'type': 'string'}}, 'abstracts': { '_id': { 'description': 'Unique identifier for submission. This generally ' - 'includes the author name and part of the title.', + 'includes the author name and part of the title.', 'required': True, - 'type': str}, + 'type': 'string'}, 'coauthors': {'description': 'names of coauthors', 'required': False, - 'type': str}, + 'type': 'string'}, 'email': {'description': 'contact email for the author.', 'required': True, - 'type': str}, + 'type': 'string'}, 'firstname': {'description': 'first name of the author.', 'required': True, - 'type': str}, + 'type': 'string'}, 'institution': {'description': 'name of the inistitution', 'required': True, - 'type': str}, + 'type': 'string'}, 'lastname': {'description': 'last name of the author.', 'required': True, - 'type': str}, + 'type': 'string'}, 'references': { 'description': 'HTML string of reference for the abstract itself', 'required': False, - 'type': str}, + 'type': 'string'}, 'text': {'description': 'HTML string of the abstract.', 'required': True, - 'type': str}, + 'type': 'string'}, 'timestamp': { 'description': 'The time when the abstract was submitted.', 'required': True, - 'type': str}, + 'type': 'string'}, 'title': {'description': 'title of the presentation/paper.', 'required': True, - 'type': str}}, + 'type': 'string'}}, 'blog': { '_id': {'description': 'short represntation, such as this-is-my-title', 'required': True, - 'type': str}, + 'type': 'string'}, 'author': {'description': 'name or AKA of author', 'required': True, - 'type': str}, + 'type': 'string'}, 'day': { 'description': 'Publication day', 'required': True, - 'type': int}, + 'type': 'integer'}, 'month': { 'description': 'Publication month', 'required': True, - 'type': str}, + 'type': 'string'}, 'original': { 'description': 'URL of original post, if this is a repost', 'required': False, - 'type': str}, + 'type': 'string'}, 'post': {'description': 'actual contents of the post', 'required': True, - 'type': str}, + 'type': 'string'}, 'title': {'description': 'full human readable title', 'required': True, - 'type': str}, + 'type': 'string'}, 'year': { 'description': 'Publication year', 'required': True, - 'type': int}}, + 'type': 'integer'}}, 'grades': { '_id': { 'description': 'unique id, typically the ' 'student-assignment-course', 'required': True, - 'type': str}, + 'type': 'string'}, 'assignment': { 'description': 'assignment id', 'required': True, - 'type': str}, - 'course': {'description': 'course id', 'required': True, 'type': str}, + 'type': 'string'}, + 'course': { + 'description': 'course id', + 'required': True, + 'type': 'string'}, 'filename': {'description': 'path to file in store', 'required': False, - 'type': str}, + 'type': 'string'}, 'scores': { 'description': 'the number of points earned on each question', 'required': True, - 'type': (int, float)}, + 'type': ('integer', 'float')}, 'student': { 'description': 'student id', 'required': True, - 'type': str}}, + 'type': 'string'}}, 'assignments': { '_id': { 'description': 'A unique id for the assignment, such a ' - 'HW01-EMCH-558-2016-S', + 'HW01-EMCH-558-2016-S', 'required': True, - 'type': str}, + 'type': 'string'}, 'category': {'description': "such as 'homework' or 'final'", 'required': True, - 'type': str}, + 'type': 'string'}, 'courses': { 'description': 'ids of the courses that have this assignment', 'required': True, - 'type': str}, + 'type': 'string'}, 'file': {'description': 'path to assignment file in store', 'required': False, - 'type': str}, + 'type': 'string'}, 'points': { 'description': 'list of number of points possible for each ' - 'question. Length is the number of questions', + 'question. Length is the number of questions', 'required': True, - 'type': (int, float)}, + 'type': ('integer', 'float')}, 'question': { 'description': 'titles for the questions on this assignment', 'required': False, - 'type': str}, + 'type': 'string'}, 'solution': {'description': 'path to solution file in store', 'required': False, - 'type': str}}, + 'type': 'string'}}, 'projects': {'_id': {'description': 'Unique project identifier.', 'required': True, - 'type': str}, + 'type': 'string'}, 'description': {'description': 'brief project description.', 'required': True, - 'type': str}, + 'type': 'string'}, 'logo': {'description': 'URL to the project logo', 'required': False, - 'type': str}, + 'type': 'string'}, 'name': {'description': 'name of the project.', 'required': True, - 'type': str}, + 'type': 'string'}, 'other': { 'description': 'other information about the project', 'required': False, - 'type': str}, + 'type': 'string'}, 'repo': { 'description': 'URL of the source code repo, if available', 'required': False, - 'type': str}, - # XXX: FIXME + 'type': 'string'}, 'team': { 'description': 'People who are/have been woking on this ' 'project.', 'required': True, - 'type': {'begin_month': {'required': False, 'type': str}, - 'begin_year': {'required': True, 'type': int}, - 'end_month': {'required': False, 'type': str}, - 'end_year': {'required': False, 'type': int}, - 'name': {'required': True, 'type': str}, - 'position': {'required': True, 'type': str}} + 'type': 'list', + 'schema': { + 'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'name': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}} }, 'website': {'description': 'URL of the website.', 'required': True, - 'type': str}}, + 'type': 'string'}}, 'proposals': { '_id': {'description': 'short represntation, such as this-is-my-name', 'required': True, - 'type': (str, int, float)}, + 'type': ('string', 'integer', 'float')}, 'ammount': {'description': 'value of award', 'required': True, - 'type': (int, float)}, + 'type': ('integer', 'float')}, 'authors': {'description': 'other investigator names', 'required': True, - 'type': str}, + 'type': 'string'}, 'currency': {'description': "typically '$' or 'USD'", 'required': True, - 'type': str}, + 'type': 'string'}, 'day': {'description': 'day that the proposal is due', 'required': True, - 'type': int}, + 'type': 'integer'}, 'durration': {'description': 'number of years', 'required': True, - 'type': (int, float)}, + 'type': ('integer', 'float')}, 'month': {'description': 'month that the proposal is due', 'required': True, - 'type': str}, + 'type': 'string'}, 'pi': {'description': 'principal investigator name', 'required': True, - 'type': str}, + 'type': 'string'}, # XXX: FIXME 'pre': {'description': 'Information about the pre-proposal', 'required': False, - 'type': dict}, + 'type': 'dict'}, 'status': {'description': "e.g. 'submitted', 'accepted', 'rejected'", 'required': True, - 'type': str}, + 'type': 'string'}, 'title': {'description': 'actual title of proposal', 'required': True, - 'type': str}, + 'type': 'string'}, 'year': {'description': 'Year that the proposal is due', 'required': True, - 'type': int}}, + 'type': 'integer'}}, 'students': { '_id': {'description': 'short represntation, such as this-is-my-name', 'required': True, - 'type': str}, + 'type': 'string'}, 'aka': {'description': 'list of aliases', 'required': False, - 'type': (list, str)}, + 'type': ('list', 'string'), + 'schema': {'type': 'string'}}, 'email': { 'description': 'email address', 'required': False, - 'type': str}, + 'type': 'string'}, 'university_id': { 'description': 'The university identifier for the student', 'required': False, - 'type': str}}, + 'type': 'string'}}, 'people': {'_id': {'description': 'unique identifier for the group member', 'required': True, - 'type': str}, + 'type': 'string'}, 'active': { 'description': 'If the person is an active member, ' 'default True.', 'required': False, - 'type': bool}, + 'type': 'boolean'}, 'aka': { 'description': 'list of aliases (also-known-as), useful ' 'for identifying the group member in ' 'citations or elsewhere.', 'required': True, - 'type': str}, + 'type': 'string'}, 'avatar': { 'description': 'URL to avatar', 'required': True, - 'type': str}, + 'type': 'string'}, 'bio': {'description': 'short biographical text', 'required': True, - 'type': str}, + 'type': 'string'}, 'collab': { 'description': 'If the person is a collaborator, default ' 'False.', 'required': False, - 'type': bool}, + 'type': 'boolean'}, 'education': { 'description': 'This contains the educational information ' 'for the group member.', 'required': True, - 'type': {'begin_month': {'required': False, 'type': str}, - 'begin_year': {'required': True, 'type': int}, - 'degree': {'required': True, 'type': str}, - 'end_month': {'required': False, 'type': str}, - 'end_year': {'required': True, 'type': int}, - 'gpa': {'required': False, 'type': (float, str)}, - 'institution': {'required': True, 'type': str}, - 'location': {'required': True, 'type': str}, - 'other': {'required': False, 'type': str}} + 'type': 'list', + 'schema': { + 'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'degree': {'required': True, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': True, 'type': 'integer'}, + 'gpa': {'required': False, 'type': ('float', 'string')}, + 'institution': {'required': True, 'type': 'string'}, + 'location': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}} }, 'email': {'description': 'email address of the group member', 'required': False, - 'type': str}, + 'type': 'string'}, 'employment': { 'description': 'Employment information, similar to ' 'educational information.', 'required': True, - 'type': {'begin_month': {'required': False, 'type': str}, - 'begin_year': {'required': True, 'type': int}, - 'end_month': {'required': False, 'type': str}, - 'end_year': {'required': False, 'type': int}, - 'location': {'required': True, 'type': str}, - 'organization': {'required': True, 'type': str}, - 'other': {'required': False, 'type': str}, - 'position': {'required': True, 'type': str}}}, + 'type': 'string', + 'schema': { + 'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'location': {'required': True, 'type': 'string'}, + 'organization': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}}}, 'funding': { 'description': 'Funding and scholarship that the group ' 'member has individually obtained in the ' 'past. **WARNING:** this is not to be ' 'confused with the **grants** collection', 'required': False, - 'type': {'currency': {'required': False, 'type': str}, - 'duration': {'required': False, 'type': str}, - 'month': {'required': False, 'type': str}, - 'name': {'required': True, 'type': str}, - 'value': {'required': True, 'type': (float, int)}, - 'year': {'required': True, 'type': int}} + 'type': 'list', + 'schema': { + 'currency': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'value': { + 'required': True, + 'type': ('float', 'integer')}, + 'year': {'required': True, 'type': 'integer'}} }, 'honors': { 'description': 'Honors that have been awarded to this ' 'group member', 'required': False, - 'type': {'description': {'required': False, 'type': str}, - 'month': {'required': False, 'type': str}, - 'name': {'required': True, 'type': str}, - 'year': {'required': True, 'type': int}} + 'type': 'list', + 'schema': { + 'description': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}} }, 'membership': { 'description': 'Profesional organizations this member is ' 'a part of', 'required': False, - 'type': {'begin_month': {'required': False, 'type': str}, - 'begin_year': {'required': True, 'type': int}, - 'description': {'required': False, 'type': str}, - 'end_month': {'required': False, 'type': str}, - 'end_year': {'required': False, 'type': int}, - 'organization': {'required': True, 'type': str}, - 'position': {'required': True, 'type': str}, - 'website': {'required': False, 'type': str}}}, + 'type': 'list', + 'schema': { + 'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'description': {'required': False, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'organization': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}}}, 'name': {'description': 'Full, canonical name for the person', 'required': True, - 'type': str}, + 'type': 'string'}, 'position': { 'description': 'such as professor, graduate student, ' 'or scientist', 'required': True, - 'type': str}, + 'type': 'string'}, 'service': { 'description': 'Service that this group member has provided', 'required': False, - 'type': {'description': {'required': False, 'type': str}, - 'duration': {'required': False, 'type': str}, - 'month': {'required': False, 'type': str}, - 'name': {'required': True, 'type': str}, - 'year': {'required': True, 'type': int}} + 'type': 'list', + 'schema': { + 'description': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}} }, 'skills': {'description': 'Skill the group member has', 'required': False, - 'type': {'category': {'required': True, 'type': str}, - 'level': {'required': True, 'type': str}, - 'name': {'required': True, 'type': str}} + 'type': 'list', + 'schema': { + 'category': {'required': True, 'type': 'string'}, + 'level': {'required': True, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}} }, 'teaching': { 'description': 'Courses that this group member has ' 'taught, if any', 'required': False, - 'type': {'course': {'required': True, 'type': str}, - 'description': {'required': False, 'type': str}, - 'end_month': {'required': False, 'type': str}, - 'end_year': {'required': False, 'type': int}, - 'materials': {'required': False, 'type': str}, - 'month': {'required': False, 'type': str}, - 'organization': {'required': True, 'type': str}, - 'position': {'required': True, 'type': str}, - 'syllabus': {'required': False, 'type': str}, - 'video': {'required': False, 'type': str}, - 'website': {'required': False, 'type': str}, - 'year': {'required': True, 'type': int}} + 'type': 'list', + 'schema': {'course': {'required': True, 'type': 'string'}, + 'description': { + 'required': False, + 'type': 'string'}, + 'end_month': { + 'required': False, + 'type': 'string'}, + 'end_year': { + 'required': False, + 'type': 'integer'}, + 'materials': { + 'required': False, + 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'organization': { + 'required': True, + 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'syllabus': { + 'required': False, + 'type': 'string'}, + 'video': {'required': False, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}} }, 'title': {'description': 'for example, Dr., etc.', 'required': False, - 'type': str} + 'type': 'string'} } } diff --git a/regolith/validators.py b/regolith/validators.py index 697c9e7cc..28ebdb7ef 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -2,8 +2,11 @@ import collections import os import re + +from cerberus import Validator from getpass import getpass + from regolith.tools import string_types @@ -116,46 +119,17 @@ def ensure_email(email): } -# TODO: run this as a lint like thing rather than raising at the first error -def validate_schema(record, schema, keys=()): - """Validate a database record against a schema - - Parameters - ---------- - record : dict - The database record to be tracked - schema : dict - The schema to validate the record against - keys : tuple - The sequential keys for the data being accessed, used for - printing/debuging errors - """ - if isinstance(record, dict): - total_keys = set(schema.keys()) - remove_keys = ['required', 'type', 'description'] - for k in remove_keys: - if k in total_keys: - total_keys.remove(k) - total_keys.update(set(record.keys())) - - for k in total_keys: - if k not in schema: - pass - if k not in record and schema[k].get('required', False): - raise ValueError('{} is required in {}'.format(k, keys)) - elif k in record and k in schema: - validate_schema(record[k], schema[k], keys + (k, )) - elif isinstance(record, collections.Iterable) and not isinstance(record, - str): - if isinstance(record[0], dict): - for r in record: - validate_schema(r, schema['type'], keys) - else: - for r in record: - validate_schema(r, schema, keys) - else: - if not isinstance(record, schema['type']): - raise ValueError('Schema expected type: {}, ' - 'got type: {} in ' - '{{{}:{}}}'.format(type(record), - schema['type'], keys, record)) +def pop_description(schema): + """Pop description off the schema since Cerberus doesn't support it""" + if isinstance(schema, dict): + schema.pop('description', None) + for v in schema.values(): + pop_description(v) + + +def validate(db, record): + from regolith.schemas import schemas + schema = schemas[db] + pop_description(schema) + v = Validator(schema, allow_unknown=True) + return v.validate(record) diff --git a/requirements/run.txt b/requirements/run.txt index 49eb843d7..3fce50576 100644 --- a/requirements/run.txt +++ b/requirements/run.txt @@ -1,3 +1,4 @@ python ruamel.yaml -jinja2 \ No newline at end of file +jinja2 +cerberus \ No newline at end of file From f8f7930b267944e7f920aca5a490c8c0d6699553 Mon Sep 17 00:00:00 2001 From: christopher Date: Thu, 14 Dec 2017 17:29:25 -0500 Subject: [PATCH 08/29] WIP: finalize schema --- regolith/app.py | 23 +++++++++++++++++++++++ regolith/schemas.py | 6 +++--- regolith/validators.py | 17 ++++++++++++++++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/regolith/app.py b/regolith/app.py index 135a122c3..9416d8cb9 100644 --- a/regolith/app.py +++ b/regolith/app.py @@ -6,6 +6,8 @@ from flask import Flask, abort, request, render_template, redirect, url_for +from regolith.validators import validate + app = Flask('regolith') @@ -59,6 +61,17 @@ def collection_page(dbname, collname): f.write(form['body']) traceback.print_exc() raise + tv, errors = validate(dbname, body) + if not tv: + td = tempfile.TemporaryDirectory() + n = os.path.join(td.name, 'regolith.txt') + print(errors) + print('Writing text file to {}. ' + 'Please try again.'.format(n)) + with open(n) as f: + f.write(form['body']) + raise ValueError('Error while validating the record') + rc.client.update_one(dbname, collname, {'_id': body['_id']}, body) status = 'saved ✓' status_id = str(body['_id']) @@ -75,6 +88,16 @@ def collection_page(dbname, collname): f.write(form['body']) traceback.print_exc() raise + tv, errors = validate(dbname, body) + if not tv: + td = tempfile.TemporaryDirectory() + n = os.path.join(td.name, 'regolith.txt') + print(errors) + print('Writing text file to {}. ' + 'Please try again.'.format(n)) + with open(n) as f: + f.write(form['body']) + raise ValueError('Error while validating the record') try: added = rc.client.insert_one(dbname, collname, body) except Exception: diff --git a/regolith/schemas.py b/regolith/schemas.py index ad19d4fa7..0f596abb0 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -200,7 +200,7 @@ 'other': { 'description': 'other information about the project', 'required': False, - 'type': 'string'}, + 'type': ['list','string']}, 'repo': { 'description': 'URL of the source code repo, if available', 'required': False, @@ -288,7 +288,7 @@ 'for identifying the group member in ' 'citations or elsewhere.', 'required': True, - 'type': 'string'}, + 'type': ['string', 'list']}, 'avatar': { 'description': 'URL to avatar', 'required': True, @@ -324,7 +324,7 @@ 'description': 'Employment information, similar to ' 'educational information.', 'required': True, - 'type': 'string', + 'type': 'list', 'schema': { 'begin_month': {'required': False, 'type': 'string'}, 'begin_year': {'required': True, 'type': 'integer'}, diff --git a/regolith/validators.py b/regolith/validators.py index 28ebdb7ef..fad39b481 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -128,8 +128,23 @@ def pop_description(schema): def validate(db, record): + """Validate a record for a given db + + Parameters + ---------- + db : str + The name of the db in question + record : dict + The record to be validated + + Returns + ------- + bool: + True is valid + + """ from regolith.schemas import schemas schema = schemas[db] pop_description(schema) v = Validator(schema, allow_unknown=True) - return v.validate(record) + return v.validate(record), v.errors From 2969e52f91a48ca4edfd5c7d818d6c2ee88b8f4a Mon Sep 17 00:00:00 2001 From: christopher Date: Sat, 16 Dec 2017 20:20:12 -0500 Subject: [PATCH 09/29] STY: pep8 --- regolith/exemplars.py | 2 +- regolith/schemas.py | 19 ++++++++++++++----- regolith/tests/test_validators.py | 8 ++++---- regolith/validators.py | 9 +++------ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/regolith/exemplars.py b/regolith/exemplars.py index 6269f7b63..df7e42b7a 100644 --- a/regolith/exemplars.py +++ b/regolith/exemplars.py @@ -1,5 +1,5 @@ """Exemplars of the database schemas""" -exemplars = {'abstracts': {'_id': 'Mouginot.Model', +EXEMPLARS = {'abstracts': {'_id': 'Mouginot.Model', 'coauthors': 'P.P.H. Wilson', 'email': 'mouginot@wisc.edu', 'firstname': 'Baptiste', diff --git a/regolith/schemas.py b/regolith/schemas.py index 0f596abb0..cab4432c1 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -1,5 +1,5 @@ """Database schemas""" -schemas = { +SCHEMAS = { 'grants': { '_id': {'description': 'short representation, such as this-is-my-name', @@ -67,7 +67,12 @@ 'type': ('integer', 'float')}}}}, 'title': {'description': 'actual title of proposal / grant', 'required': True, - 'type': 'string'}}, + 'type': 'string'}, + 'status': {'description': 'status of the grant', + 'required': True, + 'type': 'string', + 'allowed': ['pending', 'declined', 'accepted', 'in-prep']} + }, 'abstracts': { '_id': { 'description': 'Unique identifier for submission. This generally ' @@ -200,9 +205,10 @@ 'other': { 'description': 'other information about the project', 'required': False, - 'type': ['list','string']}, + 'type': ['list', 'string']}, 'repo': { - 'description': 'URL of the source code repo, if available', + 'description': 'URL of the source code repo, if ' + 'available', 'required': False, 'type': 'string'}, 'team': { @@ -220,7 +226,10 @@ }, 'website': {'description': 'URL of the website.', 'required': True, - 'type': 'string'}}, + 'type': 'string'}, + 'grant': {'description': 'Grant id if there is a grant ' + 'supporting this project', + 'type': 'string'}}, 'proposals': { '_id': {'description': 'short represntation, such as this-is-my-name', 'required': True, diff --git a/regolith/tests/test_validators.py b/regolith/tests/test_validators.py index 851f38add..f8d8b0af6 100644 --- a/regolith/tests/test_validators.py +++ b/regolith/tests/test_validators.py @@ -1,11 +1,11 @@ from regolith.validators import validate_schema -from regolith.schemas import schemas -from regolith.exemplars import exemplars +from regolith.schemas import SCHEMAS +from regolith.exemplars import EXEMPLARS import pytest -@pytest.mark.parametrize('key', schemas.keys()) +@pytest.mark.parametrize('key', SCHEMAS.keys()) def test_validation(key): - validate_schema(exemplars[key], schemas[key], (key, )) + validate_schema(EXEMPLARS[key], SCHEMAS[key], (key,)) diff --git a/regolith/validators.py b/regolith/validators.py index fad39b481..f01bee290 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -1,11 +1,8 @@ """Validators and converters for regolith input.""" -import collections import os -import re - -from cerberus import Validator from getpass import getpass +from cerberus import Validator from regolith.tools import string_types @@ -143,8 +140,8 @@ def validate(db, record): True is valid """ - from regolith.schemas import schemas - schema = schemas[db] + from regolith.schemas import SCHEMAS + schema = SCHEMAS[db] pop_description(schema) v = Validator(schema, allow_unknown=True) return v.validate(record), v.errors From e0f26bddfe24ad3c4b8dc4a4fbc236c3a1a77989 Mon Sep 17 00:00:00 2001 From: christopher Date: Sat, 16 Dec 2017 20:26:11 -0500 Subject: [PATCH 10/29] STY: pep8 import --- regolith/tests/test_validators.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/regolith/tests/test_validators.py b/regolith/tests/test_validators.py index f8d8b0af6..f92cdcf6d 100644 --- a/regolith/tests/test_validators.py +++ b/regolith/tests/test_validators.py @@ -1,11 +1,12 @@ -from regolith.validators import validate_schema -from regolith.schemas import SCHEMAS -from regolith.exemplars import EXEMPLARS import pytest +from regolith.exemplars import EXEMPLARS +from regolith.schemas import SCHEMAS +from regolith.validators import validate + @pytest.mark.parametrize('key', SCHEMAS.keys()) def test_validation(key): - validate_schema(EXEMPLARS[key], SCHEMAS[key], (key,)) + validate(key, EXEMPLARS[key]) From 5881956a0550440f71e655b9b5b54468b89e0286 Mon Sep 17 00:00:00 2001 From: christopher Date: Sat, 16 Dec 2017 20:42:42 -0500 Subject: [PATCH 11/29] MTN: move schema things to schemas --- regolith/app.py | 2 +- regolith/exemplars.py | 281 -------------------------- regolith/schemas.py | 318 +++++++++++++++++++++++++++++- regolith/tests/test_validators.py | 4 +- regolith/validators.py | 33 ---- 5 files changed, 319 insertions(+), 319 deletions(-) delete mode 100644 regolith/exemplars.py diff --git a/regolith/app.py b/regolith/app.py index 9416d8cb9..aea5bb714 100644 --- a/regolith/app.py +++ b/regolith/app.py @@ -6,7 +6,7 @@ from flask import Flask, abort, request, render_template, redirect, url_for -from regolith.validators import validate +from regolith.schemas import validate app = Flask('regolith') diff --git a/regolith/exemplars.py b/regolith/exemplars.py deleted file mode 100644 index df7e42b7a..000000000 --- a/regolith/exemplars.py +++ /dev/null @@ -1,281 +0,0 @@ -"""Exemplars of the database schemas""" -EXEMPLARS = {'abstracts': {'_id': 'Mouginot.Model', - 'coauthors': 'P.P.H. Wilson', - 'email': 'mouginot@wisc.edu', - 'firstname': 'Baptiste', - 'institution': 'University of Wisconsin-Madison', - 'lastname': 'Mouginot', - 'references': '[1] B. MOUGINOT, “cyCLASS: CLASS models for Cyclus,”, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 (2016).', - 'text': 'The CLASS team has developed high quality predictors based on pre-trained neural network...', - 'timestamp': '5/5/2017 13:15:59', - 'title': 'Model Performance Analysis'}, - 'assignments': {'_id': 'hw01-rx-power', - 'category': 'homework', - 'courses': ['EMCH-558-2016-S', - 'EMCH-758-2016-S'], - 'points': [1, 2, 3], - 'questions': ['1-9', '1-10', '1-12']}, - 'blog': {'_id': 'my-vision', - 'author': 'Anthony Scopatz', - 'day': 18, - 'month': 'September', - 'original': 'https://scopatz.com/my-vision/', - 'post': 'I would like see things move forward. Deep, I know!', - 'title': 'My Vision', - 'year': 2015}, - 'citations': {'_id': 'meurer2016sympy', - 'author': ['Meurer, Aaron', - 'Smith, Christopher P', - 'Paprocki, Mateusz', - "{\\v{C}}ert{\\'\\i}k, Ond{\\v{r}}ej", - 'Rocklin, Matthew', - 'Kumar, AMiT', - 'Ivanov, Sergiu', - 'Moore, Jason K', - 'Singh, Sartaj', - 'Rathnayake, Thilina', - 'Sean Vig', - 'Brian E Granger', - 'Richard P Muller', - 'Francesco Bonazzi', - 'Harsh Gupta', - 'Shivam Vats', - 'Fredrik Johansson', - 'Fabian Pedregosa', - 'Matthew J Curry', - 'Ashutosh Saboo', - 'Isuru Fernando', - 'Sumith Kulal', - 'Robert Cimrman', - 'Anthony Scopatz'], - 'entrytype': 'article', - 'journal': 'PeerJ Computer Science', - 'month': 'Jan', - 'pages': 'e103', - 'publisher': 'PeerJ Inc. San Francisco, USA', - 'title': 'SymPy: Symbolic computing in Python', - 'volume': '4', - 'year': '2017'}, - 'courses': {'_id': 'EMCH-552-2016-F', - 'active': False, - 'department': 'EMCH', - 'number': 552, - 'scale': [[0.875, 'A'], - [0.8125, 'B+'], - [0.75, 'B'], - [0.6875, 'C+'], - [0.625, 'C'], - [0.5625, 'D+'], - [0.5, 'D'], - [-1.0, 'F']], - 'season': 'F', - 'students': ['Human A. Person', 'Human B. Person'], - 'syllabus': 'emch552-2016-f-syllabus.pdf', - 'weights': {'class-notes': 0.15, - 'final': 0.3, - 'homework': 0.35, - 'midterm': 0.2}, - 'year': 2016}, - 'grades': { - '_id': 'Human A. Person-rx-power-hw02-EMCH-758-2017-S', - 'student': 'hap', - 'assignment': '2017-rx-power-hw02', - 'course': 'EMCH-758-2017-S', - 'scores': [1, 1.6, 3]}, - 'grants': {'_id': 'SymPy-1.1', - 'amount': 3000.0, - 'begin_day': 1, - 'begin_month': 'May', - 'begin_year': 2017, - 'call_for_proposals': 'https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ', - 'end_day': 31, - 'end_month': 'December', - 'end_year': 2017, - 'funder': 'NumFOCUS', - 'narrative': 'https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing', - 'program': 'Small Development Grants', - 'team': [ - {'institution': 'University of South Carolina', - 'name': 'Anthony Scopatz', - 'position': 'PI'}, - {'institution': 'University of South Carolina', - 'name': 'Aaron Meurer', - 'position': 'researcher'}], - 'title': 'SymPy 1.1 Release Support'}, - 'jobs': {'_id': '0004', - 'background_fields': ['Data Science', - 'Data Engineering', - 'Computer Engineering', - 'Computer Science', - 'Applied Mathematics', - 'Physics', - 'Nuclear Engineering', - 'Mechanical Engineering', - 'Or similar'], - 'compensation': [ - 'Salary and compensation will be based on prior work experience.'], - 'contact': 'Please send CV or resume to Prof. Scopatz at scopatzATcec.sc.edu.', - 'day': 1, - 'description': '

We are seeking a dedicated individual to help to aid in ...', - 'month': 'July', - 'open': False, - 'positions': ['Scientific Software Developer', - 'Programmer'], - 'start_date': 'ASAP', - 'title': 'Open Source Scientific Software Maintainer', - 'year': 2015}, - 'news': {'_id': '56b4eb6d421aa921504ef2a9', - 'author': 'Anthony Scopatz', - 'body': 'Dr. Robert Flanagan joined ERGS as a post-doctoral scholar.', - 'day': 1, - 'month': 'February', - 'year': 2016}, - 'people': {'_id': 'scopatz', - 'aka': ['Scopatz', - 'Scopatz, A', - 'Scopatz, A.', - 'Scopatz, A M', - 'Anthony Michael Scopatz'], - 'avatar': 'https://avatars1.githubusercontent.com/u/320553?v=3&s=200', - 'bio': 'Anthony Scopatz is currently an Assistant Professor', - 'education': [{'begin_year': 2008, - 'degree': 'Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program', - 'end_year': 2011, - 'institution': 'The University of Texas at Austin', - 'location': 'Austin, TX', - 'other': [ - 'Adviser: Erich A. Schneider', - 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis']}, - {'begin_year': 2006, - 'degree': 'M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program', - 'end_year': 2007, - 'institution': 'The University of Texas at Austin', - 'location': 'Austin, TX', - 'other': [ - 'Adviser: Erich A. Schneider', - 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership']}, - {'begin_year': 2002, - 'degree': 'B.S. Physics', - 'end_year': 2006, - 'institution': 'University of California, Santa Barbara', - 'location': 'Santa Barbara, CA', - 'other': [ - 'Graduated with a Major in Physics and a Minor in Mathematics']}], - 'email': 'scopatz@cec.sc.edu', - 'employment': [{'begin_year': 2015, - 'location': 'Columbia, SC', - 'organization': 'The University of South Carolina', - 'other': [ - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.', - 'PyNE: The Nuclear Engineering Toolkit.', - 'Website: http://www.ergs.sc.edu/'], - 'position': 'Assistant Professor, Mechanical Engineering Department'}, - {'begin_year': 2013, - 'end_year': 2015, - 'location': 'Madison, WI', - 'organization': 'CNERG, The University of Wisconsin-Madison', - 'other': [ - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.', - 'PyNE: The Nuclear Engineering Toolkit.', - 'Website: https://cnerg.github.io/'], - 'position': 'Associate Scientist, Engineering Physics Department'}, - {'begin_month': 'Nov', - 'begin_year': 2011, - 'end_month': 'May', - 'end_year': 2013, - 'location': 'Chicago, IL', - 'organization': 'The FLASH Center, The University of Chicago', - 'other': [ - 'NIF: Simulation of magnetic field generation from neutral plasmas using FLASH.', - 'CosmoB: Simulation of magnetic field generation from neutral plasmas using FLASH.', - 'FLASH4: High-energy density physics capabilities and utilities.', - 'Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from FLASH.', - 'OpacPlot: HDF5-based equation of state and opacity file format.', - 'Website: http://flash.uchicago.edu/site/'], - 'position': 'Research Scientist, Postdoctoral Scholar'}], - 'funding': [ - {'name': "Omega Laser User's Group Travel Award", - 'value': 1100, - 'year': 2013}, - { - 'name': "NIF User's Group Travel Award", - 'value': 1150, - 'year': 2013}], - 'membership': [{'begin_year': 2006, - 'organization': 'American Nuclear Society', - 'position': 'Member'}, - {'begin_year': 2013, - 'organization': 'Python Software Foundation', - 'position': 'Fellow'}], - 'name': 'Anthony Scopatz', - 'position': 'professor', - 'skills': [{'category': 'Programming Languages', - 'level': 'expert', - 'name': 'Python'}, - { - 'category': 'Programming Languages', - 'level': 'expert', - 'name': 'Cython'}], - 'teaching': [{ - 'course': 'EMCH 552: Intro to Nuclear Engineering', - 'description': 'This course is an introduction to nuclear physics.', - 'month': 'August', - 'organization': 'University of South Carolina', - 'position': 'Professor', - 'syllabus': 'https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc', - 'year': 2017}, - { - 'course': 'EMCH 558/758: Reactor Power Systems', - 'description': 'This course covers conventional reactors.', - 'month': 'January', - 'organization': 'University of South Carolina', - 'position': 'Professor', - 'syllabus': 'https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing', - 'year': 2017}], - 'title': 'Dr.'}, - 'projects': {'_id': 'Cyclus', - 'name': 'Cyclus', - 'description': 'Agent-Based Nuclear Fuel Cycle Simulator', - 'logo': 'http://fuelcycle.org/_static/big_c.png', - 'other': [ - 'Discrete facilities with discrete material transactions', - 'Low barrier to entry, rapid payback to adoption'], - 'repo': 'https://github.com/cyclus/cyclus/', - 'team': [{'begin_month': 'June', - 'begin_year': 2013, - 'end_month': 'July', - 'end_year': 2015, - 'name': 'Anthony Scopatz', - 'position': 'Project Lead'}], - 'website': 'http://fuelcycle.org/'}, - 'proposals': {'_id': 'mypropsal', - 'ammount': 1000000.0, - 'authors': ['Anthony Scopatz', - 'Robert Flanagan'], - 'currency': 'USD', - 'day': 18, - 'durration': 3, - 'full': { - 'benefit_of_collaboration': 'http://pdf.com/benefit_of_collaboration', - 'cv': [ - 'http://pdf.com/scopatz-cv', - 'http://pdf.com/flanagan-cv'], - 'narrative': 'http://some.com/pdf'}, - 'month': 'Aug', - 'pi': 'Anthony Scopatz', - 'pre': { - 'benefit_of_collaboration': 'http://pdf.com/benefit_of_collaboration', - 'cv': [ - 'http://pdf.com/scopatz-cv', - 'http://pdf.com/flanagan-cv'], - 'day': 2, - 'month': 'Aug', - 'narrative': 'http://some.com/pdf', - 'year': 1998}, - 'status': 'submitted', - 'title': 'A very fine proposal indeed', - 'year': 1999}, - 'students': {'_id': 'Human A. Person', - 'aka': ['H. A. Person'], - 'email': 'haperson@uni.edu', - 'university_id': 'HAP42'}} diff --git a/regolith/schemas.py b/regolith/schemas.py index cab4432c1..3b1d4909c 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -1,4 +1,6 @@ -"""Database schemas""" +"""Database schemas, examples, and tools""" +from cerberus import Validator + SCHEMAS = { 'grants': { '_id': @@ -447,3 +449,317 @@ 'type': 'string'} } } + + +def pop_description(schema): + """Pop description off the schema since Cerberus doesn't support it""" + if isinstance(schema, dict): + schema.pop('description', None) + for v in schema.values(): + pop_description(v) + + +def validate(db, record): + """Validate a record for a given db + + Parameters + ---------- + db : str + The name of the db in question + record : dict + The record to be validated + + Returns + ------- + rtn : bool + True is valid + errors: dict + The errors encountered (if any) + + """ + schema = SCHEMAS[db] + pop_description(schema) + v = Validator(schema, allow_unknown=True) + return v.validate(record), v.errors + + +EXEMPLARS = {'abstracts': {'_id': 'Mouginot.Model', + 'coauthors': 'P.P.H. Wilson', + 'email': 'mouginot@wisc.edu', + 'firstname': 'Baptiste', + 'institution': 'University of Wisconsin-Madison', + 'lastname': 'Mouginot', + 'references': '[1] B. MOUGINOT, “cyCLASS: CLASS models for Cyclus,”, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 (2016).', + 'text': 'The CLASS team has developed high quality predictors based on pre-trained neural network...', + 'timestamp': '5/5/2017 13:15:59', + 'title': 'Model Performance Analysis'}, + 'assignments': {'_id': 'hw01-rx-power', + 'category': 'homework', + 'courses': ['EMCH-558-2016-S', + 'EMCH-758-2016-S'], + 'points': [1, 2, 3], + 'questions': ['1-9', '1-10', '1-12']}, + 'blog': {'_id': 'my-vision', + 'author': 'Anthony Scopatz', + 'day': 18, + 'month': 'September', + 'original': 'https://scopatz.com/my-vision/', + 'post': 'I would like see things move forward. Deep, I know!', + 'title': 'My Vision', + 'year': 2015}, + 'citations': {'_id': 'meurer2016sympy', + 'author': ['Meurer, Aaron', + 'Smith, Christopher P', + 'Paprocki, Mateusz', + "{\\v{C}}ert{\\'\\i}k, Ond{\\v{r}}ej", + 'Rocklin, Matthew', + 'Kumar, AMiT', + 'Ivanov, Sergiu', + 'Moore, Jason K', + 'Singh, Sartaj', + 'Rathnayake, Thilina', + 'Sean Vig', + 'Brian E Granger', + 'Richard P Muller', + 'Francesco Bonazzi', + 'Harsh Gupta', + 'Shivam Vats', + 'Fredrik Johansson', + 'Fabian Pedregosa', + 'Matthew J Curry', + 'Ashutosh Saboo', + 'Isuru Fernando', + 'Sumith Kulal', + 'Robert Cimrman', + 'Anthony Scopatz'], + 'entrytype': 'article', + 'journal': 'PeerJ Computer Science', + 'month': 'Jan', + 'pages': 'e103', + 'publisher': 'PeerJ Inc. San Francisco, USA', + 'title': 'SymPy: Symbolic computing in Python', + 'volume': '4', + 'year': '2017'}, + 'courses': {'_id': 'EMCH-552-2016-F', + 'active': False, + 'department': 'EMCH', + 'number': 552, + 'scale': [[0.875, 'A'], + [0.8125, 'B+'], + [0.75, 'B'], + [0.6875, 'C+'], + [0.625, 'C'], + [0.5625, 'D+'], + [0.5, 'D'], + [-1.0, 'F']], + 'season': 'F', + 'students': ['Human A. Person', 'Human B. Person'], + 'syllabus': 'emch552-2016-f-syllabus.pdf', + 'weights': {'class-notes': 0.15, + 'final': 0.3, + 'homework': 0.35, + 'midterm': 0.2}, + 'year': 2016}, + 'grades': { + '_id': 'Human A. Person-rx-power-hw02-EMCH-758-2017-S', + 'student': 'hap', + 'assignment': '2017-rx-power-hw02', + 'course': 'EMCH-758-2017-S', + 'scores': [1, 1.6, 3]}, + 'grants': {'_id': 'SymPy-1.1', + 'amount': 3000.0, + 'begin_day': 1, + 'begin_month': 'May', + 'begin_year': 2017, + 'call_for_proposals': 'https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ', + 'end_day': 31, + 'end_month': 'December', + 'end_year': 2017, + 'funder': 'NumFOCUS', + 'narrative': 'https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing', + 'program': 'Small Development Grants', + 'team': [ + {'institution': 'University of South Carolina', + 'name': 'Anthony Scopatz', + 'position': 'PI'}, + {'institution': 'University of South Carolina', + 'name': 'Aaron Meurer', + 'position': 'researcher'}], + 'title': 'SymPy 1.1 Release Support'}, + 'jobs': {'_id': '0004', + 'background_fields': ['Data Science', + 'Data Engineering', + 'Computer Engineering', + 'Computer Science', + 'Applied Mathematics', + 'Physics', + 'Nuclear Engineering', + 'Mechanical Engineering', + 'Or similar'], + 'compensation': [ + 'Salary and compensation will be based on prior work experience.'], + 'contact': 'Please send CV or resume to Prof. Scopatz at scopatzATcec.sc.edu.', + 'day': 1, + 'description': '

We are seeking a dedicated individual to help to aid in ...', + 'month': 'July', + 'open': False, + 'positions': ['Scientific Software Developer', + 'Programmer'], + 'start_date': 'ASAP', + 'title': 'Open Source Scientific Software Maintainer', + 'year': 2015}, + 'news': {'_id': '56b4eb6d421aa921504ef2a9', + 'author': 'Anthony Scopatz', + 'body': 'Dr. Robert Flanagan joined ERGS as a post-doctoral scholar.', + 'day': 1, + 'month': 'February', + 'year': 2016}, + 'people': {'_id': 'scopatz', + 'aka': ['Scopatz', + 'Scopatz, A', + 'Scopatz, A.', + 'Scopatz, A M', + 'Anthony Michael Scopatz'], + 'avatar': 'https://avatars1.githubusercontent.com/u/320553?v=3&s=200', + 'bio': 'Anthony Scopatz is currently an Assistant Professor', + 'education': [{'begin_year': 2008, + 'degree': 'Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program', + 'end_year': 2011, + 'institution': 'The University of Texas at Austin', + 'location': 'Austin, TX', + 'other': [ + 'Adviser: Erich A. Schneider', + 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis']}, + {'begin_year': 2006, + 'degree': 'M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program', + 'end_year': 2007, + 'institution': 'The University of Texas at Austin', + 'location': 'Austin, TX', + 'other': [ + 'Adviser: Erich A. Schneider', + 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership']}, + {'begin_year': 2002, + 'degree': 'B.S. Physics', + 'end_year': 2006, + 'institution': 'University of California, Santa Barbara', + 'location': 'Santa Barbara, CA', + 'other': [ + 'Graduated with a Major in Physics and a Minor in Mathematics']}], + 'email': 'scopatz@cec.sc.edu', + 'employment': [{'begin_year': 2015, + 'location': 'Columbia, SC', + 'organization': 'The University of South Carolina', + 'other': [ + 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.', + 'PyNE: The Nuclear Engineering Toolkit.', + 'Website: http://www.ergs.sc.edu/'], + 'position': 'Assistant Professor, Mechanical Engineering Department'}, + {'begin_year': 2013, + 'end_year': 2015, + 'location': 'Madison, WI', + 'organization': 'CNERG, The University of Wisconsin-Madison', + 'other': [ + 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.', + 'PyNE: The Nuclear Engineering Toolkit.', + 'Website: https://cnerg.github.io/'], + 'position': 'Associate Scientist, Engineering Physics Department'}, + {'begin_month': 'Nov', + 'begin_year': 2011, + 'end_month': 'May', + 'end_year': 2013, + 'location': 'Chicago, IL', + 'organization': 'The FLASH Center, The University of Chicago', + 'other': [ + 'NIF: Simulation of magnetic field generation from neutral plasmas using FLASH.', + 'CosmoB: Simulation of magnetic field generation from neutral plasmas using FLASH.', + 'FLASH4: High-energy density physics capabilities and utilities.', + 'Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from FLASH.', + 'OpacPlot: HDF5-based equation of state and opacity file format.', + 'Website: http://flash.uchicago.edu/site/'], + 'position': 'Research Scientist, Postdoctoral Scholar'}], + 'funding': [ + {'name': "Omega Laser User's Group Travel Award", + 'value': 1100, + 'year': 2013}, + { + 'name': "NIF User's Group Travel Award", + 'value': 1150, + 'year': 2013}], + 'membership': [{'begin_year': 2006, + 'organization': 'American Nuclear Society', + 'position': 'Member'}, + {'begin_year': 2013, + 'organization': 'Python Software Foundation', + 'position': 'Fellow'}], + 'name': 'Anthony Scopatz', + 'position': 'professor', + 'skills': [{'category': 'Programming Languages', + 'level': 'expert', + 'name': 'Python'}, + { + 'category': 'Programming Languages', + 'level': 'expert', + 'name': 'Cython'}], + 'teaching': [{ + 'course': 'EMCH 552: Intro to Nuclear Engineering', + 'description': 'This course is an introduction to nuclear physics.', + 'month': 'August', + 'organization': 'University of South Carolina', + 'position': 'Professor', + 'syllabus': 'https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc', + 'year': 2017}, + { + 'course': 'EMCH 558/758: Reactor Power Systems', + 'description': 'This course covers conventional reactors.', + 'month': 'January', + 'organization': 'University of South Carolina', + 'position': 'Professor', + 'syllabus': 'https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing', + 'year': 2017}], + 'title': 'Dr.'}, + 'projects': {'_id': 'Cyclus', + 'name': 'Cyclus', + 'description': 'Agent-Based Nuclear Fuel Cycle Simulator', + 'logo': 'http://fuelcycle.org/_static/big_c.png', + 'other': [ + 'Discrete facilities with discrete material transactions', + 'Low barrier to entry, rapid payback to adoption'], + 'repo': 'https://github.com/cyclus/cyclus/', + 'team': [{'begin_month': 'June', + 'begin_year': 2013, + 'end_month': 'July', + 'end_year': 2015, + 'name': 'Anthony Scopatz', + 'position': 'Project Lead'}], + 'website': 'http://fuelcycle.org/'}, + 'proposals': {'_id': 'mypropsal', + 'ammount': 1000000.0, + 'authors': ['Anthony Scopatz', + 'Robert Flanagan'], + 'currency': 'USD', + 'day': 18, + 'durration': 3, + 'full': { + 'benefit_of_collaboration': 'http://pdf.com/benefit_of_collaboration', + 'cv': [ + 'http://pdf.com/scopatz-cv', + 'http://pdf.com/flanagan-cv'], + 'narrative': 'http://some.com/pdf'}, + 'month': 'Aug', + 'pi': 'Anthony Scopatz', + 'pre': { + 'benefit_of_collaboration': 'http://pdf.com/benefit_of_collaboration', + 'cv': [ + 'http://pdf.com/scopatz-cv', + 'http://pdf.com/flanagan-cv'], + 'day': 2, + 'month': 'Aug', + 'narrative': 'http://some.com/pdf', + 'year': 1998}, + 'status': 'submitted', + 'title': 'A very fine proposal indeed', + 'year': 1999}, + 'students': {'_id': 'Human A. Person', + 'aka': ['H. A. Person'], + 'email': 'haperson@uni.edu', + 'university_id': 'HAP42'}} \ No newline at end of file diff --git a/regolith/tests/test_validators.py b/regolith/tests/test_validators.py index f92cdcf6d..8692a8241 100644 --- a/regolith/tests/test_validators.py +++ b/regolith/tests/test_validators.py @@ -1,8 +1,6 @@ import pytest -from regolith.exemplars import EXEMPLARS -from regolith.schemas import SCHEMAS -from regolith.validators import validate +from regolith.schemas import SCHEMAS, validate, EXEMPLARS @pytest.mark.parametrize('key', SCHEMAS.keys()) diff --git a/regolith/validators.py b/regolith/validators.py index f01bee290..fe527d890 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -2,8 +2,6 @@ import os from getpass import getpass -from cerberus import Validator - from regolith.tools import string_types @@ -114,34 +112,3 @@ def ensure_email(email): 'stores': (always_false, ensure_stores), 'email': (always_false, ensure_email), } - - -def pop_description(schema): - """Pop description off the schema since Cerberus doesn't support it""" - if isinstance(schema, dict): - schema.pop('description', None) - for v in schema.values(): - pop_description(v) - - -def validate(db, record): - """Validate a record for a given db - - Parameters - ---------- - db : str - The name of the db in question - record : dict - The record to be validated - - Returns - ------- - bool: - True is valid - - """ - from regolith.schemas import SCHEMAS - schema = SCHEMAS[db] - pop_description(schema) - v = Validator(schema, allow_unknown=True) - return v.validate(record), v.errors From 986d5fa69c2268b42881713f8f5255b000b5ad01 Mon Sep 17 00:00:00 2001 From: christopher Date: Sat, 16 Dec 2017 20:42:56 -0500 Subject: [PATCH 12/29] STY: newline --- regolith/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regolith/schemas.py b/regolith/schemas.py index 3b1d4909c..5108f96d3 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -762,4 +762,4 @@ def validate(db, record): 'students': {'_id': 'Human A. Person', 'aka': ['H. A. Person'], 'email': 'haperson@uni.edu', - 'university_id': 'HAP42'}} \ No newline at end of file + 'university_id': 'HAP42'}} From 4dc419cc5d618fb6a0aae8088d4dc843ff4d5f29 Mon Sep 17 00:00:00 2001 From: christopher Date: Sat, 16 Dec 2017 21:22:05 -0500 Subject: [PATCH 13/29] FIX: don't allow unknown --- regolith/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regolith/schemas.py b/regolith/schemas.py index 5108f96d3..cdc823e8b 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -479,7 +479,7 @@ def validate(db, record): """ schema = SCHEMAS[db] pop_description(schema) - v = Validator(schema, allow_unknown=True) + v = Validator(schema) return v.validate(record), v.errors From f7bcfb1d4e0bd1d4f5f0e29e2abba25f523752f0 Mon Sep 17 00:00:00 2001 From: christopher Date: Sat, 16 Dec 2017 21:47:31 -0500 Subject: [PATCH 14/29] FIX: custom validator for description --- regolith/schemas.py | 9 +++++++-- regolith/tests/test_validators.py | 2 -- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/regolith/schemas.py b/regolith/schemas.py index cdc823e8b..e715f55e0 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -459,6 +459,12 @@ def pop_description(schema): pop_description(v) +class MyValidator(Validator): + def _validate_description(self, description, field, value): + if False: + self._error(field, "Shouldn't be here") + + def validate(db, record): """Validate a record for a given db @@ -478,8 +484,7 @@ def validate(db, record): """ schema = SCHEMAS[db] - pop_description(schema) - v = Validator(schema) + v = MyValidator(schema) return v.validate(record), v.errors diff --git a/regolith/tests/test_validators.py b/regolith/tests/test_validators.py index 8692a8241..85e8c4590 100644 --- a/regolith/tests/test_validators.py +++ b/regolith/tests/test_validators.py @@ -6,5 +6,3 @@ @pytest.mark.parametrize('key', SCHEMAS.keys()) def test_validation(key): validate(key, EXEMPLARS[key]) - - From 6068349152bb14c62a9948c334608c93cb29ad07 Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 14:16:51 -0500 Subject: [PATCH 15/29] WIP: write docs from schema/exemplars --- docs/conf.py | 62 ++++++++++++++++++++++++++++++++++++++++++++- regolith/schemas.py | 8 ------ 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 79d263fe6..612618864 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -102,7 +102,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'redcloud' -#html_theme = 'blackcloud' +# html_theme = 'blackcloud' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -253,3 +253,63 @@ # How to display URL addresses: 'footnote', 'no', or 'inline'. # texinfo_show_urls = 'footnote' + +schema_top_docs = { + 'abstracts': 'Abstracts for a conference or workshop. This is generally public information\n\n', + 'assignments': 'Information about assignments for classes.\n\n', + 'blank': '\n', + 'blog': 'This collection represents blog posts written by the members of the research group.\n\n', + 'citations': 'This collection should contain bibtex equivalent fields. Additionally,\nthe keys ``"entrytype"`` denotes things like ``ARTICLE``, and ``"_id"`` denotes\nthe entry identifier. Furthermore, the ``"author"`` key should be a list of\nstrings. See the Python project `BibtexParser `_\nfor more information.\n\n', + 'courses': 'This is a collection that describes a course, when it happened, and\nwho is taking it. This is likely private.\n\n', + 'grades': 'The grade for a student on an assignment. This information should be private.\n\n', + 'grants': 'This collection represents grants that have been awarded to the group.\n\n', + 'jobs': 'This collection describes the research group jobs. This is normally public data.\n\n\n', + 'news': 'This collection describes the research group news. This is normally public data.\n\n', + 'people': 'This collection describes the members of the research group. This is normally public\ndata.\n\n', + 'projects': 'This collection describes the research group projects. This is normally public data.\n\n', + 'proposals': 'This collection represents proposals that have been submitted by the group.\n\n', + 'students': 'This is a collection of student names and metadata. This should probably be private.\n\n'} + +from regolith.schemas import SCHEMAS, EXEMPLARS +from regolith.fsclient import json_to_yaml +import json +from io import StringIO + +def format_key(schema, key, indent=''): + s = '' + line_format = ':{key}: {type}, {description}\n' + s += line_format.format( + key=schema[key], + type=schema[key]['type]'], + description=schema[key]['description']) + if 'schema' in schema: + for key in schema['schema']: + s += format_key(schema['schema'], key, indent=indent+'\t') + return s + + +def build_schema_doc(key): + fn = key + '_test.rst' + with open(fn, 'w') as f: + s = '' + s += key + s += '=' * len(key) + s += 'Schema\n------\nThe following lists key names mapped to its type and meaning for each entry.\n\n' + schema = SCHEMAS[key] + line_format = ':{key}: {type}, {description}\n' + for k in schema: + s += line_format.format(key=k, type=schema[k]['type'], + description=schema[k]['description']) + s += '\n\n' + s += 'YAML Example\n------------\n\n' + s += '.. code-block:: yaml\n\n' + jd = json.dumps(EXEMPLARS[key]) + sio = StringIO() + json_to_yaml(jd, sio) + s += sio.getvalue() + s += '\n\n' + s += 'JSON/Mongo Example\n------------------\n\n' + s += '.. code-block:: json\n\n' + s += jd + s += '\n' + f.write(s) \ No newline at end of file diff --git a/regolith/schemas.py b/regolith/schemas.py index e715f55e0..9efe10ae5 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -451,14 +451,6 @@ } -def pop_description(schema): - """Pop description off the schema since Cerberus doesn't support it""" - if isinstance(schema, dict): - schema.pop('description', None) - for v in schema.values(): - pop_description(v) - - class MyValidator(Validator): def _validate_description(self, description, field, value): if False: From 263441b40eb9828c48f115cbcac50e64ad982f4e Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 15:35:44 -0500 Subject: [PATCH 16/29] WIP: auto schema doc --- docs/collections/abstracts.rst | 96 ++-- docs/collections/assignments.rst | 68 +-- docs/collections/blog.rst | 53 +-- docs/collections/grades.rst | 44 +- docs/collections/grants.rst | 148 +++--- docs/collections/people.rst | 761 ++++++++++++++----------------- docs/collections/projects.rst | 104 +++-- docs/collections/proposals.rst | 163 +++---- docs/collections/students.rst | 33 +- docs/conf.py | 79 ++-- 10 files changed, 735 insertions(+), 814 deletions(-) diff --git a/docs/collections/abstracts.rst b/docs/collections/abstracts.rst index fef82ecf6..abbe79a07 100644 --- a/docs/collections/abstracts.rst +++ b/docs/collections/abstracts.rst @@ -1,22 +1,21 @@ Abstracts -============ +========= Abstracts for a conference or workshop. This is generally public information Schema ------ The following lists key names mapped to its type and meaning for each entry. -:_id: str, Unique identifier for submission. This generally includes the author name and - part of the title. -:title: str, title of the presentation/paper. -:firstname: str, first name of the author. -:lastname: str, last name of the author. -:coauthors: str, names of coauthors, optional -:email: str, contact email for the author. -:institution: str, name of the inistitution -:references: str, HTML string of reference for the abstract itself, optional -:text: str, HTML string of the abstract. -:timestamp: float, The time when the abstract was submitted. +:_id: string, Unique identifier for submission. This generally includes the author name and part of the title., required +:coauthors: string, names of coauthors, optional +:email: string, contact email for the author., required +:firstname: string, first name of the author., required +:institution: string, name of the inistitution, required +:lastname: string, last name of the author., required +:references: string, HTML string of reference for the abstract itself, optional +:text: string, HTML string of the abstract., required +:timestamp: string, The time when the abstract was submitted., required +:title: string, title of the presentation/paper., required YAML Example @@ -24,45 +23,18 @@ YAML Example .. code-block:: yaml - Mouginot.Model: - title: Model Performance Analysis - firstname: Baptiste - lastname: Mouginot - coauthors: P.P.H. Wilson - email: mouginot@wisc.edu - institution: University of Wisconsin-Madison - references: "[1] B. MOUGINOT, “CyCLASS: CLASS models for Cyclus,”, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2\ - \ (2016).\n[2] B. Mouginot, P.P.H. Wilson, R.W. Carlsen, “Impact of Isotope Fidelity\ - \ on Fuel Cycle Calculations”, ANS Winter Conference, Las Vegas, (November 2016)\n\ - [3] B. Leniau, B. Mouginot, N. Thiollière, X. Doligez, A. Bidaud, F. Courtin,\ - \ M. Ernoult and S. David, “A neural network approach for burn-up calculation\ - \ and its application to the dynamic fuel cycle code CLASS,” Annals of Nuclear\ - \ Energy , 81 , 125 – 133 (2015).\n[4] B. Leniau, F. Courtin, B. Mouginot, N.\ - \ Thiollière, X. Doligez, A. Bidaud, “Generation of SFR Physics Models for the\ - \ Nuclear Fuel Cycle Code CLASS” PHYSOR 2016\n" - text: "The CLASS team has developed high quality predictors based on pre-trained\ - \ neural networks, allowing the estimation the evolution different neutronic parameters,\ - \ such as neutron multiplication factor or macroscopic cross sections, along the\ - \ irradiation of the fuel. This allows building various fuel fabrication and depletion\ - \ models for fuel cycle simulators. The cyCLASS package [1] has been developed\ - \ to allow the use of CLASS fabrication and cross section prediction models inside\ - \ Cyclus. cyCLASS provides a reactor facility and a fuel fabrication facility,\ - \ which are able to use any CLASS models to provide/request fuel to the entire\ - \ Cyclus ecosystem. Using cyCLASS, it has been possible to perform fuel cycle\ - \ simulations comparing different levels of archetypes fidelity[2].\n\nThis work\ - \ focuses on the analysis of the performance of some high fidelity models developed\ - \ from [3,4], extending the isotopic validity space from uranium and plutonium\ - \ to the most common transuranic elements for Light Water Reactors (LWR) and Sodium\ - \ Fast Reactors (SFR). Those extended models were required to study a transition\ - \ scenario from the actual US nuclear fleet to a SFR and LWR fleet reprocessing\ - \ the most commun transuranic elements (see “Recipe vs Model” presentation from\ - \ the same author). The present work aims to evaluate the following for each of\ - \ the models:\nthe performance relative to the training sample density,\nthe precision\ - \ topography inside and outside of the validity space,\nthe performance of the\ - \ burnup calculation for the cross section predictors.\nAs a complete set of real\ - \ data is not available to benchmark such models, their relative performances\ - \ will be evaluated with regards to the depletion tool used to train them.\n" - timestamp: 5/5/2017 13:15:59 + Mouginot.Model: + firstname: Baptiste + institution: University of Wisconsin-Madison + timestamp: 5/5/2017 13:15:59 + coauthors: P.P.H. Wilson + lastname: Mouginot + title: Model Performance Analysis + references: '[1] B. MOUGINOT, “cyCLASS: CLASS models for Cyclus,”, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 + (2016).' + email: mouginot@wisc.edu + text: The CLASS team has developed high quality predictors based on pre-trained + neural network... JSON/Mongo Example @@ -70,13 +42,15 @@ JSON/Mongo Example .. code-block:: json - {"_id": "Mouginot.Model", - "coauthors": "P.P.H. Wilson", - "email": "mouginot@wisc.edu", - "firstname": "Baptiste", - "institution": "University of Wisconsin-Madison", - "lastname": "Mouginot", - "references": "[1] B. MOUGINOT, \u201ccyCLASS: CLASS models for Cyclus,\u201d, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 (2016).", - "text": "The CLASS team has developed high quality predictors based on pre-trained neural network...", - "timestamp": "5/5/2017 13:15:59", - "title": "Model Performance Analysis"} + { + "_id": "Mouginot.Model", + "coauthors": "P.P.H. Wilson", + "email": "mouginot@wisc.edu", + "firstname": "Baptiste", + "institution": "University of Wisconsin-Madison", + "lastname": "Mouginot", + "references": "[1] B. MOUGINOT, \u201ccyCLASS: CLASS models for Cyclus,\u201d, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 (2016).", + "text": "The CLASS team has developed high quality predictors based on pre-trained neural network...", + "timestamp": "5/5/2017 13:15:59", + "title": "Model Performance Analysis" + } diff --git a/docs/collections/assignments.rst b/docs/collections/assignments.rst index 5e778f6ef..e08e9b8c3 100644 --- a/docs/collections/assignments.rst +++ b/docs/collections/assignments.rst @@ -1,20 +1,18 @@ Assignments -============ +=========== Information about assignments for classes. Schema ------ The following lists key names mapped to its type and meaning for each entry. -:_id: str, A unique id for the assignment, such a HW01-EMCH-558-2016-S -:courses: list of str, ids of the courses that have this assignment -:category: str, such as 'homework' or 'final' -:file: str, path to assignment file in store, optional -:solution: str, path to solution file in store, optional -:question: list of str, titles for the questions on this assignment, - optional, defaults to 1-indexed ints if absent. -:points: list of int or float, list of number of points possible for each - question. Length is the number of questions +:_id: string, A unique id for the assignment, such a HW01-EMCH-558-2016-S, required +:category: string, such as 'homework' or 'final', required +:courses: string, ids of the courses that have this assignment, required +:file: string, path to assignment file in store, optional +:points: ('integer', 'float'), list of number of points possible for each question. Length is the number of questions, required +:question: string, titles for the questions on this assignment, optional +:solution: string, path to solution file in store, optional YAML Example @@ -22,20 +20,19 @@ YAML Example .. code-block:: yaml - hw01-rx-power: - category: homework - courses: - - EMCH-558-2016-S - - EMCH-758-2016-S - points: - - 1 - - 2 - - 3 - questions: - - 1-9 - - 1-10 - - 1-12 - + hw01-rx-power: + courses: + - EMCH-558-2016-S + - EMCH-758-2016-S + category: homework + questions: + - 1-9 + - 1-10 + - 1-12 + points: + - 1 + - 2 + - 3 JSON/Mongo Example @@ -43,8 +40,21 @@ JSON/Mongo Example .. code-block:: json - {"_id": "hw01-rx-power", - "category": "homework", - "courses": ["EMCH-558-2016-S", "EMCH-758-2016-S"], - "points": [1, 2, 3], - "questions": ["1-9", "1-10", "1-12"]} + { + "_id": "hw01-rx-power", + "category": "homework", + "courses": [ + "EMCH-558-2016-S", + "EMCH-758-2016-S" + ], + "points": [ + 1, + 2, + 3 + ], + "questions": [ + "1-9", + "1-10", + "1-12" + ] + } diff --git a/docs/collections/blog.rst b/docs/collections/blog.rst index 9e387cb3e..ddc001b51 100644 --- a/docs/collections/blog.rst +++ b/docs/collections/blog.rst @@ -1,19 +1,19 @@ Blog -============ +==== This collection represents blog posts written by the members of the research group. Schema ------ The following lists key names mapped to its type and meaning for each entry. -:_id: str, short represntation, such as this-is-my-title -:title: str, full human readable title -:original: str, URL of original post, if this is a repost, optional -:author: str, name or AKA of author -:year: int, Publication year -:month: str, Publication month -:day: int, Publication day -:post: str, actual contents of the post +:_id: string, short represntation, such as this-is-my-title, required +:author: string, name or AKA of author, required +:day: integer, Publication day, required +:month: string, Publication month, required +:original: string, URL of original post, if this is a repost, optional +:post: string, actual contents of the post, required +:title: string, full human readable title, required +:year: integer, Publication year, required YAML Example @@ -21,15 +21,14 @@ YAML Example .. code-block:: yaml - my-vision: - title: My Vision - author: Anthony Scopatz - year: 2015 - month: September - day: 18 - original: https://scopatz.com/my-vision/ - post: | - I would like see things move forward. Deep, I know! + my-vision: + original: https://scopatz.com/my-vision/ + author: Anthony Scopatz + title: My Vision + post: I would like see things move forward. Deep, I know! + month: September + day: 18 + year: 2015 JSON/Mongo Example @@ -37,11 +36,13 @@ JSON/Mongo Example .. code-block:: json - {"_id": "my-vision", - "title": "My Vision", - "author": "Anthony Scopatz", - "year": 2015, - "month": "September", - "day": 18, - "original": "https://scopatz.com/my-vision/", - "post": "I would like see things move forward. Deep, I know!"} + { + "_id": "my-vision", + "author": "Anthony Scopatz", + "day": 18, + "month": "September", + "original": "https://scopatz.com/my-vision/", + "post": "I would like see things move forward. Deep, I know!", + "title": "My Vision", + "year": 2015 + } diff --git a/docs/collections/grades.rst b/docs/collections/grades.rst index 726af2f50..70df89a39 100644 --- a/docs/collections/grades.rst +++ b/docs/collections/grades.rst @@ -1,17 +1,17 @@ Grades -============ +====== The grade for a student on an assignment. This information should be private. Schema ------ The following lists key names mapped to its type and meaning for each entry. -:_id: str, unique id, typically the student-assignment-course -:student: str, student id -:assignment: str, assignment id -:course: str, course id -:scores: list of int or float, the number of points earned on each question -:filename: str, path to file in store, optional +:_id: string, unique id, typically the student-assignment-course, required +:assignment: string, assignment id, required +:course: string, course id, required +:filename: string, path to file in store, optional +:scores: ('integer', 'float'), the number of points earned on each question, required +:student: string, student id, required YAML Example @@ -19,13 +19,14 @@ YAML Example .. code-block:: yaml - Human A. Person-rx-power-hw02-EMCH-758-2017-S: - assignment: 2017-rx-power-hw02 - course: EMCH-758-2017-S - scores: - - 1 - - 1.6 - - 3 + Human A. Person-rx-power-hw02-EMCH-758-2017-S: + student: hap + course: EMCH-758-2017-S + assignment: 2017-rx-power-hw02 + scores: + - 1 + - 1.6 + - 3 JSON/Mongo Example @@ -33,7 +34,14 @@ JSON/Mongo Example .. code-block:: json - {"_id": "Human A. Person-rx-power-hw02-EMCH-758-2017-S", - "assignment": "2017-rx-power-hw02", - "course": "EMCH-758-2017-S", - "scores": [1, 1.6, 3]} + { + "_id": "Human A. Person-rx-power-hw02-EMCH-758-2017-S", + "assignment": "2017-rx-power-hw02", + "course": "EMCH-758-2017-S", + "scores": [ + 1, + 1.6, + 3 + ], + "student": "hap" + } diff --git a/docs/collections/grants.rst b/docs/collections/grants.rst index a0a44e3ff..2f6cfa379 100644 --- a/docs/collections/grants.rst +++ b/docs/collections/grants.rst @@ -1,40 +1,39 @@ Grants -============ +====== This collection represents grants that have been awarded to the group. Schema ------ The following lists key names mapped to its type and meaning for each entry. -:_id: str or number, short represntation, such as this-is-my-name -:title: str, actual title of proposal / grant -:funder: str, the agency funding the work -:program: str, the program the work was funded under -:grant_id: str, optional, the identfier for this work, eg #42024 -:call_for_proposals: str, optional, URL to the call for proposals -:narrative: str, optional, URL of document -:benefit_of_collaboration: str, optional, URL of document -:amount: int or float, value of award -:currency: str, typically '$' or 'USD' -:begin_year: int, start year of the grant -:begin_month: str, start month of the grant -:begin_day: int, start day of the grant, optional -:end_year: int, end year of the grant -:end_month": str, end month of the grant -:end_day: str, end day of the grant, optional -:team: list of dicts, information about the team members participating in the grant. - These dicts have the following form: - - .. code-block:: python - - [{"name": str, # should match a person's name or AKA - "position": str, # PI, Co-PI, Co-I, Researcher, etc. - "institution": str, # The institution of this investigator - "subaward_amount": int or float, # optional - "cv": str, # optional, URL of document - }, - ... - ] +:_id: ('string', 'integer', 'float'), short representation, such as this-is-my-name, required +:amount: ('integer', 'float'), value of award, required +:begin_day: integer, start day of the grant, optional +:begin_month: string, start month of the grant, required +:begin_year: integer, start year of the grant, required +:benefit_of_collaboration: string, optional +:call_for_proposals: string, optional +:currency: string, typically '$' or 'USD', optional +:end_day: ('string', 'integer'), end day of the grant, optional +:end_month": string, end month of the grant, optional +:end_year: integer, end year of the grant, required +:funder: string, the agency funding the work, required +:grant_id: string, the identfier for this work, optional +:narrative: string, optional +:program: string, the program the work was funded under, required +:status: string, status of the grant, required +:team: list, information about the team members participating in the grant., required + :subaward_amount: ('integer', 'float'), optional + :cv: string, optional + :institution: string, optional + :name: string, optional + :position: string, optional + :subaward_amount: ('integer', 'float'), optional + :cv: string, optional + :institution: string, optional + :name: string, optional + :position: string, optional +:title: string, actual title of proposal / grant, required YAML Example @@ -42,28 +41,26 @@ YAML Example .. code-block:: yaml - - SymPy-1.1: - amount: 3000.0 - begin_day: 1 - begin_month: May - begin_year: 2017 - call_for_proposals: https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ - end_day: 31 - end_month: December - end_year: 2017 - funder: NumFOCUS - narrative: https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing - program: Small Development Grants - team: - - institution: University of South Carolina - name: Anthony Scopatz - position: PI - - institution: University of South Carolina - name: Aaron Meurer - position: researcher - title: SymPy 1.1 Release Support - + SymPy-1.1: + call_for_proposals: https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ + end_month: December + title: SymPy 1.1 Release Support + begin_month: May + funder: NumFOCUS + team: + - institution: University of South Carolina + name: Anthony Scopatz + position: PI + - institution: University of South Carolina + name: Aaron Meurer + position: researcher + begin_year: 2017 + end_year: 2017 + begin_day: 1 + narrative: https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing + program: Small Development Grants + amount: 3000.0 + end_day: 31 JSON/Mongo Example @@ -71,23 +68,30 @@ JSON/Mongo Example .. code-block:: json - {"_id": "SymPy-1.1", - "amount": 3000.0, - "begin_day": 1, - "begin_month": "May", - "begin_year": 2017, - "call_for_proposals": "https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ", - "end_day": 31, - "end_month": "December", - "end_year": 2017, - "funder": "NumFOCUS", - "narrative": "https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing", - "program": "Small Development Grants", - "team": [{"institution": "University of South Carolina", - "name": "Anthony Scopatz", - "position": "PI"}, - {"institution": "University of South Carolina", - "name": "Aaron Meurer", - "position": "researcher"} - ], - "title": "SymPy 1.1 Release Support"} + { + "_id": "SymPy-1.1", + "amount": 3000.0, + "begin_day": 1, + "begin_month": "May", + "begin_year": 2017, + "call_for_proposals": "https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ", + "end_day": 31, + "end_month": "December", + "end_year": 2017, + "funder": "NumFOCUS", + "narrative": "https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing", + "program": "Small Development Grants", + "team": [ + { + "institution": "University of South Carolina", + "name": "Anthony Scopatz", + "position": "PI" + }, + { + "institution": "University of South Carolina", + "name": "Aaron Meurer", + "position": "researcher" + } + ], + "title": "SymPy 1.1 Release Support" + } diff --git a/docs/collections/people.rst b/docs/collections/people.rst index 72d356408..1e56cd409 100644 --- a/docs/collections/people.rst +++ b/docs/collections/people.rst @@ -1,5 +1,5 @@ People -============ +====== This collection describes the members of the research group. This is normally public data. @@ -7,146 +7,79 @@ Schema ------ The following lists key names mapped to its type and meaning for each entry. -:_id: str, unique identifier for the group member -:name: str, Full, canonical name for the person -:title: str, for example, Dr., etc. -:position: str, such as professor, graduate student, or scientist -:email: str, email address of the group member, optional. -:aka: list of str, list of aliases (also-known-as), useful for identifying - the group member in citations or elsewhere. -:avatar: str, URL to avatar -:bio: str, short biographical text -:active: bool, If the person is an active member, default True. -:collab: bool, If the person is a collaborator, default False. -:education: list of dictionaries, This contains the educational information for - the group member. These dicts have the following form: - - .. code-block:: python - - [{"institution": str, - "location": str, - "degree": str, - "begin_year": int, - "begin_month": str, # optional - "end_year": int, - "end_month": str, # optional - "gpa": float or str, # optional - "other": [str], # list of strings of other pieces of information - }, - ... - ] - -:employment: list of dicts, Employment information, similar to educational information. - These dicts have the following form: - - .. code-block:: python - - [{"organization": str, - "location": str, - "position": str, - "begin_year": int, - "begin_month": str, # optional - "end_year": int, - "end_month": str, # optional - "other": [str], # list of strings of other pieces of information - }, - ... - ] - -:funding: list of dicts, Funding and scholarship that the group member has - individually obtained in the past, optional. **WARNING:** this is not to be confused - with the **grants** collection, that may refee back to group members. - These dicts have the following form: - - .. code-block:: python - - [{"name": str, - "value": float, - "currency": str, # optional, defaults to "$" - "year": int, - "month": str, # optional - "duration": str or int, # optional length of award - }, - ... - ] - -:service: list of dicts, Service that this group member has provided, optional. - These dicts have the following form: - - .. code-block:: python - - [{"name": str, - "description": str, # optional - "year": int, - "month": str, # optional - "duration": str or int, # optional length of service - }, - ... - ] - -:honors: list of dicts, Honors that have been awarded to this group member, optional. - These dicts have the following form: - - .. code-block:: python - - [{"name": str, - "description": str, # optional - "year": int, - "month": str, # optional - }, - ... - ] - -:teaching: list of dicts, Courses that this group member has taught, if any, optional. - These dicts have the following form: - - .. code-block:: python - - [{"course": str, # name of the course - "organization": str, - "position": str, - "year": int, - "month": str, # optional - "end_year": int, # optional - "end_month": str, # optional - "description": str, # optional - "website": str, # optional URL - "syllabus": str, # optional URL - "video": str, # optional URL - "materials": str, # optional URL - }, - ... - ] - -:membership: list of dicts, Profesional organizations this member is a part of, optional. - These dicts have the following form: - - .. code-block:: python - - [{"organization": str, - "position": str, - "description": str, # optional - "begin_year": int, - "begin_month": str, # optional - "end_year": int, # optional - "end_month": str, # optional - "website": str, # optional URL - }, - ... - ] - -:skills: list of dicts, Skill the group member has, optional - These dicts have the following form: - - .. code-block:: python - - [{"name": str, - "category": str, - "level": str - }, - ... - ] - +:_id: string, unique identifier for the group member, required +:active: boolean, If the person is an active member, default True., optional +:aka: ['string', 'list'], list of aliases (also-known-as), useful for identifying the group member in citations or elsewhere., required +:avatar: string, URL to avatar, required +:bio: string, short biographical text, required +:collab: boolean, If the person is a collaborator, default False., optional +:education: list, This contains the educational information for the group member., required + :end_month: string, optional + :begin_year: integer, optional + :end_year: integer, optional + :institution: string, optional + :begin_month: string, optional + :other: string, optional + :degree: string, optional + :gpa: ('float', 'string'), optional + :location: string, optional +:email: string, email address of the group member, optional +:employment: list, Employment information, similar to educational information., required + :end_month: string, optional + :begin_year: integer, optional + :end_year: integer, optional + :position: string, optional + :begin_month: string, optional + :other: string, optional + :organization: string, optional + :location: string, optional +:funding: list, Funding and scholarship that the group member has individually obtained in the past. **WARNING:** this is not to be confused with the **grants** collection, optional + :duration: string, optional + :currency: string, optional + :month: string, optional + :value: ('float', 'integer'), optional + :name: string, optional + :year: integer, optional +:honors: list, Honors that have been awarded to this group member, optional + :month: string, optional + :year: integer, optional + :name: string, optional + :description: string, optional +:membership: list, Profesional organizations this member is a part of, optional + :end_month: string, optional + :begin_year: integer, optional + :end_year: integer, optional + :website: string, optional + :description: string, optional + :begin_month: string, optional + :position: string, optional + :organization: string, optional +:name: string, Full, canonical name for the person, required +:position: string, such as professor, graduate student, or scientist, required +:service: list, Service that this group member has provided, optional + :month: string, optional + :year: integer, optional + :name: string, optional + :duration: string, optional + :description: string, optional +:skills: list, Skill the group member has, optional + :name: string, optional + :level: string, optional + :category: string, optional +:teaching: list, Courses that this group member has taught, if any, optional + :end_month: string, optional + :description: string, optional + :month: string, optional + :position: string, optional + :organization: string, optional + :syllabus: string, optional + :course: string, optional + :video: string, optional + :end_year: integer, optional + :materials: string, optional + :website: string, optional + :year: integer, optional +:title: string, for example, Dr., etc., optional YAML Example @@ -154,208 +87,114 @@ YAML Example .. code-block:: yaml - scopatz: - name: Anthony Scopatz - position: professor - title: Dr. - email: scopatz@cec.sc.edu - aka: - - Scopatz, A. M. - - Scopatz, Anthony - - Scopatz, Anthony M - - Scopatz, Anthony M. - - A. M. Scopatz - - Anthony M Scopatz - - Anthony M. Scopatz - - Anthony Michael Scopatz - avatar: https://avatars1.githubusercontent.com/u/320553?v=3&s=200 - bio: Anthony Scopatz is currently an Assistant Professor at the University of South - Carolina in the Nuclear Engineering program in the Mechanical Engineering Department. - He is a computational physicist and long time Python developer. Anthony holds - his BS in Physics from UC, Santa Barbara and a Ph.D. in Mechanical / Nuclear Engineering - from UT Austin. A former Enthought employee, he spent his post-doctoral studies - at the FLASH Center at the University of Chicago in the Astrophysics Department. - Then he became a Staff Scientist at the University of Wisconsin-Maidson in Engineering - Physics. Anthony's research interests revolve around essential physics modeling - of the nuclear fuel cycle, and information theory & entropy. Anthony is proudly - a fellow of the Python Software Foundation and has published and spoken at numerous - conferences on a variety of science & software development topics. - education: - - begin_year: 2008 - degree: Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program - end_year: 2011 - institution: The University of Texas at Austin - location: Austin, TX - other: - - 'Adviser: Erich A. Schneider' - - 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis' - - begin_year: 2006 - degree: M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program - end_year: 2007 - institution: The University of Texas at Austin - location: Austin, TX - other: - - 'Adviser: Erich A. Schneider' - - 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership' - - begin_year: 2002 - degree: B.S. Physics - end_year: 2006 - institution: University of California, Santa Barbara - location: Santa Barbara, CA - other: - - Graduated with a Major in Physics and a Minor in Mathematics - employment: - - begin_year: 2015 - location: Columbia, SC - organization: The University of South Carolina - other: - - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' - - 'PyNE: The Nuclear Engineering Toolkit.' - - 'Website: http://www.ergs.sc.edu/' - position: Assistant Professor, Mechanical Engineering Department - - begin_year: 2013 - end_year: 2015 - location: Madison, WI - organization: CNERG, The University of Wisconsin-Madison - other: - - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' - - 'PyNE: The Nuclear Engineering Toolkit.' - - 'Website: https://cnerg.github.io/' - position: Associate Scientist, Engineering Physics Department - - begin_month: Nov - begin_year: 2011 - end_month: May - end_year: 2013 - location: Chicago, IL - organization: The FLASH Center, The University of Chicago - other: - - 'NIF: Simulation of magnetic field generation from neutral plasmas using - FLASH.' - - 'CosmoB: Simulation of magnetic field generation from neutral plasmas using - FLASH.' - - 'FLASH4: High-energy density physics capabilities and utilities.' - - 'Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from - FLASH.' - - 'OpacPlot: HDF5-based equation of state and opacity file format.' - - 'Website: http://flash.uchicago.edu/site/' - position: Research Scientist, Postdoctoral Scholar, Astronomy & Astrophysics - Dept. - - begin_month: May - begin_year: 2010 - end_month: October - end_year: 2011 - location: Austin, TX - organization: Enthought, Inc. - other: - - 'PlotTool: Time series data visualization for J.P. Morgan Chase.' - - 'PivotTable: Data cube configuration, management, and visualization for - J.P. Morgan Chase.' - - 'WIPP-PA: Waste Isolation Pilot Project Performance Assessment tool for - Sandia Nat’l Labs.' - - 'EasyGuide2: Dental implant imaging software for Keystone Dental.' - position: Scientific Software Developer - - begin_year: 2002 - end_year: 2005 - location: Santa Barbara, CA - organization: University of California, Santa Barbara - other: - - Terahertz Continuous Wave Spectroscopy - - Studied the effect of Terahertz waves on biomacromolecules in water. - position: Undergraduate Research Assistant - - begin_month: June - begin_year: 2004 - end_month: August - end_year: 2004 - location: Los Alamos, NM - organization: Los Alamos National Laboratory - other: - - Terahertz Pump-Probe Spectroscopy - - Looked at the effect of Terahertz waves on Erbium-Arsenide on a Gallium-Arsenide - substrate. - position: Summer Internship/Visiting Scientist - funding: - - name: Omega Laser User's Group Travel Award - value: 1100 - year: 2013 - - name: NIF User's Group Travel Award - value: 1150 - year: 2013 - - name: Innovations in Fuel Cycle Research, 2nd place Systems Engineering and - Analysis - value: 2500 - year: 2010 - - name: 'MCNPX Bug #100' - value: 20 - year: 2009 - membership: - - begin_year: 2006 - organization: American Nuclear Society - position: Member - - begin_year: 2013 - organization: Python Software Foundation - position: Fellow - - begin_year: 2011 - end_year: 2014 - organization: NumFOCUS - position: Founding Treasurer & Board Member - website: http://numfocus.org - service: - - name: SciPy 2014 Communications Chair - year: 2014 - - name: SciPy 2013 Communications Chair - year: 2013 - - name: SciPy 2012 Open Spaces Chair - year: 2012 - - name: SciPy 2011 Chair of Python & Core Technologies Track - year: 2011 - - name: PyCon 2010 Program Committee Member - year: 2010 - - name: Peer Reviewer for Nuclear Engineering & Design (NED-D-09-00256) - year: 2009 - skills: - - category: Programming Languages - level: expert - name: Python - - category: Programming Languages - level: expert - name: Cython - - category: Programming Languages - level: expert - name: C - - category: Programming Languages - level: expert - name: C++ - - category: Programming Languages - level: expert - name: Bash - - category: Specialized Software - level: expert - name: Linux - - category: Specialized Software - level: intermediate - name: MongoDB - teaching: - - course: 'EMCH 552: Intro to Nuclear Engineering' - description: This course is an introduction to nuclear physics and engineering, - covering the important underlying science and mathematics to nuclear power - generation. Unlike previous sememsters, this course was taught in a flipped - style. - month: August - organization: University of South Carolina - position: Professor - syllabus: https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc - year: 2017 - - course: 'EMCH 558/758: Reactor Power Systems' - description: This course covers conventional Pressurized Water Reactors (PWR) - and Boiling Water Reactors (BWR) system energy designs for transport, containment, - and accident prevention and mitigation. Near term, developmental, and proposed - enhanced capability reactor system designs. Cross listed as an undergraduate - and graduate course. - month: January - organization: University of South Carolina - position: Professor - syllabus: https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing - year: 2017 + scopatz: + position: professor + avatar: https://avatars1.githubusercontent.com/u/320553?v=3&s=200 + bio: Anthony Scopatz is currently an Assistant Professor + name: Anthony Scopatz + membership: + - begin_year: 2006 + organization: American Nuclear Society + position: Member + - begin_year: 2013 + organization: Python Software Foundation + position: Fellow + funding: + - value: 1100 + name: Omega Laser User's Group Travel Award + year: 2013 + - value: 1150 + name: NIF User's Group Travel Award + year: 2013 + aka: + - Scopatz + - Scopatz, A + - Scopatz, A. + - Scopatz, A M + - Anthony Michael Scopatz + email: scopatz@cec.sc.edu + employment: + - begin_year: 2015 + location: Columbia, SC + other: + - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' + - 'PyNE: The Nuclear Engineering Toolkit.' + - 'Website: http://www.ergs.sc.edu/' + organization: The University of South Carolina + position: Assistant Professor, Mechanical Engineering Department + - begin_year: 2013 + end_year: 2015 + position: Associate Scientist, Engineering Physics Department + other: + - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' + - 'PyNE: The Nuclear Engineering Toolkit.' + - 'Website: https://cnerg.github.io/' + organization: CNERG, The University of Wisconsin-Madison + location: Madison, WI + - end_month: May + begin_year: 2011 + end_year: 2013 + position: Research Scientist, Postdoctoral Scholar + begin_month: Nov + other: + - 'NIF: Simulation of magnetic field generation from neutral plasmas using + FLASH.' + - 'CosmoB: Simulation of magnetic field generation from neutral plasmas using + FLASH.' + - 'FLASH4: High-energy density physics capabilities and utilities.' + - 'Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from + FLASH.' + - 'OpacPlot: HDF5-based equation of state and opacity file format.' + - 'Website: http://flash.uchicago.edu/site/' + organization: The FLASH Center, The University of Chicago + location: Chicago, IL + skills: + - level: expert + name: Python + category: Programming Languages + - level: expert + name: Cython + category: Programming Languages + teaching: + - course: 'EMCH 552: Intro to Nuclear Engineering' + syllabus: https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc + position: Professor + month: August + description: This course is an introduction to nuclear physics. + organization: University of South Carolina + year: 2017 + - course: 'EMCH 558/758: Reactor Power Systems' + syllabus: https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing + position: Professor + month: January + description: This course covers conventional reactors. + organization: University of South Carolina + year: 2017 + education: + - other: + - 'Adviser: Erich A. Schneider' + - 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis' + end_year: 2011 + institution: The University of Texas at Austin + degree: Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program + begin_year: 2008 + location: Austin, TX + - other: + - 'Adviser: Erich A. Schneider' + - 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership' + end_year: 2007 + institution: The University of Texas at Austin + degree: M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program + begin_year: 2006 + location: Austin, TX + - other: + - Graduated with a Major in Physics and a Minor in Mathematics + end_year: 2006 + institution: University of California, Santa Barbara + degree: B.S. Physics + begin_year: 2002 + location: Santa Barbara, CA + title: Dr. JSON/Mongo Example @@ -363,91 +202,151 @@ JSON/Mongo Example .. code-block:: json - {"_id": "scopatz", - "name": "Anthony Scopatz", - "position": "professor", - "title": "Dr.", - "aka": ["Scopatz", "Scopatz, A", "Scopatz, A.", "Scopatz, A M", "Anthony Michael Scopatz"], - "avatar": "https://avatars1.githubusercontent.com/u/320553?v=3&s=200", - "email": "scopatz@cec.sc.edu", - "bio": "Anthony Scopatz is currently an Assistant Professor", - "education": [{"begin_year": 2008, - "degree": "Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program", - "end_year": 2011, - "institution": "The University of Texas at Austin", - "location": "Austin, TX", - "other": ["Adviser: Erich A. Schneider", - "Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis"]}, - {"begin_year": 2006, - "degree": "M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program", - "end_year": 2007, - "institution": "The University of Texas at Austin", - "location": "Austin, TX", - "other": ["Adviser: Erich A. Schneider", - "Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership"]}, - {"begin_year": 2002, - "degree": "B.S. Physics", - "end_year": 2006, - "institution": "University of California, Santa Barbara", - "location": "Santa Barbara, CA", - "other": ["Graduated with a Major in Physics and a Minor in Mathematics"]}], - "employment": [{"begin_year": 2015, - "location": "Columbia, SC", - "organization": "The University of South Carolina", - "other": ["Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.", - "PyNE: The Nuclear Engineering Toolkit.", - "Website: http://www.ergs.sc.edu/"], - "position": "Assistant Professor, Mechanical Engineering Department"}, - {"begin_year": 2013, - "end_year": 2015, - "location": "Madison, WI", - "organization": "CNERG, The University of Wisconsin-Madison", - "other": ["Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.", - "PyNE: The Nuclear Engineering Toolkit.", - "Website: https://cnerg.github.io/"], - "position": "Associate Scientist, Engineering Physics Department"}, - {"begin_month": "Nov", - "begin_year": 2011, - "end_month": "May", - "end_year": 2013, - "location": "Chicago, IL", - "organization": "The FLASH Center, The University of Chicago", - "other": ["NIF: Simulation of magnetic field generation from neutral plasmas using FLASH.", "CosmoB: Simulation of magnetic field generation from neutral plasmas using FLASH.", - "FLASH4: High-energy density physics capabilities and utilities.", - "Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from FLASH.", - "OpacPlot: HDF5-based equation of state and opacity file format.", - "Website: http://flash.uchicago.edu/site/"], - "position": "Research Scientist, Postdoctoral Scholar"}], - "funding": [{"name": "Omega Laser User's Group Travel Award", - "value": 1100, - "year": 2013}, - {"name": "NIF User's Group Travel Award", - "value": 1150, - "year": 2013}], - "membership": [{"begin_year": 2006, - "organization": "American Nuclear Society", - "position": "Member"}, - {"begin_year": 2013, - "organization": "Python Software Foundation", - "position": "Fellow"}], - "skills": [{"category": "Programming Languages", - "level": "expert", - "name": "Python"}, - {"category": "Programming Languages", - "level": "expert", - "name": "Cython"}], - "teaching": [{"course": "EMCH 552: Intro to Nuclear Engineering", - "description": "This course is an introduction to nuclear physics.", - "month": "August", - "organization": "University of South Carolina", - "position": "Professor", - "syllabus": "https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc", - "year": 2017}, - {"course": "EMCH 558/758: Reactor Power Systems", - "description": "This course covers conventional reactors.", - "month": "January", - "organization": "University of South Carolina", - "position": "Professor", - "syllabus": "https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing", - "year": 2017}] - } + { + "_id": "scopatz", + "aka": [ + "Scopatz", + "Scopatz, A", + "Scopatz, A.", + "Scopatz, A M", + "Anthony Michael Scopatz" + ], + "avatar": "https://avatars1.githubusercontent.com/u/320553?v=3&s=200", + "bio": "Anthony Scopatz is currently an Assistant Professor", + "education": [ + { + "begin_year": 2008, + "degree": "Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program", + "end_year": 2011, + "institution": "The University of Texas at Austin", + "location": "Austin, TX", + "other": [ + "Adviser: Erich A. Schneider", + "Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis" + ] + }, + { + "begin_year": 2006, + "degree": "M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program", + "end_year": 2007, + "institution": "The University of Texas at Austin", + "location": "Austin, TX", + "other": [ + "Adviser: Erich A. Schneider", + "Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership" + ] + }, + { + "begin_year": 2002, + "degree": "B.S. Physics", + "end_year": 2006, + "institution": "University of California, Santa Barbara", + "location": "Santa Barbara, CA", + "other": [ + "Graduated with a Major in Physics and a Minor in Mathematics" + ] + } + ], + "email": "scopatz@cec.sc.edu", + "employment": [ + { + "begin_year": 2015, + "location": "Columbia, SC", + "organization": "The University of South Carolina", + "other": [ + "Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.", + "PyNE: The Nuclear Engineering Toolkit.", + "Website: http://www.ergs.sc.edu/" + ], + "position": "Assistant Professor, Mechanical Engineering Department" + }, + { + "begin_year": 2013, + "end_year": 2015, + "location": "Madison, WI", + "organization": "CNERG, The University of Wisconsin-Madison", + "other": [ + "Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.", + "PyNE: The Nuclear Engineering Toolkit.", + "Website: https://cnerg.github.io/" + ], + "position": "Associate Scientist, Engineering Physics Department" + }, + { + "begin_month": "Nov", + "begin_year": 2011, + "end_month": "May", + "end_year": 2013, + "location": "Chicago, IL", + "organization": "The FLASH Center, The University of Chicago", + "other": [ + "NIF: Simulation of magnetic field generation from neutral plasmas using FLASH.", + "CosmoB: Simulation of magnetic field generation from neutral plasmas using FLASH.", + "FLASH4: High-energy density physics capabilities and utilities.", + "Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from FLASH.", + "OpacPlot: HDF5-based equation of state and opacity file format.", + "Website: http://flash.uchicago.edu/site/" + ], + "position": "Research Scientist, Postdoctoral Scholar" + } + ], + "funding": [ + { + "name": "Omega Laser User's Group Travel Award", + "value": 1100, + "year": 2013 + }, + { + "name": "NIF User's Group Travel Award", + "value": 1150, + "year": 2013 + } + ], + "membership": [ + { + "begin_year": 2006, + "organization": "American Nuclear Society", + "position": "Member" + }, + { + "begin_year": 2013, + "organization": "Python Software Foundation", + "position": "Fellow" + } + ], + "name": "Anthony Scopatz", + "position": "professor", + "skills": [ + { + "category": "Programming Languages", + "level": "expert", + "name": "Python" + }, + { + "category": "Programming Languages", + "level": "expert", + "name": "Cython" + } + ], + "teaching": [ + { + "course": "EMCH 552: Intro to Nuclear Engineering", + "description": "This course is an introduction to nuclear physics.", + "month": "August", + "organization": "University of South Carolina", + "position": "Professor", + "syllabus": "https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc", + "year": 2017 + }, + { + "course": "EMCH 558/758: Reactor Power Systems", + "description": "This course covers conventional reactors.", + "month": "January", + "organization": "University of South Carolina", + "position": "Professor", + "syllabus": "https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing", + "year": 2017 + } + ], + "title": "Dr." + } diff --git a/docs/collections/projects.rst b/docs/collections/projects.rst index 2494119b3..91150e4ce 100644 --- a/docs/collections/projects.rst +++ b/docs/collections/projects.rst @@ -1,32 +1,26 @@ Projects -============ +======== This collection describes the research group projects. This is normally public data. Schema ------ The following lists key names mapped to its type and meaning for each entry. -:_id: str, Unique project identifier. -:name: str, name of the project. -:description: str, brief project description. -:website: str, URL of the website. -:repo: str, URL of the source code repo, if available, optional. -:logo: str, URL to the project logo, optional. -:other: list of str, other information about the project, optional/ -:team: list of dict, People who are/have been woking on this project. - These dicts have the following structure: - - .. code-block:: python - - [{"name": str, # should match a person's name or AKA - "position": str, - "begin_year": int, - "begin_month": str, # optional - "end_year": int, # optional - "end_month": str, # optional - }, - ... - ] +:_id: string, Unique project identifier., required +:description: string, brief project description., required +:grant: string, Grant id if there is a grant supporting this project, optional +:logo: string, URL to the project logo, optional +:name: string, name of the project., required +:other: ['list', 'string'], other information about the project, optional +:repo: string, URL of the source code repo, if available, optional +:team: list, People who are/have been woking on this project., required + :end_month: string, optional + :begin_year: integer, optional + :end_year: integer, optional + :position: string, optional + :begin_month: string, optional + :name: string, optional +:website: string, URL of the website., required YAML Example @@ -34,21 +28,22 @@ YAML Example .. code-block:: yaml - Cyclus: - description: Agent-Based Nuclear Fuel Cycle Simulator - logo: http://fuelcycle.org/_static/big_c.png - other: - - Discrete facilities with discrete material transactions - - Low barrier to entry, rapid payback to adoption - repo: https://github.com/cyclus/cyclus/ - team: - - begin_month: June - begin_year: 2013 - end_month: July - end_year: 2015 - name: Anthony Scopatz - position: Project Lead - website: http://fuelcycle.org/ + Cyclus: + other: + - Discrete facilities with discrete material transactions + - Low barrier to entry, rapid payback to adoption + logo: http://fuelcycle.org/_static/big_c.png + website: http://fuelcycle.org/ + description: Agent-Based Nuclear Fuel Cycle Simulator + team: + - end_month: July + begin_year: 2013 + end_year: 2015 + position: Project Lead + begin_month: June + name: Anthony Scopatz + repo: https://github.com/cyclus/cyclus/ + name: Cyclus JSON/Mongo Example @@ -56,16 +51,25 @@ JSON/Mongo Example .. code-block:: json - {"_id": "Cyclus", - "description": "Agent-Based Nuclear Fuel Cycle Simulator", - "logo": "http://fuelcycle.org/_static/big_c.png", - "other": ["Discrete facilities with discrete material transactions", - "Low barrier to entry, rapid payback to adoption"], - "repo": "https://github.com/cyclus/cyclus/", - "team": [{"begin_month": "June", - "begin_year": 2013, - "end_month": "July", - "end_year": 2015, - "name": "Anthony Scopatz", - "position": "Project Lead"}], - "website": "http://fuelcycle.org/"} + { + "_id": "Cyclus", + "description": "Agent-Based Nuclear Fuel Cycle Simulator", + "logo": "http://fuelcycle.org/_static/big_c.png", + "name": "Cyclus", + "other": [ + "Discrete facilities with discrete material transactions", + "Low barrier to entry, rapid payback to adoption" + ], + "repo": "https://github.com/cyclus/cyclus/", + "team": [ + { + "begin_month": "June", + "begin_year": 2013, + "end_month": "July", + "end_year": 2015, + "name": "Anthony Scopatz", + "position": "Project Lead" + } + ], + "website": "http://fuelcycle.org/" + } diff --git a/docs/collections/proposals.rst b/docs/collections/proposals.rst index f69c4985c..45a7ec1f7 100644 --- a/docs/collections/proposals.rst +++ b/docs/collections/proposals.rst @@ -1,44 +1,23 @@ Proposals -============ +========= This collection represents proposals that have been submitted by the group. Schema ------ The following lists key names mapped to its type and meaning for each entry. -:_id: str or number, short represntation, such as this-is-my-name -:title: str, actual title of proposal -:pi: str, principal investigator name -:authors: list of str, other investigator names -:status: str, e.g. 'submitted', 'accepted', 'rejected' -:ammount: int or float, value of award -:currency: str, typically '$' or 'USD' -:durration: int or float, number of years -:year: int, Year that the proposal is due -:month: str, month that the proposal is due -:day: int, day that the proposal is due -:pre: dict, Information about the pre-proposal, optional. - This dict has the following form: - - .. code-block:: python - - {"year": int, - "month": str, - "day": int, - "narrative": str, # URL of document - "benefit_of_collaboration": str, # URL of document - "cv": list of str, # URL to documents - } - -:full: Information about the pre-proposal, optional - This dict has the following form: - - .. code-block:: python - - {"narrative": str, # URL of document - "benefit_of_collaboration": str, # URL of document - "cv": list of str, # URL to documents - } +:_id: ('string', 'integer', 'float'), short represntation, such as this-is-my-name, required +:ammount: ('integer', 'float'), value of award, required +:authors: string, other investigator names, required +:currency: string, typically '$' or 'USD', required +:day: integer, day that the proposal is due, required +:durration: ('integer', 'float'), number of years, required +:month: string, month that the proposal is due, required +:pi: string, principal investigator name, required +:pre: dict, Information about the pre-proposal, optional +:status: string, e.g. 'submitted', 'accepted', 'rejected', required +:title: string, actual title of proposal, required +:year: integer, Year that the proposal is due, required YAML Example @@ -46,34 +25,34 @@ YAML Example .. code-block:: yaml - mypropsal: - title: A very fine proposal indeed - pi: Anthony Scopatz - authors: - - Anthony Scopatz - - Robert Flanagan - status: submitted - ammount: 1000000.0 - currency: USD - durration: 3 - year: 1999 - month: Aug - day: 18 - pre: - year: 1998 - month: Aug - day: 2 - narrative: http://some.com/pdf - benefit_of_collaboration: http://pdf.com/benefit_of_collaboration - cv: - - http://pdf.com/scopatz-cv - - http://pdf.com/flanagan-cv - full: - narrative: http://some.com/pdf - benefit_of_collaboration: http://pdf.com/benefit_of_collaboration - cv: - - http://pdf.com/scopatz-cv - - http://pdf.com/flanagan-cv + mypropsal: + durration: 3 + pre: + cv: + - http://pdf.com/scopatz-cv + - http://pdf.com/flanagan-cv + narrative: http://some.com/pdf + month: Aug + benefit_of_collaboration: http://pdf.com/benefit_of_collaboration + day: 2 + year: 1998 + month: Aug + status: submitted + day: 18 + full: + narrative: http://some.com/pdf + cv: + - http://pdf.com/scopatz-cv + - http://pdf.com/flanagan-cv + benefit_of_collaboration: http://pdf.com/benefit_of_collaboration + ammount: 1000000.0 + pi: Anthony Scopatz + currency: USD + authors: + - Anthony Scopatz + - Robert Flanagan + year: 1999 + title: A very fine proposal indeed JSON/Mongo Example @@ -81,28 +60,38 @@ JSON/Mongo Example .. code-block:: json - {"_id": "mypropsal", - "title": "A very fine proposal indeed", - "pi": "Anthony Scopatz", - "authors": ["Anthony Scopatz", - "Robert Flanagan"], - "status": "submitted", - "ammount": 1000000.0, - "currency": "USD", - "durration": 3, - "year": 1999, - "month": "Aug", - "day": 18, - "pre": {"year": 1998, - "month": "Aug", - "day": 2, - "narrative": "http://some.com/pdf", - "benefit_of_collaboration": "http://pdf.com/benefit_of_collaboration", - "cv": ["http://pdf.com/scopatz-cv", - "http://pdf.com/flanagan-cv"]}, - "full": {"narrative": "http://some.com/pdf" - "benefit_of_collaboration": "http://pdf.com/benefit_of_collaboration", - "cv": ["http://pdf.com/scopatz-cv", - "http://pdf.com/flanagan-cv"]} - } - + { + "_id": "mypropsal", + "ammount": 1000000.0, + "authors": [ + "Anthony Scopatz", + "Robert Flanagan" + ], + "currency": "USD", + "day": 18, + "durration": 3, + "full": { + "benefit_of_collaboration": "http://pdf.com/benefit_of_collaboration", + "cv": [ + "http://pdf.com/scopatz-cv", + "http://pdf.com/flanagan-cv" + ], + "narrative": "http://some.com/pdf" + }, + "month": "Aug", + "pi": "Anthony Scopatz", + "pre": { + "benefit_of_collaboration": "http://pdf.com/benefit_of_collaboration", + "cv": [ + "http://pdf.com/scopatz-cv", + "http://pdf.com/flanagan-cv" + ], + "day": 2, + "month": "Aug", + "narrative": "http://some.com/pdf", + "year": 1998 + }, + "status": "submitted", + "title": "A very fine proposal indeed", + "year": 1999 + } diff --git a/docs/collections/students.rst b/docs/collections/students.rst index 7ea2562e3..9a8bdb318 100644 --- a/docs/collections/students.rst +++ b/docs/collections/students.rst @@ -1,26 +1,27 @@ Students -============ +======== This is a collection of student names and metadata. This should probably be private. Schema ------ The following lists key names mapped to its type and meaning for each entry. -:_id: str, short represntation, such as this-is-my-name -:aka: list of str, list of aliases, optional -:email: str, email address, optional -:university_id: str, The university identifier for the student, optional +:_id: string, short represntation, such as this-is-my-name, required +:aka: ('list', 'string'), list of aliases, optional +:email: string, email address, optional +:university_id: string, The university identifier for the student, optional + YAML Example ------------ .. code-block:: yaml - Human A. Person: - aka: - - H. A. Person - email: haperson@uni.edu - university_id: HAP42 + Human A. Person: + aka: + - H. A. Person + email: haperson@uni.edu + university_id: HAP42 JSON/Mongo Example @@ -28,7 +29,11 @@ JSON/Mongo Example .. code-block:: json - {"_id": "Human A. Person", - "aka": ["H. A. Person"], - "email": "haperson@uni.edu", - "university_id": "HAP42"} + { + "_id": "Human A. Person", + "aka": [ + "H. A. Person" + ], + "email": "haperson@uni.edu", + "university_id": "HAP42" + } diff --git a/docs/conf.py b/docs/conf.py index 612618864..3a5bd745c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,8 +12,15 @@ # serve to show the default. import sys, os +import json +import tempfile +from textwrap import indent + import cloud_sptheme as csp + from regolith import __version__ as REGOLITH_VERSION +from regolith.fsclient import json_to_yaml +from regolith.schemas import SCHEMAS, EXEMPLARS # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -270,46 +277,66 @@ 'proposals': 'This collection represents proposals that have been submitted by the group.\n\n', 'students': 'This is a collection of student names and metadata. This should probably be private.\n\n'} -from regolith.schemas import SCHEMAS, EXEMPLARS -from regolith.fsclient import json_to_yaml -import json -from io import StringIO +from pprint import pprint +import string + -def format_key(schema, key, indent=''): +def format_key(schema, key, indent_str=''): s = '' - line_format = ':{key}: {type}, {description}\n' - s += line_format.format( - key=schema[key], - type=schema[key]['type]'], - description=schema[key]['description']) - if 'schema' in schema: - for key in schema['schema']: - s += format_key(schema['schema'], key, indent=indent+'\t') + line_format = ':{key}: {type}, {description}, required\n' + line_format_o = ':{key}: {type}, {description}, optional\n' + if not schema.get('required', False): + lf = line_format_o + else: + lf = line_format + if 'type' in schema and 'description' in schema: + s += indent(lf.format(key=key, + description=schema.get('description', ''), + type=schema.get('type', '')), indent_str) + elif 'type' in schema[key]: + s += indent(lf.format(key=key, + description=schema[key].get('description', ''), + type=schema[key].get('type', '')), indent_str) + s = s.replace(', , ', ', ') + for inner_key in schema.get('schema', ()): + s += format_key(schema['schema'], inner_key, + indent_str=indent_str + '\t') + return s def build_schema_doc(key): - fn = key + '_test.rst' + fn = 'collections/' + key + '.rst' with open(fn, 'w') as f: s = '' - s += key - s += '=' * len(key) + s += key.title() + '\n' + s += '=' * len(key) + '\n' + s += schema_top_docs[key] s += 'Schema\n------\nThe following lists key names mapped to its type and meaning for each entry.\n\n' schema = SCHEMAS[key] - line_format = ':{key}: {type}, {description}\n' - for k in schema: - s += line_format.format(key=k, type=schema[k]['type'], - description=schema[k]['description']) + schema_list = list(schema.keys()) + schema_list.sort() + for k in schema_list: + s += format_key(schema[k], key=k) s += '\n\n' s += 'YAML Example\n------------\n\n' s += '.. code-block:: yaml\n\n' - jd = json.dumps(EXEMPLARS[key]) - sio = StringIO() - json_to_yaml(jd, sio) - s += sio.getvalue() + temp = tempfile.NamedTemporaryFile() + temp2 = tempfile.NamedTemporaryFile() + with open(temp.name, 'w') as ff: + json.dump(EXEMPLARS[key], ff) + jd = json.dumps(EXEMPLARS[key], sort_keys=True, + indent=4, separators=(',', ': ')) + json_to_yaml(temp.name, temp2.name) + with open(temp2.name, 'r') as ff: + s += indent(ff.read(), '\t') s += '\n\n' s += 'JSON/Mongo Example\n------------------\n\n' s += '.. code-block:: json\n\n' - s += jd + s += indent(jd, '\t') s += '\n' - f.write(s) \ No newline at end of file + f.write(s) + + +for k in SCHEMAS: + build_schema_doc(k) From d0a0ac71d16ca46f1f3c7d38e236273fedbffa98 Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 15:42:55 -0500 Subject: [PATCH 17/29] FIX: proper internal dict format --- docs/collections/abstracts.rst | 10 +- docs/collections/assignments.rst | 8 +- docs/collections/blog.rst | 8 +- docs/collections/grades.rst | 2 +- docs/collections/grants.rst | 37 ++++--- docs/collections/people.rst | 178 ++++++++++++++++--------------- docs/collections/projects.rst | 23 ++-- docs/collections/proposals.rst | 30 +++--- docs/collections/students.rst | 1 + docs/conf.py | 2 + 10 files changed, 157 insertions(+), 142 deletions(-) diff --git a/docs/collections/abstracts.rst b/docs/collections/abstracts.rst index abbe79a07..f21d3c6e8 100644 --- a/docs/collections/abstracts.rst +++ b/docs/collections/abstracts.rst @@ -24,17 +24,17 @@ YAML Example .. code-block:: yaml Mouginot.Model: - firstname: Baptiste institution: University of Wisconsin-Madison - timestamp: 5/5/2017 13:15:59 - coauthors: P.P.H. Wilson - lastname: Mouginot + firstname: Baptiste + email: mouginot@wisc.edu title: Model Performance Analysis references: '[1] B. MOUGINOT, “cyCLASS: CLASS models for Cyclus,”, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 (2016).' - email: mouginot@wisc.edu + coauthors: P.P.H. Wilson + timestamp: 5/5/2017 13:15:59 text: The CLASS team has developed high quality predictors based on pre-trained neural network... + lastname: Mouginot JSON/Mongo Example diff --git a/docs/collections/assignments.rst b/docs/collections/assignments.rst index e08e9b8c3..e6f1040f8 100644 --- a/docs/collections/assignments.rst +++ b/docs/collections/assignments.rst @@ -25,14 +25,14 @@ YAML Example - EMCH-558-2016-S - EMCH-758-2016-S category: homework - questions: - - 1-9 - - 1-10 - - 1-12 points: - 1 - 2 - 3 + questions: + - 1-9 + - 1-10 + - 1-12 JSON/Mongo Example diff --git a/docs/collections/blog.rst b/docs/collections/blog.rst index ddc001b51..d5f955518 100644 --- a/docs/collections/blog.rst +++ b/docs/collections/blog.rst @@ -22,13 +22,13 @@ YAML Example .. code-block:: yaml my-vision: - original: https://scopatz.com/my-vision/ - author: Anthony Scopatz + year: 2015 title: My Vision + day: 18 post: I would like see things move forward. Deep, I know! + author: Anthony Scopatz + original: https://scopatz.com/my-vision/ month: September - day: 18 - year: 2015 JSON/Mongo Example diff --git a/docs/collections/grades.rst b/docs/collections/grades.rst index 70df89a39..72825019a 100644 --- a/docs/collections/grades.rst +++ b/docs/collections/grades.rst @@ -22,11 +22,11 @@ YAML Example Human A. Person-rx-power-hw02-EMCH-758-2017-S: student: hap course: EMCH-758-2017-S - assignment: 2017-rx-power-hw02 scores: - 1 - 1.6 - 3 + assignment: 2017-rx-power-hw02 JSON/Mongo Example diff --git a/docs/collections/grants.rst b/docs/collections/grants.rst index 2f6cfa379..e46416652 100644 --- a/docs/collections/grants.rst +++ b/docs/collections/grants.rst @@ -23,16 +23,19 @@ The following lists key names mapped to its type and meaning for each entry. :program: string, the program the work was funded under, required :status: string, status of the grant, required :team: list, information about the team members participating in the grant., required - :subaward_amount: ('integer', 'float'), optional + + + :position: string, optional :cv: string, optional - :institution: string, optional :name: string, optional - :position: string, optional :subaward_amount: ('integer', 'float'), optional - :cv: string, optional :institution: string, optional - :name: string, optional + :position: string, optional + :cv: string, optional + :name: string, optional + :subaward_amount: ('integer', 'float'), optional + :institution: string, optional :title: string, actual title of proposal / grant, required @@ -42,25 +45,25 @@ YAML Example .. code-block:: yaml SymPy-1.1: - call_for_proposals: https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ - end_month: December - title: SymPy 1.1 Release Support + amount: 3000.0 begin_month: May - funder: NumFOCUS + begin_year: 2017 team: - - institution: University of South Carolina - name: Anthony Scopatz + - name: Anthony Scopatz position: PI - - institution: University of South Carolina - name: Aaron Meurer + institution: University of South Carolina + - name: Aaron Meurer position: researcher - begin_year: 2017 + institution: University of South Carolina + call_for_proposals: https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ end_year: 2017 - begin_day: 1 + end_month: December + end_day: 31 narrative: https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing + title: SymPy 1.1 Release Support + funder: NumFOCUS program: Small Development Grants - amount: 3000.0 - end_day: 31 + begin_day: 1 JSON/Mongo Example diff --git a/docs/collections/people.rst b/docs/collections/people.rst index 1e56cd409..62448829f 100644 --- a/docs/collections/people.rst +++ b/docs/collections/people.rst @@ -14,71 +14,79 @@ The following lists key names mapped to its type and meaning for each entry. :bio: string, short biographical text, required :collab: boolean, If the person is a collaborator, default False., optional :education: list, This contains the educational information for the group member., required + :end_month: string, optional - :begin_year: integer, optional - :end_year: integer, optional - :institution: string, optional + :gpa: ('float', 'string'), optional :begin_month: string, optional + :location: string, optional + :begin_year: integer, optional :other: string, optional :degree: string, optional - :gpa: ('float', 'string'), optional - :location: string, optional + :end_year: integer, optional + :institution: string, optional :email: string, email address of the group member, optional :employment: list, Employment information, similar to educational information., required + :end_month: string, optional - :begin_year: integer, optional - :end_year: integer, optional :position: string, optional :begin_month: string, optional + :location: string, optional + :begin_year: integer, optional :other: string, optional + :end_year: integer, optional :organization: string, optional - :location: string, optional :funding: list, Funding and scholarship that the group member has individually obtained in the past. **WARNING:** this is not to be confused with the **grants** collection, optional + + :year: integer, optional + :name: string, optional :duration: string, optional :currency: string, optional - :month: string, optional :value: ('float', 'integer'), optional - :name: string, optional - :year: integer, optional -:honors: list, Honors that have been awarded to this group member, optional :month: string, optional - :year: integer, optional +:honors: list, Honors that have been awarded to this group member, optional + :name: string, optional + :year: integer, optional :description: string, optional + :month: string, optional :membership: list, Profesional organizations this member is a part of, optional + :end_month: string, optional - :begin_year: integer, optional - :end_year: integer, optional - :website: string, optional :description: string, optional :begin_month: string, optional + :begin_year: integer, optional :position: string, optional + :website: string, optional + :end_year: integer, optional :organization: string, optional :name: string, Full, canonical name for the person, required :position: string, such as professor, graduate student, or scientist, required :service: list, Service that this group member has provided, optional - :month: string, optional - :year: integer, optional + :name: string, optional - :duration: string, optional + :year: integer, optional :description: string, optional + :month: string, optional + :duration: string, optional :skills: list, Skill the group member has, optional + :name: string, optional - :level: string, optional :category: string, optional + :level: string, optional :teaching: list, Courses that this group member has taught, if any, optional - :end_month: string, optional + :description: string, optional - :month: string, optional - :position: string, optional - :organization: string, optional - :syllabus: string, optional - :course: string, optional - :video: string, optional :end_year: integer, optional - :materials: string, optional :website: string, optional + :position: string, optional + :end_month: string, optional + :video: string, optional :year: integer, optional + :course: string, optional + :month: string, optional + :syllabus: string, optional + :materials: string, optional + :organization: string, optional :title: string, for example, Dr., etc., optional @@ -88,54 +96,69 @@ YAML Example .. code-block:: yaml scopatz: - position: professor - avatar: https://avatars1.githubusercontent.com/u/320553?v=3&s=200 - bio: Anthony Scopatz is currently an Assistant Professor - name: Anthony Scopatz membership: - begin_year: 2006 - organization: American Nuclear Society position: Member + organization: American Nuclear Society - begin_year: 2013 - organization: Python Software Foundation position: Fellow - funding: - - value: 1100 - name: Omega Laser User's Group Travel Award - year: 2013 - - value: 1150 - name: NIF User's Group Travel Award - year: 2013 + organization: Python Software Foundation aka: - Scopatz - Scopatz, A - Scopatz, A. - Scopatz, A M - Anthony Michael Scopatz + avatar: https://avatars1.githubusercontent.com/u/320553?v=3&s=200 + education: + - begin_year: 2008 + location: Austin, TX + other: + - 'Adviser: Erich A. Schneider' + - 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis' + degree: Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program + end_year: 2011 + institution: The University of Texas at Austin + - begin_year: 2006 + location: Austin, TX + other: + - 'Adviser: Erich A. Schneider' + - 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership' + degree: M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program + end_year: 2007 + institution: The University of Texas at Austin + - begin_year: 2002 + location: Santa Barbara, CA + other: + - Graduated with a Major in Physics and a Minor in Mathematics + degree: B.S. Physics + end_year: 2006 + institution: University of California, Santa Barbara + position: professor email: scopatz@cec.sc.edu employment: - begin_year: 2015 - location: Columbia, SC other: - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' - 'PyNE: The Nuclear Engineering Toolkit.' - 'Website: http://www.ergs.sc.edu/' - organization: The University of South Carolina position: Assistant Professor, Mechanical Engineering Department - - begin_year: 2013 - end_year: 2015 - position: Associate Scientist, Engineering Physics Department + organization: The University of South Carolina + location: Columbia, SC + - end_year: 2015 + begin_year: 2013 + location: Madison, WI other: - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' - 'PyNE: The Nuclear Engineering Toolkit.' - 'Website: https://cnerg.github.io/' + position: Associate Scientist, Engineering Physics Department organization: CNERG, The University of Wisconsin-Madison - location: Madison, WI - end_month: May - begin_year: 2011 end_year: 2013 - position: Research Scientist, Postdoctoral Scholar begin_month: Nov + begin_year: 2011 + location: Chicago, IL other: - 'NIF: Simulation of magnetic field generation from neutral plasmas using FLASH.' @@ -146,55 +169,40 @@ YAML Example FLASH.' - 'OpacPlot: HDF5-based equation of state and opacity file format.' - 'Website: http://flash.uchicago.edu/site/' + position: Research Scientist, Postdoctoral Scholar organization: The FLASH Center, The University of Chicago - location: Chicago, IL skills: - level: expert - name: Python category: Programming Languages + name: Python - level: expert - name: Cython category: Programming Languages + name: Cython teaching: - - course: 'EMCH 552: Intro to Nuclear Engineering' - syllabus: https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc - position: Professor - month: August + - year: 2017 description: This course is an introduction to nuclear physics. + month: August organization: University of South Carolina - year: 2017 - - course: 'EMCH 558/758: Reactor Power Systems' - syllabus: https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing + syllabus: https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc position: Professor - month: January + course: 'EMCH 552: Intro to Nuclear Engineering' + - year: 2017 description: This course covers conventional reactors. + month: January organization: University of South Carolina - year: 2017 - education: - - other: - - 'Adviser: Erich A. Schneider' - - 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis' - end_year: 2011 - institution: The University of Texas at Austin - degree: Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program - begin_year: 2008 - location: Austin, TX - - other: - - 'Adviser: Erich A. Schneider' - - 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership' - end_year: 2007 - institution: The University of Texas at Austin - degree: M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program - begin_year: 2006 - location: Austin, TX - - other: - - Graduated with a Major in Physics and a Minor in Mathematics - end_year: 2006 - institution: University of California, Santa Barbara - degree: B.S. Physics - begin_year: 2002 - location: Santa Barbara, CA + syllabus: https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing + position: Professor + course: 'EMCH 558/758: Reactor Power Systems' + bio: Anthony Scopatz is currently an Assistant Professor title: Dr. + name: Anthony Scopatz + funding: + - name: Omega Laser User's Group Travel Award + year: 2013 + value: 1100 + - name: NIF User's Group Travel Award + year: 2013 + value: 1150 JSON/Mongo Example diff --git a/docs/collections/projects.rst b/docs/collections/projects.rst index 91150e4ce..f25bddc8e 100644 --- a/docs/collections/projects.rst +++ b/docs/collections/projects.rst @@ -14,12 +14,13 @@ The following lists key names mapped to its type and meaning for each entry. :other: ['list', 'string'], other information about the project, optional :repo: string, URL of the source code repo, if available, optional :team: list, People who are/have been woking on this project., required + :end_month: string, optional + :name: string, optional + :begin_month: string, optional :begin_year: integer, optional - :end_year: integer, optional :position: string, optional - :begin_month: string, optional - :name: string, optional + :end_year: integer, optional :website: string, URL of the website., required @@ -29,21 +30,21 @@ YAML Example .. code-block:: yaml Cyclus: - other: - - Discrete facilities with discrete material transactions - - Low barrier to entry, rapid payback to adoption logo: http://fuelcycle.org/_static/big_c.png - website: http://fuelcycle.org/ description: Agent-Based Nuclear Fuel Cycle Simulator + name: Cyclus team: - end_month: July - begin_year: 2013 + name: Anthony Scopatz end_year: 2015 - position: Project Lead begin_month: June - name: Anthony Scopatz + begin_year: 2013 + position: Project Lead + website: http://fuelcycle.org/ + other: + - Discrete facilities with discrete material transactions + - Low barrier to entry, rapid payback to adoption repo: https://github.com/cyclus/cyclus/ - name: Cyclus JSON/Mongo Example diff --git a/docs/collections/proposals.rst b/docs/collections/proposals.rst index 45a7ec1f7..1d1a30166 100644 --- a/docs/collections/proposals.rst +++ b/docs/collections/proposals.rst @@ -26,33 +26,33 @@ YAML Example .. code-block:: yaml mypropsal: - durration: 3 - pre: + day: 18 + full: + benefit_of_collaboration: http://pdf.com/benefit_of_collaboration + narrative: http://some.com/pdf cv: - http://pdf.com/scopatz-cv - http://pdf.com/flanagan-cv - narrative: http://some.com/pdf - month: Aug - benefit_of_collaboration: http://pdf.com/benefit_of_collaboration - day: 2 - year: 1998 - month: Aug + ammount: 1000000.0 status: submitted - day: 18 - full: + durration: 3 + year: 1999 + title: A very fine proposal indeed + pre: + benefit_of_collaboration: http://pdf.com/benefit_of_collaboration narrative: http://some.com/pdf + day: 2 cv: - http://pdf.com/scopatz-cv - http://pdf.com/flanagan-cv - benefit_of_collaboration: http://pdf.com/benefit_of_collaboration - ammount: 1000000.0 - pi: Anthony Scopatz + month: Aug + year: 1998 currency: USD authors: - Anthony Scopatz - Robert Flanagan - year: 1999 - title: A very fine proposal indeed + pi: Anthony Scopatz + month: Aug JSON/Mongo Example diff --git a/docs/collections/students.rst b/docs/collections/students.rst index 9a8bdb318..39549511d 100644 --- a/docs/collections/students.rst +++ b/docs/collections/students.rst @@ -8,6 +8,7 @@ The following lists key names mapped to its type and meaning for each entry. :_id: string, short represntation, such as this-is-my-name, required :aka: ('list', 'string'), list of aliases, optional + :email: string, email address, optional :university_id: string, The university identifier for the student, optional diff --git a/docs/conf.py b/docs/conf.py index 3a5bd745c..a819f740e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -298,6 +298,8 @@ def format_key(schema, key, indent_str=''): description=schema[key].get('description', ''), type=schema[key].get('type', '')), indent_str) s = s.replace(', , ', ', ') + if schema.get('schema', False): + s += '\n' for inner_key in schema.get('schema', ()): s += format_key(schema['schema'], inner_key, indent_str=indent_str + '\t') From c026c1b54d8a89a9d23b1233ae15b55d6c386ed4 Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 15:59:55 -0500 Subject: [PATCH 18/29] DOC: remove auto docs, add to their own auto folder, add auto folder to gitignore --- .gitignore | 5 +- docs/collections/abstracts.rst | 56 ----- docs/collections/assignments.rst | 60 ------ docs/collections/blog.rst | 48 ----- docs/collections/grades.rst | 47 ---- docs/collections/grants.rst | 100 --------- docs/collections/index.rst | 18 +- docs/collections/people.rst | 360 ------------------------------- docs/collections/projects.rst | 76 ------- docs/collections/proposals.rst | 97 --------- docs/collections/students.rst | 40 ---- docs/conf.py | 8 +- 12 files changed, 15 insertions(+), 900 deletions(-) delete mode 100644 docs/collections/abstracts.rst delete mode 100644 docs/collections/assignments.rst delete mode 100644 docs/collections/blog.rst delete mode 100644 docs/collections/grades.rst delete mode 100644 docs/collections/grants.rst delete mode 100644 docs/collections/people.rst delete mode 100644 docs/collections/projects.rst delete mode 100644 docs/collections/proposals.rst delete mode 100644 docs/collections/students.rst diff --git a/.gitignore b/.gitignore index b8aca4966..a3a4f1f84 100644 --- a/.gitignore +++ b/.gitignore @@ -57,4 +57,7 @@ docs/_build/ target/ # Rever -rever/ \ No newline at end of file +rever/ + +# Auto sphinx docs +docs/collections/auto \ No newline at end of file diff --git a/docs/collections/abstracts.rst b/docs/collections/abstracts.rst deleted file mode 100644 index f21d3c6e8..000000000 --- a/docs/collections/abstracts.rst +++ /dev/null @@ -1,56 +0,0 @@ -Abstracts -========= -Abstracts for a conference or workshop. This is generally public information - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: string, Unique identifier for submission. This generally includes the author name and part of the title., required -:coauthors: string, names of coauthors, optional -:email: string, contact email for the author., required -:firstname: string, first name of the author., required -:institution: string, name of the inistitution, required -:lastname: string, last name of the author., required -:references: string, HTML string of reference for the abstract itself, optional -:text: string, HTML string of the abstract., required -:timestamp: string, The time when the abstract was submitted., required -:title: string, title of the presentation/paper., required - - -YAML Example ------------- - -.. code-block:: yaml - - Mouginot.Model: - institution: University of Wisconsin-Madison - firstname: Baptiste - email: mouginot@wisc.edu - title: Model Performance Analysis - references: '[1] B. MOUGINOT, “cyCLASS: CLASS models for Cyclus,”, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 - (2016).' - coauthors: P.P.H. Wilson - timestamp: 5/5/2017 13:15:59 - text: The CLASS team has developed high quality predictors based on pre-trained - neural network... - lastname: Mouginot - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - { - "_id": "Mouginot.Model", - "coauthors": "P.P.H. Wilson", - "email": "mouginot@wisc.edu", - "firstname": "Baptiste", - "institution": "University of Wisconsin-Madison", - "lastname": "Mouginot", - "references": "[1] B. MOUGINOT, \u201ccyCLASS: CLASS models for Cyclus,\u201d, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 (2016).", - "text": "The CLASS team has developed high quality predictors based on pre-trained neural network...", - "timestamp": "5/5/2017 13:15:59", - "title": "Model Performance Analysis" - } diff --git a/docs/collections/assignments.rst b/docs/collections/assignments.rst deleted file mode 100644 index e6f1040f8..000000000 --- a/docs/collections/assignments.rst +++ /dev/null @@ -1,60 +0,0 @@ -Assignments -=========== -Information about assignments for classes. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: string, A unique id for the assignment, such a HW01-EMCH-558-2016-S, required -:category: string, such as 'homework' or 'final', required -:courses: string, ids of the courses that have this assignment, required -:file: string, path to assignment file in store, optional -:points: ('integer', 'float'), list of number of points possible for each question. Length is the number of questions, required -:question: string, titles for the questions on this assignment, optional -:solution: string, path to solution file in store, optional - - -YAML Example ------------- - -.. code-block:: yaml - - hw01-rx-power: - courses: - - EMCH-558-2016-S - - EMCH-758-2016-S - category: homework - points: - - 1 - - 2 - - 3 - questions: - - 1-9 - - 1-10 - - 1-12 - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - { - "_id": "hw01-rx-power", - "category": "homework", - "courses": [ - "EMCH-558-2016-S", - "EMCH-758-2016-S" - ], - "points": [ - 1, - 2, - 3 - ], - "questions": [ - "1-9", - "1-10", - "1-12" - ] - } diff --git a/docs/collections/blog.rst b/docs/collections/blog.rst deleted file mode 100644 index d5f955518..000000000 --- a/docs/collections/blog.rst +++ /dev/null @@ -1,48 +0,0 @@ -Blog -==== -This collection represents blog posts written by the members of the research group. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: string, short represntation, such as this-is-my-title, required -:author: string, name or AKA of author, required -:day: integer, Publication day, required -:month: string, Publication month, required -:original: string, URL of original post, if this is a repost, optional -:post: string, actual contents of the post, required -:title: string, full human readable title, required -:year: integer, Publication year, required - - -YAML Example ------------- - -.. code-block:: yaml - - my-vision: - year: 2015 - title: My Vision - day: 18 - post: I would like see things move forward. Deep, I know! - author: Anthony Scopatz - original: https://scopatz.com/my-vision/ - month: September - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - { - "_id": "my-vision", - "author": "Anthony Scopatz", - "day": 18, - "month": "September", - "original": "https://scopatz.com/my-vision/", - "post": "I would like see things move forward. Deep, I know!", - "title": "My Vision", - "year": 2015 - } diff --git a/docs/collections/grades.rst b/docs/collections/grades.rst deleted file mode 100644 index 72825019a..000000000 --- a/docs/collections/grades.rst +++ /dev/null @@ -1,47 +0,0 @@ -Grades -====== -The grade for a student on an assignment. This information should be private. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: string, unique id, typically the student-assignment-course, required -:assignment: string, assignment id, required -:course: string, course id, required -:filename: string, path to file in store, optional -:scores: ('integer', 'float'), the number of points earned on each question, required -:student: string, student id, required - - -YAML Example ------------- - -.. code-block:: yaml - - Human A. Person-rx-power-hw02-EMCH-758-2017-S: - student: hap - course: EMCH-758-2017-S - scores: - - 1 - - 1.6 - - 3 - assignment: 2017-rx-power-hw02 - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - { - "_id": "Human A. Person-rx-power-hw02-EMCH-758-2017-S", - "assignment": "2017-rx-power-hw02", - "course": "EMCH-758-2017-S", - "scores": [ - 1, - 1.6, - 3 - ], - "student": "hap" - } diff --git a/docs/collections/grants.rst b/docs/collections/grants.rst deleted file mode 100644 index e46416652..000000000 --- a/docs/collections/grants.rst +++ /dev/null @@ -1,100 +0,0 @@ -Grants -====== -This collection represents grants that have been awarded to the group. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: ('string', 'integer', 'float'), short representation, such as this-is-my-name, required -:amount: ('integer', 'float'), value of award, required -:begin_day: integer, start day of the grant, optional -:begin_month: string, start month of the grant, required -:begin_year: integer, start year of the grant, required -:benefit_of_collaboration: string, optional -:call_for_proposals: string, optional -:currency: string, typically '$' or 'USD', optional -:end_day: ('string', 'integer'), end day of the grant, optional -:end_month": string, end month of the grant, optional -:end_year: integer, end year of the grant, required -:funder: string, the agency funding the work, required -:grant_id: string, the identfier for this work, optional -:narrative: string, optional -:program: string, the program the work was funded under, required -:status: string, status of the grant, required -:team: list, information about the team members participating in the grant., required - - - :position: string, optional - :cv: string, optional - :name: string, optional - :subaward_amount: ('integer', 'float'), optional - :institution: string, optional - - :position: string, optional - :cv: string, optional - :name: string, optional - :subaward_amount: ('integer', 'float'), optional - :institution: string, optional -:title: string, actual title of proposal / grant, required - - -YAML Example ------------- - -.. code-block:: yaml - - SymPy-1.1: - amount: 3000.0 - begin_month: May - begin_year: 2017 - team: - - name: Anthony Scopatz - position: PI - institution: University of South Carolina - - name: Aaron Meurer - position: researcher - institution: University of South Carolina - call_for_proposals: https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ - end_year: 2017 - end_month: December - end_day: 31 - narrative: https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing - title: SymPy 1.1 Release Support - funder: NumFOCUS - program: Small Development Grants - begin_day: 1 - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - { - "_id": "SymPy-1.1", - "amount": 3000.0, - "begin_day": 1, - "begin_month": "May", - "begin_year": 2017, - "call_for_proposals": "https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ", - "end_day": 31, - "end_month": "December", - "end_year": 2017, - "funder": "NumFOCUS", - "narrative": "https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing", - "program": "Small Development Grants", - "team": [ - { - "institution": "University of South Carolina", - "name": "Anthony Scopatz", - "position": "PI" - }, - { - "institution": "University of South Carolina", - "name": "Aaron Meurer", - "position": "researcher" - } - ], - "title": "SymPy 1.1 Release Support" - } diff --git a/docs/collections/index.rst b/docs/collections/index.rst index 953f66474..74483c10a 100644 --- a/docs/collections/index.rst +++ b/docs/collections/index.rst @@ -8,16 +8,16 @@ The following contain the regolith schemas and examples in both YAML and JSON/Mo .. toctree:: :maxdepth: 1 - abstracts - assignments - blog + auto/abstracts + auto/assignments + auto/blog citations courses - grades - grants + auto/grades + auto/grants jobs news - people - projects - proposals - students + auto/people + auto/projects + auto/proposals + auto/students diff --git a/docs/collections/people.rst b/docs/collections/people.rst deleted file mode 100644 index 62448829f..000000000 --- a/docs/collections/people.rst +++ /dev/null @@ -1,360 +0,0 @@ -People -====== -This collection describes the members of the research group. This is normally public -data. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: string, unique identifier for the group member, required -:active: boolean, If the person is an active member, default True., optional -:aka: ['string', 'list'], list of aliases (also-known-as), useful for identifying the group member in citations or elsewhere., required -:avatar: string, URL to avatar, required -:bio: string, short biographical text, required -:collab: boolean, If the person is a collaborator, default False., optional -:education: list, This contains the educational information for the group member., required - - :end_month: string, optional - :gpa: ('float', 'string'), optional - :begin_month: string, optional - :location: string, optional - :begin_year: integer, optional - :other: string, optional - :degree: string, optional - :end_year: integer, optional - :institution: string, optional -:email: string, email address of the group member, optional -:employment: list, Employment information, similar to educational information., required - - :end_month: string, optional - :position: string, optional - :begin_month: string, optional - :location: string, optional - :begin_year: integer, optional - :other: string, optional - :end_year: integer, optional - :organization: string, optional -:funding: list, Funding and scholarship that the group member has individually obtained in the past. **WARNING:** this is not to be confused with the **grants** collection, optional - - :year: integer, optional - :name: string, optional - :duration: string, optional - :currency: string, optional - :value: ('float', 'integer'), optional - :month: string, optional -:honors: list, Honors that have been awarded to this group member, optional - - :name: string, optional - :year: integer, optional - :description: string, optional - :month: string, optional -:membership: list, Profesional organizations this member is a part of, optional - - :end_month: string, optional - :description: string, optional - :begin_month: string, optional - :begin_year: integer, optional - :position: string, optional - :website: string, optional - :end_year: integer, optional - :organization: string, optional -:name: string, Full, canonical name for the person, required -:position: string, such as professor, graduate student, or scientist, required -:service: list, Service that this group member has provided, optional - - :name: string, optional - :year: integer, optional - :description: string, optional - :month: string, optional - :duration: string, optional -:skills: list, Skill the group member has, optional - - :name: string, optional - :category: string, optional - :level: string, optional -:teaching: list, Courses that this group member has taught, if any, optional - - :description: string, optional - :end_year: integer, optional - :website: string, optional - :position: string, optional - :end_month: string, optional - :video: string, optional - :year: integer, optional - :course: string, optional - :month: string, optional - :syllabus: string, optional - :materials: string, optional - :organization: string, optional -:title: string, for example, Dr., etc., optional - - -YAML Example ------------- - -.. code-block:: yaml - - scopatz: - membership: - - begin_year: 2006 - position: Member - organization: American Nuclear Society - - begin_year: 2013 - position: Fellow - organization: Python Software Foundation - aka: - - Scopatz - - Scopatz, A - - Scopatz, A. - - Scopatz, A M - - Anthony Michael Scopatz - avatar: https://avatars1.githubusercontent.com/u/320553?v=3&s=200 - education: - - begin_year: 2008 - location: Austin, TX - other: - - 'Adviser: Erich A. Schneider' - - 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis' - degree: Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program - end_year: 2011 - institution: The University of Texas at Austin - - begin_year: 2006 - location: Austin, TX - other: - - 'Adviser: Erich A. Schneider' - - 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership' - degree: M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program - end_year: 2007 - institution: The University of Texas at Austin - - begin_year: 2002 - location: Santa Barbara, CA - other: - - Graduated with a Major in Physics and a Minor in Mathematics - degree: B.S. Physics - end_year: 2006 - institution: University of California, Santa Barbara - position: professor - email: scopatz@cec.sc.edu - employment: - - begin_year: 2015 - other: - - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' - - 'PyNE: The Nuclear Engineering Toolkit.' - - 'Website: http://www.ergs.sc.edu/' - position: Assistant Professor, Mechanical Engineering Department - organization: The University of South Carolina - location: Columbia, SC - - end_year: 2015 - begin_year: 2013 - location: Madison, WI - other: - - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' - - 'PyNE: The Nuclear Engineering Toolkit.' - - 'Website: https://cnerg.github.io/' - position: Associate Scientist, Engineering Physics Department - organization: CNERG, The University of Wisconsin-Madison - - end_month: May - end_year: 2013 - begin_month: Nov - begin_year: 2011 - location: Chicago, IL - other: - - 'NIF: Simulation of magnetic field generation from neutral plasmas using - FLASH.' - - 'CosmoB: Simulation of magnetic field generation from neutral plasmas using - FLASH.' - - 'FLASH4: High-energy density physics capabilities and utilities.' - - 'Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from - FLASH.' - - 'OpacPlot: HDF5-based equation of state and opacity file format.' - - 'Website: http://flash.uchicago.edu/site/' - position: Research Scientist, Postdoctoral Scholar - organization: The FLASH Center, The University of Chicago - skills: - - level: expert - category: Programming Languages - name: Python - - level: expert - category: Programming Languages - name: Cython - teaching: - - year: 2017 - description: This course is an introduction to nuclear physics. - month: August - organization: University of South Carolina - syllabus: https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc - position: Professor - course: 'EMCH 552: Intro to Nuclear Engineering' - - year: 2017 - description: This course covers conventional reactors. - month: January - organization: University of South Carolina - syllabus: https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing - position: Professor - course: 'EMCH 558/758: Reactor Power Systems' - bio: Anthony Scopatz is currently an Assistant Professor - title: Dr. - name: Anthony Scopatz - funding: - - name: Omega Laser User's Group Travel Award - year: 2013 - value: 1100 - - name: NIF User's Group Travel Award - year: 2013 - value: 1150 - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - { - "_id": "scopatz", - "aka": [ - "Scopatz", - "Scopatz, A", - "Scopatz, A.", - "Scopatz, A M", - "Anthony Michael Scopatz" - ], - "avatar": "https://avatars1.githubusercontent.com/u/320553?v=3&s=200", - "bio": "Anthony Scopatz is currently an Assistant Professor", - "education": [ - { - "begin_year": 2008, - "degree": "Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program", - "end_year": 2011, - "institution": "The University of Texas at Austin", - "location": "Austin, TX", - "other": [ - "Adviser: Erich A. Schneider", - "Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis" - ] - }, - { - "begin_year": 2006, - "degree": "M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program", - "end_year": 2007, - "institution": "The University of Texas at Austin", - "location": "Austin, TX", - "other": [ - "Adviser: Erich A. Schneider", - "Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership" - ] - }, - { - "begin_year": 2002, - "degree": "B.S. Physics", - "end_year": 2006, - "institution": "University of California, Santa Barbara", - "location": "Santa Barbara, CA", - "other": [ - "Graduated with a Major in Physics and a Minor in Mathematics" - ] - } - ], - "email": "scopatz@cec.sc.edu", - "employment": [ - { - "begin_year": 2015, - "location": "Columbia, SC", - "organization": "The University of South Carolina", - "other": [ - "Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.", - "PyNE: The Nuclear Engineering Toolkit.", - "Website: http://www.ergs.sc.edu/" - ], - "position": "Assistant Professor, Mechanical Engineering Department" - }, - { - "begin_year": 2013, - "end_year": 2015, - "location": "Madison, WI", - "organization": "CNERG, The University of Wisconsin-Madison", - "other": [ - "Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.", - "PyNE: The Nuclear Engineering Toolkit.", - "Website: https://cnerg.github.io/" - ], - "position": "Associate Scientist, Engineering Physics Department" - }, - { - "begin_month": "Nov", - "begin_year": 2011, - "end_month": "May", - "end_year": 2013, - "location": "Chicago, IL", - "organization": "The FLASH Center, The University of Chicago", - "other": [ - "NIF: Simulation of magnetic field generation from neutral plasmas using FLASH.", - "CosmoB: Simulation of magnetic field generation from neutral plasmas using FLASH.", - "FLASH4: High-energy density physics capabilities and utilities.", - "Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from FLASH.", - "OpacPlot: HDF5-based equation of state and opacity file format.", - "Website: http://flash.uchicago.edu/site/" - ], - "position": "Research Scientist, Postdoctoral Scholar" - } - ], - "funding": [ - { - "name": "Omega Laser User's Group Travel Award", - "value": 1100, - "year": 2013 - }, - { - "name": "NIF User's Group Travel Award", - "value": 1150, - "year": 2013 - } - ], - "membership": [ - { - "begin_year": 2006, - "organization": "American Nuclear Society", - "position": "Member" - }, - { - "begin_year": 2013, - "organization": "Python Software Foundation", - "position": "Fellow" - } - ], - "name": "Anthony Scopatz", - "position": "professor", - "skills": [ - { - "category": "Programming Languages", - "level": "expert", - "name": "Python" - }, - { - "category": "Programming Languages", - "level": "expert", - "name": "Cython" - } - ], - "teaching": [ - { - "course": "EMCH 552: Intro to Nuclear Engineering", - "description": "This course is an introduction to nuclear physics.", - "month": "August", - "organization": "University of South Carolina", - "position": "Professor", - "syllabus": "https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc", - "year": 2017 - }, - { - "course": "EMCH 558/758: Reactor Power Systems", - "description": "This course covers conventional reactors.", - "month": "January", - "organization": "University of South Carolina", - "position": "Professor", - "syllabus": "https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing", - "year": 2017 - } - ], - "title": "Dr." - } diff --git a/docs/collections/projects.rst b/docs/collections/projects.rst deleted file mode 100644 index f25bddc8e..000000000 --- a/docs/collections/projects.rst +++ /dev/null @@ -1,76 +0,0 @@ -Projects -======== -This collection describes the research group projects. This is normally public data. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: string, Unique project identifier., required -:description: string, brief project description., required -:grant: string, Grant id if there is a grant supporting this project, optional -:logo: string, URL to the project logo, optional -:name: string, name of the project., required -:other: ['list', 'string'], other information about the project, optional -:repo: string, URL of the source code repo, if available, optional -:team: list, People who are/have been woking on this project., required - - :end_month: string, optional - :name: string, optional - :begin_month: string, optional - :begin_year: integer, optional - :position: string, optional - :end_year: integer, optional -:website: string, URL of the website., required - - -YAML Example ------------- - -.. code-block:: yaml - - Cyclus: - logo: http://fuelcycle.org/_static/big_c.png - description: Agent-Based Nuclear Fuel Cycle Simulator - name: Cyclus - team: - - end_month: July - name: Anthony Scopatz - end_year: 2015 - begin_month: June - begin_year: 2013 - position: Project Lead - website: http://fuelcycle.org/ - other: - - Discrete facilities with discrete material transactions - - Low barrier to entry, rapid payback to adoption - repo: https://github.com/cyclus/cyclus/ - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - { - "_id": "Cyclus", - "description": "Agent-Based Nuclear Fuel Cycle Simulator", - "logo": "http://fuelcycle.org/_static/big_c.png", - "name": "Cyclus", - "other": [ - "Discrete facilities with discrete material transactions", - "Low barrier to entry, rapid payback to adoption" - ], - "repo": "https://github.com/cyclus/cyclus/", - "team": [ - { - "begin_month": "June", - "begin_year": 2013, - "end_month": "July", - "end_year": 2015, - "name": "Anthony Scopatz", - "position": "Project Lead" - } - ], - "website": "http://fuelcycle.org/" - } diff --git a/docs/collections/proposals.rst b/docs/collections/proposals.rst deleted file mode 100644 index 1d1a30166..000000000 --- a/docs/collections/proposals.rst +++ /dev/null @@ -1,97 +0,0 @@ -Proposals -========= -This collection represents proposals that have been submitted by the group. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: ('string', 'integer', 'float'), short represntation, such as this-is-my-name, required -:ammount: ('integer', 'float'), value of award, required -:authors: string, other investigator names, required -:currency: string, typically '$' or 'USD', required -:day: integer, day that the proposal is due, required -:durration: ('integer', 'float'), number of years, required -:month: string, month that the proposal is due, required -:pi: string, principal investigator name, required -:pre: dict, Information about the pre-proposal, optional -:status: string, e.g. 'submitted', 'accepted', 'rejected', required -:title: string, actual title of proposal, required -:year: integer, Year that the proposal is due, required - - -YAML Example ------------- - -.. code-block:: yaml - - mypropsal: - day: 18 - full: - benefit_of_collaboration: http://pdf.com/benefit_of_collaboration - narrative: http://some.com/pdf - cv: - - http://pdf.com/scopatz-cv - - http://pdf.com/flanagan-cv - ammount: 1000000.0 - status: submitted - durration: 3 - year: 1999 - title: A very fine proposal indeed - pre: - benefit_of_collaboration: http://pdf.com/benefit_of_collaboration - narrative: http://some.com/pdf - day: 2 - cv: - - http://pdf.com/scopatz-cv - - http://pdf.com/flanagan-cv - month: Aug - year: 1998 - currency: USD - authors: - - Anthony Scopatz - - Robert Flanagan - pi: Anthony Scopatz - month: Aug - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - { - "_id": "mypropsal", - "ammount": 1000000.0, - "authors": [ - "Anthony Scopatz", - "Robert Flanagan" - ], - "currency": "USD", - "day": 18, - "durration": 3, - "full": { - "benefit_of_collaboration": "http://pdf.com/benefit_of_collaboration", - "cv": [ - "http://pdf.com/scopatz-cv", - "http://pdf.com/flanagan-cv" - ], - "narrative": "http://some.com/pdf" - }, - "month": "Aug", - "pi": "Anthony Scopatz", - "pre": { - "benefit_of_collaboration": "http://pdf.com/benefit_of_collaboration", - "cv": [ - "http://pdf.com/scopatz-cv", - "http://pdf.com/flanagan-cv" - ], - "day": 2, - "month": "Aug", - "narrative": "http://some.com/pdf", - "year": 1998 - }, - "status": "submitted", - "title": "A very fine proposal indeed", - "year": 1999 - } diff --git a/docs/collections/students.rst b/docs/collections/students.rst deleted file mode 100644 index 39549511d..000000000 --- a/docs/collections/students.rst +++ /dev/null @@ -1,40 +0,0 @@ -Students -======== -This is a collection of student names and metadata. This should probably be private. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: string, short represntation, such as this-is-my-name, required -:aka: ('list', 'string'), list of aliases, optional - -:email: string, email address, optional -:university_id: string, The university identifier for the student, optional - - -YAML Example ------------- - -.. code-block:: yaml - - Human A. Person: - aka: - - H. A. Person - email: haperson@uni.edu - university_id: HAP42 - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - { - "_id": "Human A. Person", - "aka": [ - "H. A. Person" - ], - "email": "haperson@uni.edu", - "university_id": "HAP42" - } diff --git a/docs/conf.py b/docs/conf.py index a819f740e..059ce9728 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,7 +11,6 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os import json import tempfile from textwrap import indent @@ -277,9 +276,6 @@ 'proposals': 'This collection represents proposals that have been submitted by the group.\n\n', 'students': 'This is a collection of student names and metadata. This should probably be private.\n\n'} -from pprint import pprint -import string - def format_key(schema, key, indent_str=''): s = '' @@ -308,7 +304,7 @@ def format_key(schema, key, indent_str=''): def build_schema_doc(key): - fn = 'collections/' + key + '.rst' + fn = 'collections/auto/' + key + '.rst' with open(fn, 'w') as f: s = '' s += key.title() + '\n' @@ -326,7 +322,7 @@ def build_schema_doc(key): temp = tempfile.NamedTemporaryFile() temp2 = tempfile.NamedTemporaryFile() with open(temp.name, 'w') as ff: - json.dump(EXEMPLARS[key], ff) + json.dump(EXEMPLARS[key], ff, sort_keys=True) jd = json.dumps(EXEMPLARS[key], sort_keys=True, indent=4, separators=(',', ': ')) json_to_yaml(temp.name, temp2.name) From 55d36195cdf0dbb6d8f6aaf6f70283a1cbf13ce3 Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 16:03:31 -0500 Subject: [PATCH 19/29] FIX: comments from @scopatz --- docs/collections/blank.rst | 21 - docs/conf.py | 3 +- regolith/schemas.py | 959 +++++++++++++++++++------------------ 3 files changed, 481 insertions(+), 502 deletions(-) delete mode 100644 docs/collections/blank.rst diff --git a/docs/collections/blank.rst b/docs/collections/blank.rst deleted file mode 100644 index 061df6465..000000000 --- a/docs/collections/blank.rst +++ /dev/null @@ -1,21 +0,0 @@ - -============ - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str, - - -YAML Example ------------- - -.. code-block:: yaml - - -JSON/Mongo Example ------------------- - -.. code-block:: json - diff --git a/docs/conf.py b/docs/conf.py index 059ce9728..04c6f2e62 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -79,7 +79,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build', 'collections/blank.rst'] +exclude_patterns = ['_build', ] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None @@ -263,7 +263,6 @@ schema_top_docs = { 'abstracts': 'Abstracts for a conference or workshop. This is generally public information\n\n', 'assignments': 'Information about assignments for classes.\n\n', - 'blank': '\n', 'blog': 'This collection represents blog posts written by the members of the research group.\n\n', 'citations': 'This collection should contain bibtex equivalent fields. Additionally,\nthe keys ``"entrytype"`` denotes things like ``ARTICLE``, and ``"_id"`` denotes\nthe entry identifier. Furthermore, the ``"author"`` key should be a list of\nstrings. See the Python project `BibtexParser `_\nfor more information.\n\n', 'courses': 'This is a collection that describes a course, when it happened, and\nwho is taking it. This is likely private.\n\n', diff --git a/regolith/schemas.py b/regolith/schemas.py index 9efe10ae5..6f4a9b201 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -1,485 +1,6 @@ """Database schemas, examples, and tools""" from cerberus import Validator -SCHEMAS = { - 'grants': { - '_id': - {'description': 'short representation, such as this-is-my-name', - 'required': True, - 'type': ('string', 'integer', 'float')}, - 'amount': {'description': 'value of award', - 'required': True, - 'type': ('integer', 'float')}, - 'begin_day': {'description': 'start day of the grant', - 'required': False, - 'type': 'integer'}, - 'begin_month': {'description': 'start month of the grant', - 'required': True, - 'type': 'string'}, - 'begin_year': {'description': 'start year of the grant', - 'required': True, - 'type': 'integer'}, - 'benefit_of_collaboration': {'description': '', - 'required': False, - 'type': 'string'}, - 'call_for_proposals': {'description': '', - 'required': False, - 'type': 'string'}, - 'currency': {'description': "typically '$' or 'USD'", - 'required': False, - 'type': 'string'}, - 'end_day': {'description': 'end day of the grant', - 'required': False, - 'type': ('string', 'integer')}, - 'end_month"': {'description': 'end month of the grant', - 'required': False, - 'type': 'string'}, - 'end_year': {'description': 'end year of the grant', - 'required': True, - 'type': 'integer'}, - 'funder': {'description': 'the agency funding the work', - 'required': True, - 'type': 'string'}, - 'grant_id': {'description': 'the identfier for this work', - 'required': False, - 'type': 'string'}, - 'narrative': { - 'description': '', - 'required': False, - 'type': 'string'}, - 'program': {'description': 'the program the work was funded under', - 'required': True, - 'type': 'string'}, - 'team': { - 'description': 'information about the team members participating ' - 'in the grant.', - 'required': True, - 'type': 'list', - 'schema': {'type': 'dict', - 'schema': {'cv': {'required': False, 'type': 'string'}, - 'institution': { - 'required': True, - 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'position': { - 'required': True, - 'type': 'string'}, - 'subaward_amount': { - 'required': False, - 'type': ('integer', 'float')}}}}, - 'title': {'description': 'actual title of proposal / grant', - 'required': True, - 'type': 'string'}, - 'status': {'description': 'status of the grant', - 'required': True, - 'type': 'string', - 'allowed': ['pending', 'declined', 'accepted', 'in-prep']} - }, - 'abstracts': { - '_id': { - 'description': 'Unique identifier for submission. This generally ' - 'includes the author name and part of the title.', - 'required': True, - 'type': 'string'}, - 'coauthors': {'description': 'names of coauthors', - 'required': False, - 'type': 'string'}, - 'email': {'description': 'contact email for the author.', - 'required': True, - 'type': 'string'}, - 'firstname': {'description': 'first name of the author.', - 'required': True, - 'type': 'string'}, - 'institution': {'description': 'name of the inistitution', - 'required': True, - 'type': 'string'}, - 'lastname': {'description': 'last name of the author.', - 'required': True, - 'type': 'string'}, - 'references': { - 'description': 'HTML string of reference for the abstract itself', - 'required': False, - 'type': 'string'}, - 'text': {'description': 'HTML string of the abstract.', - 'required': True, - 'type': 'string'}, - 'timestamp': { - 'description': 'The time when the abstract was submitted.', - 'required': True, - 'type': 'string'}, - 'title': {'description': 'title of the presentation/paper.', - 'required': True, - 'type': 'string'}}, - 'blog': { - '_id': {'description': 'short represntation, such as this-is-my-title', - 'required': True, - 'type': 'string'}, - 'author': {'description': 'name or AKA of author', - 'required': True, - 'type': 'string'}, - 'day': { - 'description': 'Publication day', - 'required': True, - 'type': 'integer'}, - 'month': { - 'description': 'Publication month', - 'required': True, - 'type': 'string'}, - 'original': { - 'description': 'URL of original post, if this is a repost', - 'required': False, - 'type': 'string'}, - 'post': {'description': 'actual contents of the post', - 'required': True, - 'type': 'string'}, - 'title': {'description': 'full human readable title', - 'required': True, - 'type': 'string'}, - 'year': { - 'description': 'Publication year', - 'required': True, - 'type': 'integer'}}, - 'grades': { - '_id': { - 'description': 'unique id, typically the ' - 'student-assignment-course', - 'required': True, - 'type': 'string'}, - 'assignment': { - 'description': 'assignment id', - 'required': True, - 'type': 'string'}, - 'course': { - 'description': 'course id', - 'required': True, - 'type': 'string'}, - 'filename': {'description': 'path to file in store', - 'required': False, - 'type': 'string'}, - 'scores': { - 'description': 'the number of points earned on each question', - 'required': True, - 'type': ('integer', 'float')}, - 'student': { - 'description': 'student id', - 'required': True, - 'type': 'string'}}, - 'assignments': { - '_id': { - 'description': 'A unique id for the assignment, such a ' - 'HW01-EMCH-558-2016-S', - 'required': True, - 'type': 'string'}, - 'category': {'description': "such as 'homework' or 'final'", - 'required': True, - 'type': 'string'}, - 'courses': { - 'description': 'ids of the courses that have this assignment', - 'required': True, - 'type': 'string'}, - 'file': {'description': 'path to assignment file in store', - 'required': False, - 'type': 'string'}, - 'points': { - 'description': 'list of number of points possible for each ' - 'question. Length is the number of questions', - 'required': True, - 'type': ('integer', 'float')}, - 'question': { - 'description': 'titles for the questions on this assignment', - 'required': False, - 'type': 'string'}, - 'solution': {'description': 'path to solution file in store', - 'required': False, - 'type': 'string'}}, - 'projects': {'_id': {'description': 'Unique project identifier.', - 'required': True, - 'type': 'string'}, - 'description': {'description': 'brief project description.', - 'required': True, - 'type': 'string'}, - 'logo': {'description': 'URL to the project logo', - 'required': False, - 'type': 'string'}, - 'name': {'description': 'name of the project.', - 'required': True, - 'type': 'string'}, - 'other': { - 'description': 'other information about the project', - 'required': False, - 'type': ['list', 'string']}, - 'repo': { - 'description': 'URL of the source code repo, if ' - 'available', - 'required': False, - 'type': 'string'}, - 'team': { - 'description': 'People who are/have been woking on this ' - 'project.', - 'required': True, - 'type': 'list', - 'schema': { - 'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'name': {'required': True, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}} - }, - 'website': {'description': 'URL of the website.', - 'required': True, - 'type': 'string'}, - 'grant': {'description': 'Grant id if there is a grant ' - 'supporting this project', - 'type': 'string'}}, - 'proposals': { - '_id': {'description': 'short represntation, such as this-is-my-name', - 'required': True, - 'type': ('string', 'integer', 'float')}, - 'ammount': {'description': 'value of award', - 'required': True, - 'type': ('integer', 'float')}, - 'authors': {'description': 'other investigator names', - 'required': True, - 'type': 'string'}, - 'currency': {'description': "typically '$' or 'USD'", - 'required': True, - 'type': 'string'}, - 'day': {'description': 'day that the proposal is due', - 'required': True, - 'type': 'integer'}, - 'durration': {'description': 'number of years', - 'required': True, - 'type': ('integer', 'float')}, - 'month': {'description': 'month that the proposal is due', - 'required': True, - 'type': 'string'}, - 'pi': {'description': 'principal investigator name', - 'required': True, - 'type': 'string'}, - # XXX: FIXME - 'pre': {'description': 'Information about the pre-proposal', - 'required': False, - 'type': 'dict'}, - 'status': {'description': "e.g. 'submitted', 'accepted', 'rejected'", - 'required': True, - 'type': 'string'}, - 'title': {'description': 'actual title of proposal', - 'required': True, - 'type': 'string'}, - 'year': {'description': 'Year that the proposal is due', - 'required': True, - 'type': 'integer'}}, - 'students': { - '_id': {'description': 'short represntation, such as this-is-my-name', - 'required': True, - 'type': 'string'}, - 'aka': {'description': 'list of aliases', - 'required': False, - 'type': ('list', 'string'), - 'schema': {'type': 'string'}}, - 'email': { - 'description': 'email address', - 'required': False, - 'type': 'string'}, - 'university_id': { - 'description': 'The university identifier for the student', - 'required': False, - 'type': 'string'}}, - 'people': {'_id': {'description': 'unique identifier for the group member', - 'required': True, - 'type': 'string'}, - 'active': { - 'description': 'If the person is an active member, ' - 'default True.', - 'required': False, - 'type': 'boolean'}, - 'aka': { - 'description': 'list of aliases (also-known-as), useful ' - 'for identifying the group member in ' - 'citations or elsewhere.', - 'required': True, - 'type': ['string', 'list']}, - 'avatar': { - 'description': 'URL to avatar', - 'required': True, - 'type': 'string'}, - 'bio': {'description': 'short biographical text', - 'required': True, - 'type': 'string'}, - 'collab': { - 'description': 'If the person is a collaborator, default ' - 'False.', - 'required': False, - 'type': 'boolean'}, - 'education': { - 'description': 'This contains the educational information ' - 'for the group member.', - 'required': True, - 'type': 'list', - 'schema': { - 'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'degree': {'required': True, 'type': 'string'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': True, 'type': 'integer'}, - 'gpa': {'required': False, 'type': ('float', 'string')}, - 'institution': {'required': True, 'type': 'string'}, - 'location': {'required': True, 'type': 'string'}, - 'other': {'required': False, 'type': 'string'}} - }, - 'email': {'description': 'email address of the group member', - 'required': False, - 'type': 'string'}, - 'employment': { - 'description': 'Employment information, similar to ' - 'educational information.', - 'required': True, - 'type': 'list', - 'schema': { - 'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'location': {'required': True, 'type': 'string'}, - 'organization': {'required': True, 'type': 'string'}, - 'other': {'required': False, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}}}, - 'funding': { - 'description': 'Funding and scholarship that the group ' - 'member has individually obtained in the ' - 'past. **WARNING:** this is not to be ' - 'confused with the **grants** collection', - 'required': False, - 'type': 'list', - 'schema': { - 'currency': {'required': False, 'type': 'string'}, - 'duration': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'value': { - 'required': True, - 'type': ('float', 'integer')}, - 'year': {'required': True, 'type': 'integer'}} - }, - 'honors': { - 'description': 'Honors that have been awarded to this ' - 'group member', - 'required': False, - 'type': 'list', - 'schema': { - 'description': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'year': {'required': True, 'type': 'integer'}} - }, - 'membership': { - 'description': 'Profesional organizations this member is ' - 'a part of', - 'required': False, - 'type': 'list', - 'schema': { - 'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'description': {'required': False, 'type': 'string'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'organization': {'required': True, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}, - 'website': {'required': False, 'type': 'string'}}}, - 'name': {'description': 'Full, canonical name for the person', - 'required': True, - 'type': 'string'}, - 'position': { - 'description': 'such as professor, graduate student, ' - 'or scientist', - 'required': True, - 'type': 'string'}, - 'service': { - 'description': 'Service that this group member has provided', - 'required': False, - 'type': 'list', - 'schema': { - 'description': {'required': False, 'type': 'string'}, - 'duration': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'year': {'required': True, 'type': 'integer'}} - }, - 'skills': {'description': 'Skill the group member has', - 'required': False, - 'type': 'list', - 'schema': { - 'category': {'required': True, 'type': 'string'}, - 'level': {'required': True, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}} - }, - 'teaching': { - 'description': 'Courses that this group member has ' - 'taught, if any', - 'required': False, - 'type': 'list', - 'schema': {'course': {'required': True, 'type': 'string'}, - 'description': { - 'required': False, - 'type': 'string'}, - 'end_month': { - 'required': False, - 'type': 'string'}, - 'end_year': { - 'required': False, - 'type': 'integer'}, - 'materials': { - 'required': False, - 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'organization': { - 'required': True, - 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}, - 'syllabus': { - 'required': False, - 'type': 'string'}, - 'video': {'required': False, 'type': 'string'}, - 'website': {'required': False, 'type': 'string'}, - 'year': {'required': True, 'type': 'integer'}} - }, - 'title': {'description': 'for example, Dr., etc.', - 'required': False, - 'type': 'string'} - } -} - - -class MyValidator(Validator): - def _validate_description(self, description, field, value): - if False: - self._error(field, "Shouldn't be here") - - -def validate(db, record): - """Validate a record for a given db - - Parameters - ---------- - db : str - The name of the db in question - record : dict - The record to be validated - - Returns - ------- - rtn : bool - True is valid - errors: dict - The errors encountered (if any) - - """ - schema = SCHEMAS[db] - v = MyValidator(schema) - return v.validate(record), v.errors - - EXEMPLARS = {'abstracts': {'_id': 'Mouginot.Model', 'coauthors': 'P.P.H. Wilson', 'email': 'mouginot@wisc.edu', @@ -760,3 +281,483 @@ def validate(db, record): 'aka': ['H. A. Person'], 'email': 'haperson@uni.edu', 'university_id': 'HAP42'}} + +SCHEMAS = { + 'grants': { + '_id': + {'description': 'short representation, such as this-is-my-name', + 'required': True, + 'type': ('string', 'integer', 'float')}, + 'amount': {'description': 'value of award', + 'required': True, + 'type': ('integer', 'float')}, + 'begin_day': {'description': 'start day of the grant', + 'required': False, + 'type': 'integer'}, + 'begin_month': {'description': 'start month of the grant', + 'required': True, + 'type': 'string'}, + 'begin_year': {'description': 'start year of the grant', + 'required': True, + 'type': 'integer'}, + 'benefit_of_collaboration': {'description': '', + 'required': False, + 'type': 'string'}, + 'call_for_proposals': {'description': '', + 'required': False, + 'type': 'string'}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': False, + 'type': 'string'}, + 'end_day': {'description': 'end day of the grant', + 'required': False, + 'type': ('string', 'integer')}, + 'end_month"': {'description': 'end month of the grant', + 'required': False, + 'type': 'string'}, + 'end_year': {'description': 'end year of the grant', + 'required': True, + 'type': 'integer'}, + 'funder': {'description': 'the agency funding the work', + 'required': True, + 'type': 'string'}, + 'grant_id': {'description': 'the identfier for this work', + 'required': False, + 'type': 'string'}, + 'narrative': { + 'description': '', + 'required': False, + 'type': 'string'}, + 'program': {'description': 'the program the work was funded under', + 'required': True, + 'type': 'string'}, + 'team': { + 'description': 'information about the team members participating ' + 'in the grant.', + 'required': True, + 'type': 'list', + 'schema': {'type': 'dict', + 'schema': {'cv': {'required': False, 'type': 'string'}, + 'institution': { + 'required': True, + 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'position': { + 'required': True, + 'type': 'string'}, + 'subaward_amount': { + 'required': False, + 'type': ('integer', 'float')}}}}, + 'title': {'description': 'actual title of proposal / grant', + 'required': True, + 'type': 'string'}, + 'status': {'description': 'status of the grant', + 'required': True, + 'type': 'string', + 'allowed': ['pending', 'declined', 'accepted', 'in-prep']} + }, + 'abstracts': { + '_id': { + 'description': 'Unique identifier for submission. This generally ' + 'includes the author name and part of the title.', + 'required': True, + 'type': 'string'}, + 'coauthors': {'description': 'names of coauthors', + 'required': False, + 'type': 'string'}, + 'email': {'description': 'contact email for the author.', + 'required': True, + 'type': 'string'}, + 'firstname': {'description': 'first name of the author.', + 'required': True, + 'type': 'string'}, + 'institution': {'description': 'name of the inistitution', + 'required': True, + 'type': 'string'}, + 'lastname': {'description': 'last name of the author.', + 'required': True, + 'type': 'string'}, + 'references': { + 'description': 'HTML string of reference for the abstract itself', + 'required': False, + 'type': 'string'}, + 'text': {'description': 'HTML string of the abstract.', + 'required': True, + 'type': 'string'}, + 'timestamp': { + 'description': 'The time when the abstract was submitted.', + 'required': True, + 'type': 'string'}, + 'title': {'description': 'title of the presentation/paper.', + 'required': True, + 'type': 'string'}}, + 'blog': { + '_id': {'description': 'short represntation, such as this-is-my-title', + 'required': True, + 'type': 'string'}, + 'author': {'description': 'name or AKA of author', + 'required': True, + 'type': 'string'}, + 'day': { + 'description': 'Publication day', + 'required': True, + 'type': 'integer'}, + 'month': { + 'description': 'Publication month', + 'required': True, + 'type': 'string'}, + 'original': { + 'description': 'URL of original post, if this is a repost', + 'required': False, + 'type': 'string'}, + 'post': {'description': 'actual contents of the post', + 'required': True, + 'type': 'string'}, + 'title': {'description': 'full human readable title', + 'required': True, + 'type': 'string'}, + 'year': { + 'description': 'Publication year', + 'required': True, + 'type': 'integer'}}, + 'grades': { + '_id': { + 'description': 'unique id, typically the ' + 'student-assignment-course', + 'required': True, + 'type': 'string'}, + 'assignment': { + 'description': 'assignment id', + 'required': True, + 'type': 'string'}, + 'course': { + 'description': 'course id', + 'required': True, + 'type': 'string'}, + 'filename': {'description': 'path to file in store', + 'required': False, + 'type': 'string'}, + 'scores': { + 'description': 'the number of points earned on each question', + 'required': True, + 'type': ('integer', 'float')}, + 'student': { + 'description': 'student id', + 'required': True, + 'type': 'string'}}, + 'assignments': { + '_id': { + 'description': 'A unique id for the assignment, such a ' + 'HW01-EMCH-558-2016-S', + 'required': True, + 'type': 'string'}, + 'category': {'description': "such as 'homework' or 'final'", + 'required': True, + 'type': 'string'}, + 'courses': { + 'description': 'ids of the courses that have this assignment', + 'required': True, + 'type': 'string'}, + 'file': {'description': 'path to assignment file in store', + 'required': False, + 'type': 'string'}, + 'points': { + 'description': 'list of number of points possible for each ' + 'question. Length is the number of questions', + 'required': True, + 'type': ('integer', 'float')}, + 'question': { + 'description': 'titles for the questions on this assignment', + 'required': False, + 'type': 'string'}, + 'solution': {'description': 'path to solution file in store', + 'required': False, + 'type': 'string'}}, + 'projects': {'_id': {'description': 'Unique project identifier.', + 'required': True, + 'type': 'string'}, + 'description': {'description': 'brief project description.', + 'required': True, + 'type': 'string'}, + 'logo': {'description': 'URL to the project logo', + 'required': False, + 'type': 'string'}, + 'name': {'description': 'name of the project.', + 'required': True, + 'type': 'string'}, + 'other': { + 'description': 'other information about the project', + 'required': False, + 'type': ['list', 'string']}, + 'repo': { + 'description': 'URL of the source code repo, if ' + 'available', + 'required': False, + 'type': 'string'}, + 'team': { + 'description': 'People who are/have been woking on this ' + 'project.', + 'required': True, + 'type': 'list', + 'schema': { + 'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'name': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}} + }, + 'website': {'description': 'URL of the website.', + 'required': True, + 'type': 'string'}, + 'grant': {'description': 'Grant id if there is a grant ' + 'supporting this project', + 'type': 'string'}}, + 'proposals': { + '_id': {'description': 'short represntation, such as this-is-my-name', + 'required': True, + 'type': ('string', 'integer', 'float')}, + 'ammount': {'description': 'value of award', + 'required': True, + 'type': ('integer', 'float')}, + 'authors': {'description': 'other investigator names', + 'required': True, + 'type': 'string'}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': True, + 'type': 'string'}, + 'day': {'description': 'day that the proposal is due', + 'required': True, + 'type': 'integer'}, + 'durration': {'description': 'number of years', + 'required': True, + 'type': ('integer', 'float')}, + 'month': {'description': 'month that the proposal is due', + 'required': True, + 'type': 'string'}, + 'pi': {'description': 'principal investigator name', + 'required': True, + 'type': 'string'}, + # XXX: FIXME + 'pre': {'description': 'Information about the pre-proposal', + 'required': False, + 'type': 'dict'}, + 'status': {'description': "e.g. 'submitted', 'accepted', 'rejected'", + 'required': True, + 'type': 'string'}, + 'title': {'description': 'actual title of proposal', + 'required': True, + 'type': 'string'}, + 'year': {'description': 'Year that the proposal is due', + 'required': True, + 'type': 'integer'}}, + 'students': { + '_id': {'description': 'short represntation, such as this-is-my-name', + 'required': True, + 'type': 'string'}, + 'aka': {'description': 'list of aliases', + 'required': False, + 'type': ('list', 'string'), + 'schema': {'type': 'string'}}, + 'email': { + 'description': 'email address', + 'required': False, + 'type': 'string'}, + 'university_id': { + 'description': 'The university identifier for the student', + 'required': False, + 'type': 'string'}}, + 'people': {'_id': {'description': 'unique identifier for the group member', + 'required': True, + 'type': 'string'}, + 'active': { + 'description': 'If the person is an active member, ' + 'default True.', + 'required': False, + 'type': 'boolean'}, + 'aka': { + 'description': 'list of aliases (also-known-as), useful ' + 'for identifying the group member in ' + 'citations or elsewhere.', + 'required': True, + 'type': ['string', 'list']}, + 'avatar': { + 'description': 'URL to avatar', + 'required': True, + 'type': 'string'}, + 'bio': {'description': 'short biographical text', + 'required': True, + 'type': 'string'}, + 'collab': { + 'description': 'If the person is a collaborator, default ' + 'False.', + 'required': False, + 'type': 'boolean'}, + 'education': { + 'description': 'This contains the educational information ' + 'for the group member.', + 'required': True, + 'type': 'list', + 'schema': { + 'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'degree': {'required': True, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': True, 'type': 'integer'}, + 'gpa': {'required': False, 'type': ('float', 'string')}, + 'institution': {'required': True, 'type': 'string'}, + 'location': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}} + }, + 'email': {'description': 'email address of the group member', + 'required': False, + 'type': 'string'}, + 'employment': { + 'description': 'Employment information, similar to ' + 'educational information.', + 'required': True, + 'type': 'list', + 'schema': { + 'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'location': {'required': True, 'type': 'string'}, + 'organization': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}}}, + 'funding': { + 'description': 'Funding and scholarship that the group ' + 'member has individually obtained in the ' + 'past. **WARNING:** this is not to be ' + 'confused with the **grants** collection', + 'required': False, + 'type': 'list', + 'schema': { + 'currency': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'value': { + 'required': True, + 'type': ('float', 'integer')}, + 'year': {'required': True, 'type': 'integer'}} + }, + 'honors': { + 'description': 'Honors that have been awarded to this ' + 'group member', + 'required': False, + 'type': 'list', + 'schema': { + 'description': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}} + }, + 'membership': { + 'description': 'Profesional organizations this member is ' + 'a part of', + 'required': False, + 'type': 'list', + 'schema': { + 'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'description': {'required': False, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'organization': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}}}, + 'name': {'description': 'Full, canonical name for the person', + 'required': True, + 'type': 'string'}, + 'position': { + 'description': 'such as professor, graduate student, ' + 'or scientist', + 'required': True, + 'type': 'string'}, + 'service': { + 'description': 'Service that this group member has provided', + 'required': False, + 'type': 'list', + 'schema': { + 'description': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}} + }, + 'skills': {'description': 'Skill the group member has', + 'required': False, + 'type': 'list', + 'schema': { + 'category': {'required': True, 'type': 'string'}, + 'level': {'required': True, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}} + }, + 'teaching': { + 'description': 'Courses that this group member has ' + 'taught, if any', + 'required': False, + 'type': 'list', + 'schema': {'course': {'required': True, 'type': 'string'}, + 'description': { + 'required': False, + 'type': 'string'}, + 'end_month': { + 'required': False, + 'type': 'string'}, + 'end_year': { + 'required': False, + 'type': 'integer'}, + 'materials': { + 'required': False, + 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'organization': { + 'required': True, + 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'syllabus': { + 'required': False, + 'type': 'string'}, + 'video': {'required': False, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}} + }, + 'title': {'description': 'for example, Dr., etc.', + 'required': False, + 'type': 'string'} + } +} + + +class NoDescriptionValidator(Validator): + def _validate_description(self, description, field, value): + if False: + pass + + +def validate(db, record): + """Validate a record for a given db + + Parameters + ---------- + db : str + The name of the db in question + record : dict + The record to be validated + + Returns + ------- + rtn : bool + True is valid + errors: dict + The errors encountered (if any) + + """ + schema = SCHEMAS[db] + v = NoDescriptionValidator(schema) + return v.validate(record), v.errors + + From 1394df2a539da1bfd53501caaeb98e7ff46f871b Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 16:04:59 -0500 Subject: [PATCH 20/29] ENH: move top docs to schemas module --- docs/conf.py | 17 +---------------- regolith/schemas.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 04c6f2e62..648ca0abd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,7 +19,7 @@ from regolith import __version__ as REGOLITH_VERSION from regolith.fsclient import json_to_yaml -from regolith.schemas import SCHEMAS, EXEMPLARS +from regolith.schemas import SCHEMAS, EXEMPLARS, schema_top_docs # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -260,21 +260,6 @@ # How to display URL addresses: 'footnote', 'no', or 'inline'. # texinfo_show_urls = 'footnote' -schema_top_docs = { - 'abstracts': 'Abstracts for a conference or workshop. This is generally public information\n\n', - 'assignments': 'Information about assignments for classes.\n\n', - 'blog': 'This collection represents blog posts written by the members of the research group.\n\n', - 'citations': 'This collection should contain bibtex equivalent fields. Additionally,\nthe keys ``"entrytype"`` denotes things like ``ARTICLE``, and ``"_id"`` denotes\nthe entry identifier. Furthermore, the ``"author"`` key should be a list of\nstrings. See the Python project `BibtexParser `_\nfor more information.\n\n', - 'courses': 'This is a collection that describes a course, when it happened, and\nwho is taking it. This is likely private.\n\n', - 'grades': 'The grade for a student on an assignment. This information should be private.\n\n', - 'grants': 'This collection represents grants that have been awarded to the group.\n\n', - 'jobs': 'This collection describes the research group jobs. This is normally public data.\n\n\n', - 'news': 'This collection describes the research group news. This is normally public data.\n\n', - 'people': 'This collection describes the members of the research group. This is normally public\ndata.\n\n', - 'projects': 'This collection describes the research group projects. This is normally public data.\n\n', - 'proposals': 'This collection represents proposals that have been submitted by the group.\n\n', - 'students': 'This is a collection of student names and metadata. This should probably be private.\n\n'} - def format_key(schema, key, indent_str=''): s = '' diff --git a/regolith/schemas.py b/regolith/schemas.py index 6f4a9b201..a7f4d696b 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -761,3 +761,17 @@ def validate(db, record): return v.validate(record), v.errors +schema_top_docs = { + 'abstracts': 'Abstracts for a conference or workshop. This is generally public information\n\n', + 'assignments': 'Information about assignments for classes.\n\n', + 'blog': 'This collection represents blog posts written by the members of the research group.\n\n', + 'citations': 'This collection should contain bibtex equivalent fields. Additionally,\nthe keys ``"entrytype"`` denotes things like ``ARTICLE``, and ``"_id"`` denotes\nthe entry identifier. Furthermore, the ``"author"`` key should be a list of\nstrings. See the Python project `BibtexParser `_\nfor more information.\n\n', + 'courses': 'This is a collection that describes a course, when it happened, and\nwho is taking it. This is likely private.\n\n', + 'grades': 'The grade for a student on an assignment. This information should be private.\n\n', + 'grants': 'This collection represents grants that have been awarded to the group.\n\n', + 'jobs': 'This collection describes the research group jobs. This is normally public data.\n\n\n', + 'news': 'This collection describes the research group news. This is normally public data.\n\n', + 'people': 'This collection describes the members of the research group. This is normally public\ndata.\n\n', + 'projects': 'This collection describes the research group projects. This is normally public data.\n\n', + 'proposals': 'This collection represents proposals that have been submitted by the group.\n\n', + 'students': 'This is a collection of student names and metadata. This should probably be private.\n\n'} \ No newline at end of file From 61ffe55255cbf3ca874ef71eadd8be9356396a85 Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 16:05:39 -0500 Subject: [PATCH 21/29] STY: pep8 line width --- regolith/schemas.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/regolith/schemas.py b/regolith/schemas.py index a7f4d696b..c65215934 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -762,16 +762,33 @@ def validate(db, record): schema_top_docs = { - 'abstracts': 'Abstracts for a conference or workshop. This is generally public information\n\n', + 'abstracts': 'Abstracts for a conference or workshop. This is generally ' + 'public information\n\n', 'assignments': 'Information about assignments for classes.\n\n', - 'blog': 'This collection represents blog posts written by the members of the research group.\n\n', - 'citations': 'This collection should contain bibtex equivalent fields. Additionally,\nthe keys ``"entrytype"`` denotes things like ``ARTICLE``, and ``"_id"`` denotes\nthe entry identifier. Furthermore, the ``"author"`` key should be a list of\nstrings. See the Python project `BibtexParser `_\nfor more information.\n\n', - 'courses': 'This is a collection that describes a course, when it happened, and\nwho is taking it. This is likely private.\n\n', - 'grades': 'The grade for a student on an assignment. This information should be private.\n\n', - 'grants': 'This collection represents grants that have been awarded to the group.\n\n', - 'jobs': 'This collection describes the research group jobs. This is normally public data.\n\n\n', - 'news': 'This collection describes the research group news. This is normally public data.\n\n', - 'people': 'This collection describes the members of the research group. This is normally public\ndata.\n\n', - 'projects': 'This collection describes the research group projects. This is normally public data.\n\n', - 'proposals': 'This collection represents proposals that have been submitted by the group.\n\n', - 'students': 'This is a collection of student names and metadata. This should probably be private.\n\n'} \ No newline at end of file + 'blog': 'This collection represents blog posts written by the members of ' + 'the research group.\n\n', + 'citations': 'This collection should contain bibtex equivalent fields. ' + 'Additionally,\nthe keys ``"entrytype"`` denotes things ' + 'like ``ARTICLE``, and ``"_id"`` denotes\nthe entry ' + 'identifier. Furthermore, the ``"author"`` key should be a ' + 'list of\nstrings. See the Python project `BibtexParser ' + '`_\nfor more ' + 'information.\n\n', + 'courses': 'This is a collection that describes a course, when it ' + 'happened, and\nwho is taking it. This is likely private.\n\n', + 'grades': 'The grade for a student on an assignment. This information ' + 'should be private.\n\n', + 'grants': 'This collection represents grants that have been awarded to ' + 'the group.\n\n', + 'jobs': 'This collection describes the research group jobs. This is ' + 'normally public data.\n\n\n', + 'news': 'This collection describes the research group news. This is ' + 'normally public data.\n\n', + 'people': 'This collection describes the members of the research group. ' + 'This is normally public\ndata.\n\n', + 'projects': 'This collection describes the research group projects. This ' + 'is normally public data.\n\n', + 'proposals': 'This collection represents proposals that have been ' + 'submitted by the group.\n\n', + 'students': 'This is a collection of student names and metadata. This ' + 'should probably be private.\n\n'} \ No newline at end of file From 9c3c8d64eaf0ca8f046d9bebae1458b36272a4f4 Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 16:05:52 -0500 Subject: [PATCH 22/29] STY: pep8 newline --- regolith/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regolith/schemas.py b/regolith/schemas.py index c65215934..3e10b97a9 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -791,4 +791,4 @@ def validate(db, record): 'proposals': 'This collection represents proposals that have been ' 'submitted by the group.\n\n', 'students': 'This is a collection of student names and metadata. This ' - 'should probably be private.\n\n'} \ No newline at end of file + 'should probably be private.\n\n'} From 79b75a94147ad6a53781afd7494ba9da0ad35360 Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 16:20:49 -0500 Subject: [PATCH 23/29] ENH: alph. yamls and added top docs to schema --- docs/conf.py | 7 +- regolith/fsclient.py | 9 +- regolith/schemas.py | 830 ++++++++++++++++++------------------------- 3 files changed, 360 insertions(+), 486 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 648ca0abd..ebdc9972a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,7 +19,7 @@ from regolith import __version__ as REGOLITH_VERSION from regolith.fsclient import json_to_yaml -from regolith.schemas import SCHEMAS, EXEMPLARS, schema_top_docs +from regolith.schemas import SCHEMAS, EXEMPLARS # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -293,13 +293,14 @@ def build_schema_doc(key): s = '' s += key.title() + '\n' s += '=' * len(key) + '\n' - s += schema_top_docs[key] + s += SCHEMAS[key]['_description']['description'] s += 'Schema\n------\nThe following lists key names mapped to its type and meaning for each entry.\n\n' schema = SCHEMAS[key] schema_list = list(schema.keys()) schema_list.sort() for k in schema_list: - s += format_key(schema[k], key=k) + if k not in ['_description']: + s += format_key(schema[k], key=k) s += '\n\n' s += 'YAML Example\n------------\n\n' s += '.. code-block:: yaml\n\n' diff --git a/regolith/fsclient.py b/regolith/fsclient.py index ba4639ca4..69d56d24b 100644 --- a/regolith/fsclient.py +++ b/regolith/fsclient.py @@ -6,6 +6,7 @@ from collections import defaultdict from ruamel.yaml import YAML +import ruamel.yaml from regolith.tools import dbdirname, dbpathname @@ -48,10 +49,14 @@ def dump_yaml(filename, docs, inst=None): """Dumps a dict of documents into a file.""" inst = YAML() if inst is None else inst inst.indent(mapping=2, sequence=4, offset=2) - for doc in docs.values(): + my_sorted_dict = ruamel.yaml.comments.CommentedMap() + for k, doc in docs.items(): _id = doc.pop('_id') + my_sorted_dict[k] = ruamel.yaml.comments.CommentedMap() + for kk in sorted(doc.keys()): + my_sorted_dict[k][kk] = doc[kk] with open(filename, 'w') as fh: - inst.dump(docs, stream=fh) + inst.dump(my_sorted_dict, stream=fh) def json_to_yaml(inp, out): diff --git a/regolith/schemas.py b/regolith/schemas.py index 3e10b97a9..9ba550fa4 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -282,454 +282,355 @@ 'email': 'haperson@uni.edu', 'university_id': 'HAP42'}} -SCHEMAS = { - 'grants': { - '_id': - {'description': 'short representation, such as this-is-my-name', - 'required': True, - 'type': ('string', 'integer', 'float')}, - 'amount': {'description': 'value of award', - 'required': True, - 'type': ('integer', 'float')}, - 'begin_day': {'description': 'start day of the grant', - 'required': False, - 'type': 'integer'}, - 'begin_month': {'description': 'start month of the grant', - 'required': True, - 'type': 'string'}, - 'begin_year': {'description': 'start year of the grant', - 'required': True, - 'type': 'integer'}, - 'benefit_of_collaboration': {'description': '', - 'required': False, - 'type': 'string'}, - 'call_for_proposals': {'description': '', - 'required': False, - 'type': 'string'}, - 'currency': {'description': "typically '$' or 'USD'", - 'required': False, - 'type': 'string'}, - 'end_day': {'description': 'end day of the grant', - 'required': False, - 'type': ('string', 'integer')}, - 'end_month"': {'description': 'end month of the grant', - 'required': False, - 'type': 'string'}, - 'end_year': {'description': 'end year of the grant', - 'required': True, - 'type': 'integer'}, - 'funder': {'description': 'the agency funding the work', - 'required': True, - 'type': 'string'}, - 'grant_id': {'description': 'the identfier for this work', - 'required': False, - 'type': 'string'}, - 'narrative': { - 'description': '', - 'required': False, - 'type': 'string'}, - 'program': {'description': 'the program the work was funded under', - 'required': True, - 'type': 'string'}, - 'team': { - 'description': 'information about the team members participating ' - 'in the grant.', - 'required': True, - 'type': 'list', - 'schema': {'type': 'dict', - 'schema': {'cv': {'required': False, 'type': 'string'}, - 'institution': { - 'required': True, - 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'position': { - 'required': True, - 'type': 'string'}, - 'subaward_amount': { - 'required': False, - 'type': ('integer', 'float')}}}}, - 'title': {'description': 'actual title of proposal / grant', - 'required': True, - 'type': 'string'}, - 'status': {'description': 'status of the grant', - 'required': True, - 'type': 'string', - 'allowed': ['pending', 'declined', 'accepted', 'in-prep']} - }, - 'abstracts': { - '_id': { - 'description': 'Unique identifier for submission. This generally ' - 'includes the author name and part of the title.', - 'required': True, - 'type': 'string'}, - 'coauthors': {'description': 'names of coauthors', - 'required': False, - 'type': 'string'}, - 'email': {'description': 'contact email for the author.', - 'required': True, - 'type': 'string'}, - 'firstname': {'description': 'first name of the author.', - 'required': True, - 'type': 'string'}, - 'institution': {'description': 'name of the inistitution', - 'required': True, - 'type': 'string'}, - 'lastname': {'description': 'last name of the author.', - 'required': True, - 'type': 'string'}, - 'references': { - 'description': 'HTML string of reference for the abstract itself', - 'required': False, - 'type': 'string'}, - 'text': {'description': 'HTML string of the abstract.', - 'required': True, - 'type': 'string'}, - 'timestamp': { - 'description': 'The time when the abstract was submitted.', - 'required': True, - 'type': 'string'}, - 'title': {'description': 'title of the presentation/paper.', - 'required': True, - 'type': 'string'}}, - 'blog': { - '_id': {'description': 'short represntation, such as this-is-my-title', - 'required': True, - 'type': 'string'}, - 'author': {'description': 'name or AKA of author', - 'required': True, - 'type': 'string'}, - 'day': { - 'description': 'Publication day', - 'required': True, - 'type': 'integer'}, - 'month': { - 'description': 'Publication month', - 'required': True, - 'type': 'string'}, - 'original': { - 'description': 'URL of original post, if this is a repost', - 'required': False, - 'type': 'string'}, - 'post': {'description': 'actual contents of the post', - 'required': True, - 'type': 'string'}, - 'title': {'description': 'full human readable title', - 'required': True, - 'type': 'string'}, - 'year': { - 'description': 'Publication year', - 'required': True, - 'type': 'integer'}}, - 'grades': { - '_id': { - 'description': 'unique id, typically the ' - 'student-assignment-course', - 'required': True, - 'type': 'string'}, - 'assignment': { - 'description': 'assignment id', - 'required': True, - 'type': 'string'}, - 'course': { - 'description': 'course id', - 'required': True, - 'type': 'string'}, - 'filename': {'description': 'path to file in store', - 'required': False, - 'type': 'string'}, - 'scores': { - 'description': 'the number of points earned on each question', - 'required': True, - 'type': ('integer', 'float')}, - 'student': { - 'description': 'student id', - 'required': True, - 'type': 'string'}}, - 'assignments': { - '_id': { - 'description': 'A unique id for the assignment, such a ' - 'HW01-EMCH-558-2016-S', - 'required': True, - 'type': 'string'}, - 'category': {'description': "such as 'homework' or 'final'", - 'required': True, - 'type': 'string'}, - 'courses': { - 'description': 'ids of the courses that have this assignment', - 'required': True, - 'type': 'string'}, - 'file': {'description': 'path to assignment file in store', - 'required': False, - 'type': 'string'}, - 'points': { - 'description': 'list of number of points possible for each ' - 'question. Length is the number of questions', - 'required': True, - 'type': ('integer', 'float')}, - 'question': { - 'description': 'titles for the questions on this assignment', - 'required': False, - 'type': 'string'}, - 'solution': {'description': 'path to solution file in store', - 'required': False, - 'type': 'string'}}, - 'projects': {'_id': {'description': 'Unique project identifier.', - 'required': True, - 'type': 'string'}, - 'description': {'description': 'brief project description.', - 'required': True, - 'type': 'string'}, - 'logo': {'description': 'URL to the project logo', - 'required': False, - 'type': 'string'}, - 'name': {'description': 'name of the project.', - 'required': True, - 'type': 'string'}, - 'other': { - 'description': 'other information about the project', - 'required': False, - 'type': ['list', 'string']}, - 'repo': { - 'description': 'URL of the source code repo, if ' - 'available', - 'required': False, - 'type': 'string'}, - 'team': { - 'description': 'People who are/have been woking on this ' - 'project.', - 'required': True, - 'type': 'list', - 'schema': { - 'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'name': {'required': True, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}} - }, - 'website': {'description': 'URL of the website.', - 'required': True, - 'type': 'string'}, - 'grant': {'description': 'Grant id if there is a grant ' - 'supporting this project', - 'type': 'string'}}, - 'proposals': { - '_id': {'description': 'short represntation, such as this-is-my-name', - 'required': True, - 'type': ('string', 'integer', 'float')}, - 'ammount': {'description': 'value of award', - 'required': True, - 'type': ('integer', 'float')}, - 'authors': {'description': 'other investigator names', - 'required': True, - 'type': 'string'}, - 'currency': {'description': "typically '$' or 'USD'", - 'required': True, - 'type': 'string'}, - 'day': {'description': 'day that the proposal is due', - 'required': True, - 'type': 'integer'}, - 'durration': {'description': 'number of years', - 'required': True, - 'type': ('integer', 'float')}, - 'month': {'description': 'month that the proposal is due', - 'required': True, - 'type': 'string'}, - 'pi': {'description': 'principal investigator name', - 'required': True, - 'type': 'string'}, - # XXX: FIXME - 'pre': {'description': 'Information about the pre-proposal', - 'required': False, - 'type': 'dict'}, - 'status': {'description': "e.g. 'submitted', 'accepted', 'rejected'", - 'required': True, - 'type': 'string'}, - 'title': {'description': 'actual title of proposal', - 'required': True, - 'type': 'string'}, - 'year': {'description': 'Year that the proposal is due', - 'required': True, - 'type': 'integer'}}, - 'students': { - '_id': {'description': 'short represntation, such as this-is-my-name', - 'required': True, - 'type': 'string'}, - 'aka': {'description': 'list of aliases', - 'required': False, - 'type': ('list', 'string'), - 'schema': {'type': 'string'}}, - 'email': { - 'description': 'email address', - 'required': False, - 'type': 'string'}, - 'university_id': { - 'description': 'The university identifier for the student', - 'required': False, - 'type': 'string'}}, - 'people': {'_id': {'description': 'unique identifier for the group member', - 'required': True, - 'type': 'string'}, - 'active': { - 'description': 'If the person is an active member, ' - 'default True.', - 'required': False, - 'type': 'boolean'}, - 'aka': { - 'description': 'list of aliases (also-known-as), useful ' - 'for identifying the group member in ' - 'citations or elsewhere.', - 'required': True, - 'type': ['string', 'list']}, - 'avatar': { - 'description': 'URL to avatar', - 'required': True, - 'type': 'string'}, - 'bio': {'description': 'short biographical text', - 'required': True, - 'type': 'string'}, - 'collab': { - 'description': 'If the person is a collaborator, default ' - 'False.', - 'required': False, - 'type': 'boolean'}, - 'education': { - 'description': 'This contains the educational information ' - 'for the group member.', - 'required': True, - 'type': 'list', - 'schema': { - 'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'degree': {'required': True, 'type': 'string'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': True, 'type': 'integer'}, - 'gpa': {'required': False, 'type': ('float', 'string')}, - 'institution': {'required': True, 'type': 'string'}, - 'location': {'required': True, 'type': 'string'}, - 'other': {'required': False, 'type': 'string'}} - }, - 'email': {'description': 'email address of the group member', - 'required': False, - 'type': 'string'}, - 'employment': { - 'description': 'Employment information, similar to ' - 'educational information.', - 'required': True, - 'type': 'list', - 'schema': { - 'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'location': {'required': True, 'type': 'string'}, - 'organization': {'required': True, 'type': 'string'}, - 'other': {'required': False, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}}}, - 'funding': { - 'description': 'Funding and scholarship that the group ' - 'member has individually obtained in the ' - 'past. **WARNING:** this is not to be ' - 'confused with the **grants** collection', - 'required': False, - 'type': 'list', - 'schema': { - 'currency': {'required': False, 'type': 'string'}, - 'duration': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'value': { - 'required': True, - 'type': ('float', 'integer')}, - 'year': {'required': True, 'type': 'integer'}} - }, - 'honors': { - 'description': 'Honors that have been awarded to this ' - 'group member', - 'required': False, - 'type': 'list', - 'schema': { - 'description': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'year': {'required': True, 'type': 'integer'}} - }, - 'membership': { - 'description': 'Profesional organizations this member is ' - 'a part of', - 'required': False, - 'type': 'list', - 'schema': { - 'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'description': {'required': False, 'type': 'string'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'organization': {'required': True, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}, - 'website': {'required': False, 'type': 'string'}}}, - 'name': {'description': 'Full, canonical name for the person', - 'required': True, - 'type': 'string'}, - 'position': { - 'description': 'such as professor, graduate student, ' - 'or scientist', - 'required': True, - 'type': 'string'}, - 'service': { - 'description': 'Service that this group member has provided', - 'required': False, - 'type': 'list', - 'schema': { - 'description': {'required': False, 'type': 'string'}, - 'duration': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'year': {'required': True, 'type': 'integer'}} - }, - 'skills': {'description': 'Skill the group member has', - 'required': False, - 'type': 'list', - 'schema': { - 'category': {'required': True, 'type': 'string'}, - 'level': {'required': True, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}} - }, - 'teaching': { - 'description': 'Courses that this group member has ' - 'taught, if any', - 'required': False, - 'type': 'list', - 'schema': {'course': {'required': True, 'type': 'string'}, - 'description': { - 'required': False, - 'type': 'string'}, - 'end_month': { - 'required': False, - 'type': 'string'}, - 'end_year': { - 'required': False, - 'type': 'integer'}, - 'materials': { - 'required': False, - 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'organization': { - 'required': True, - 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}, - 'syllabus': { - 'required': False, - 'type': 'string'}, - 'video': {'required': False, 'type': 'string'}, - 'website': {'required': False, 'type': 'string'}, - 'year': {'required': True, 'type': 'integer'}} - }, - 'title': {'description': 'for example, Dr., etc.', - 'required': False, - 'type': 'string'} - } -} +SCHEMAS = {'abstracts': {'_description': {'description': 'Abstracts for a conference or workshop. This is generally public information\n\n'}, + '_id': {'description': 'Unique identifier for submission. This generally includes the author name and part of the title.', + 'required': True, + 'type': 'string'}, + 'coauthors': {'description': 'names of coauthors', + 'required': False, + 'type': 'string'}, + 'email': {'description': 'contact email for the author.', + 'required': True, + 'type': 'string'}, + 'firstname': {'description': 'first name of the author.', + 'required': True, + 'type': 'string'}, + 'institution': {'description': 'name of the inistitution', + 'required': True, + 'type': 'string'}, + 'lastname': {'description': 'last name of the author.', + 'required': True, + 'type': 'string'}, + 'references': {'description': 'HTML string of reference for the abstract itself', + 'required': False, + 'type': 'string'}, + 'text': {'description': 'HTML string of the abstract.', + 'required': True, + 'type': 'string'}, + 'timestamp': {'description': 'The time when the abstract was submitted.', + 'required': True, + 'type': 'string'}, + 'title': {'description': 'title of the presentation/paper.', + 'required': True, + 'type': 'string'}}, + 'assignments': {'_description': {'description': 'Information about assignments for classes.\n\n'}, + '_id': {'description': 'A unique id for the assignment, such a HW01-EMCH-558-2016-S', + 'required': True, + 'type': 'string'}, + 'category': {'description': "such as 'homework' or 'final'", + 'required': True, + 'type': 'string'}, + 'courses': {'description': 'ids of the courses that have this assignment', + 'required': True, + 'type': 'string'}, + 'file': {'description': 'path to assignment file in store', + 'required': False, + 'type': 'string'}, + 'points': {'description': 'list of number of points possible for each question. Length is the number of questions', + 'required': True, + 'type': ('integer', 'float')}, + 'question': {'description': 'titles for the questions on this assignment', + 'required': False, + 'type': 'string'}, + 'solution': {'description': 'path to solution file in store', + 'required': False, + 'type': 'string'}}, + 'blog': {'_description': {'description': 'This collection represents blog posts written by the members of the research group.\n\n'}, + '_id': {'description': 'short represntation, such as this-is-my-title', + 'required': True, + 'type': 'string'}, + 'author': {'description': 'name or AKA of author', + 'required': True, + 'type': 'string'}, + 'day': {'description': 'Publication day', + 'required': True, + 'type': 'integer'}, + 'month': {'description': 'Publication month', + 'required': True, + 'type': 'string'}, + 'original': {'description': 'URL of original post, if this is a repost', + 'required': False, + 'type': 'string'}, + 'post': {'description': 'actual contents of the post', + 'required': True, + 'type': 'string'}, + 'title': {'description': 'full human readable title', + 'required': True, + 'type': 'string'}, + 'year': {'description': 'Publication year', + 'required': True, + 'type': 'integer'}}, + 'grades': {'_description': {'description': 'The grade for a student on an assignment. This information should be private.\n\n'}, + '_id': {'description': 'unique id, typically the student-assignment-course', + 'required': True, + 'type': 'string'}, + 'assignment': {'description': 'assignment id', + 'required': True, + 'type': 'string'}, + 'course': {'description': 'course id', 'required': True, 'type': 'string'}, + 'filename': {'description': 'path to file in store', + 'required': False, + 'type': 'string'}, + 'scores': {'description': 'the number of points earned on each question', + 'required': True, + 'type': ('integer', 'float')}, + 'student': {'description': 'student id', + 'required': True, + 'type': 'string'}}, + 'grants': {'_description': {'description': 'This collection represents grants that have been awarded to the group.\n\n'}, + '_id': {'description': 'short representation, such as this-is-my-name', + 'required': True, + 'type': ('string', 'integer', 'float')}, + 'amount': {'description': 'value of award', + 'required': True, + 'type': ('integer', 'float')}, + 'begin_day': {'description': 'start day of the grant', + 'required': False, + 'type': 'integer'}, + 'begin_month': {'description': 'start month of the grant', + 'required': True, + 'type': 'string'}, + 'begin_year': {'description': 'start year of the grant', + 'required': True, + 'type': 'integer'}, + 'benefit_of_collaboration': {'description': '', + 'required': False, + 'type': 'string'}, + 'call_for_proposals': {'description': '', + 'required': False, + 'type': 'string'}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': False, + 'type': 'string'}, + 'end_day': {'description': 'end day of the grant', + 'required': False, + 'type': ('string', 'integer')}, + 'end_month"': {'description': 'end month of the grant', + 'required': False, + 'type': 'string'}, + 'end_year': {'description': 'end year of the grant', + 'required': True, + 'type': 'integer'}, + 'funder': {'description': 'the agency funding the work', + 'required': True, + 'type': 'string'}, + 'grant_id': {'description': 'the identfier for this work', + 'required': False, + 'type': 'string'}, + 'narrative': {'description': '', 'required': False, 'type': 'string'}, + 'program': {'description': 'the program the work was funded under', + 'required': True, + 'type': 'string'}, + 'status': {'allowed': ['pending', 'declined', 'accepted', 'in-prep'], + 'description': 'status of the grant', + 'required': True, + 'type': 'string'}, + 'team': {'description': 'information about the team members participating in the grant.', + 'required': True, + 'schema': {'schema': {'cv': {'required': False, 'type': 'string'}, + 'institution': {'required': True, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'subaward_amount': {'required': False, 'type': ('integer', 'float')}}, + 'type': 'dict'}, + 'type': 'list'}, + 'title': {'description': 'actual title of proposal / grant', + 'required': True, + 'type': 'string'}}, + 'people': {'_description': {'description': 'This collection describes the members of the research group. This is normally public\ndata.\n\n'}, + '_id': {'description': 'unique identifier for the group member', + 'required': True, + 'type': 'string'}, + 'active': {'description': 'If the person is an active member, default True.', + 'required': False, + 'type': 'boolean'}, + 'aka': {'description': 'list of aliases (also-known-as), useful for identifying the group member in citations or elsewhere.', + 'required': True, + 'type': ['string', 'list']}, + 'avatar': {'description': 'URL to avatar', + 'required': True, + 'type': 'string'}, + 'bio': {'description': 'short biographical text', + 'required': True, + 'type': 'string'}, + 'collab': {'description': 'If the person is a collaborator, default False.', + 'required': False, + 'type': 'boolean'}, + 'education': {'description': 'This contains the educational information for the group member.', + 'required': True, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'degree': {'required': True, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': True, 'type': 'integer'}, + 'gpa': {'required': False, 'type': ('float', 'string')}, + 'institution': {'required': True, 'type': 'string'}, + 'location': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}}, + 'type': 'list'}, + 'email': {'description': 'email address of the group member', + 'required': False, + 'type': 'string'}, + 'employment': {'description': 'Employment information, similar to educational information.', + 'required': True, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'location': {'required': True, 'type': 'string'}, + 'organization': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}}, + 'type': 'list'}, + 'funding': {'description': 'Funding and scholarship that the group member has individually obtained in the past. **WARNING:** this is not to be confused with the **grants** collection', + 'required': False, + 'schema': {'currency': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'value': {'required': True, 'type': ('float', 'integer')}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'honors': {'description': 'Honors that have been awarded to this group member', + 'required': False, + 'schema': {'description': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'membership': {'description': 'Profesional organizations this member is a part of', + 'required': False, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'description': {'required': False, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'organization': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}}, + 'type': 'list'}, + 'name': {'description': 'Full, canonical name for the person', + 'required': True, + 'type': 'string'}, + 'position': {'description': 'such as professor, graduate student, or scientist', + 'required': True, + 'type': 'string'}, + 'service': {'description': 'Service that this group member has provided', + 'required': False, + 'schema': {'description': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'skills': {'description': 'Skill the group member has', + 'required': False, + 'schema': {'category': {'required': True, 'type': 'string'}, + 'level': {'required': True, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}}, + 'type': 'list'}, + 'teaching': {'description': 'Courses that this group member has taught, if any', + 'required': False, + 'schema': {'course': {'required': True, 'type': 'string'}, + 'description': {'required': False, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'materials': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'organization': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'syllabus': {'required': False, 'type': 'string'}, + 'video': {'required': False, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'title': {'description': 'for example, Dr., etc.', + 'required': False, + 'type': 'string'}}, + 'projects': {'_description': {'description': 'This collection describes the research group projects. This is normally public data.\n\n'}, + '_id': {'description': 'Unique project identifier.', + 'required': True, + 'type': 'string'}, + 'description': {'description': 'brief project description.', + 'required': True, + 'type': 'string'}, + 'grant': {'description': 'Grant id if there is a grant supporting this project', + 'type': 'string'}, + 'logo': {'description': 'URL to the project logo', + 'required': False, + 'type': 'string'}, + 'name': {'description': 'name of the project.', + 'required': True, + 'type': 'string'}, + 'other': {'description': 'other information about the project', + 'required': False, + 'type': ['list', 'string']}, + 'repo': {'description': 'URL of the source code repo, if available', + 'required': False, + 'type': 'string'}, + 'team': {'description': 'People who are/have been woking on this project.', + 'required': True, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'name': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}}, + 'type': 'list'}, + 'website': {'description': 'URL of the website.', + 'required': True, + 'type': 'string'}}, + 'proposals': {'_description': {'description': 'This collection represents proposals that have been submitted by the group.\n\n'}, + '_id': {'description': 'short represntation, such as this-is-my-name', + 'required': True, + 'type': ('string', 'integer', 'float')}, + 'ammount': {'description': 'value of award', + 'required': True, + 'type': ('integer', 'float')}, + 'authors': {'description': 'other investigator names', + 'required': True, + 'type': 'string'}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': True, + 'type': 'string'}, + 'day': {'description': 'day that the proposal is due', + 'required': True, + 'type': 'integer'}, + 'durration': {'description': 'number of years', + 'required': True, + 'type': ('integer', 'float')}, + 'month': {'description': 'month that the proposal is due', + 'required': True, + 'type': 'string'}, + 'pi': {'description': 'principal investigator name', + 'required': True, + 'type': 'string'}, + 'pre': {'description': 'Information about the pre-proposal', + 'required': False, + 'type': 'dict'}, + 'status': {'description': "e.g. 'submitted', 'accepted', 'rejected'", + 'required': True, + 'type': 'string'}, + 'title': {'description': 'actual title of proposal', + 'required': True, + 'type': 'string'}, + 'year': {'description': 'Year that the proposal is due', + 'required': True, + 'type': 'integer'}}, + 'students': {'_description': {'description': 'This is a collection of student names and metadata. This should probably be private.\n\n'}, + '_id': {'description': 'short represntation, such as this-is-my-name', + 'required': True, + 'type': 'string'}, + 'aka': {'description': 'list of aliases', + 'required': False, + 'schema': {'type': 'string'}, + 'type': ('list', 'string')}, + 'email': {'description': 'email address', + 'required': False, + 'type': 'string'}, + 'university_id': {'description': 'The university identifier for the student', + 'required': False, + 'type': 'string'}}} class NoDescriptionValidator(Validator): @@ -759,36 +660,3 @@ def validate(db, record): schema = SCHEMAS[db] v = NoDescriptionValidator(schema) return v.validate(record), v.errors - - -schema_top_docs = { - 'abstracts': 'Abstracts for a conference or workshop. This is generally ' - 'public information\n\n', - 'assignments': 'Information about assignments for classes.\n\n', - 'blog': 'This collection represents blog posts written by the members of ' - 'the research group.\n\n', - 'citations': 'This collection should contain bibtex equivalent fields. ' - 'Additionally,\nthe keys ``"entrytype"`` denotes things ' - 'like ``ARTICLE``, and ``"_id"`` denotes\nthe entry ' - 'identifier. Furthermore, the ``"author"`` key should be a ' - 'list of\nstrings. See the Python project `BibtexParser ' - '`_\nfor more ' - 'information.\n\n', - 'courses': 'This is a collection that describes a course, when it ' - 'happened, and\nwho is taking it. This is likely private.\n\n', - 'grades': 'The grade for a student on an assignment. This information ' - 'should be private.\n\n', - 'grants': 'This collection represents grants that have been awarded to ' - 'the group.\n\n', - 'jobs': 'This collection describes the research group jobs. This is ' - 'normally public data.\n\n\n', - 'news': 'This collection describes the research group news. This is ' - 'normally public data.\n\n', - 'people': 'This collection describes the members of the research group. ' - 'This is normally public\ndata.\n\n', - 'projects': 'This collection describes the research group projects. This ' - 'is normally public data.\n\n', - 'proposals': 'This collection represents proposals that have been ' - 'submitted by the group.\n\n', - 'students': 'This is a collection of student names and metadata. This ' - 'should probably be private.\n\n'} From 6222199c55679e67d942451d70f12cd36440bc8b Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 18 Dec 2017 16:22:54 -0500 Subject: [PATCH 24/29] STY: pep8 line width --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index ebdc9972a..348656ded 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -294,7 +294,8 @@ def build_schema_doc(key): s += key.title() + '\n' s += '=' * len(key) + '\n' s += SCHEMAS[key]['_description']['description'] - s += 'Schema\n------\nThe following lists key names mapped to its type and meaning for each entry.\n\n' + s += 'Schema\n------\nThe following lists key names mapped to its ' \ + 'type and meaning for each entry.\n\n ' schema = SCHEMAS[key] schema_list = list(schema.keys()) schema_list.sort() From 4420cb52364a97584db05096bbe4ae8c9c65fb21 Mon Sep 17 00:00:00 2001 From: christopher Date: Mon, 1 Jan 2018 21:06:10 -0500 Subject: [PATCH 25/29] FIX: comments from @scopatz --- .gitignore | 10 +- docs/collections/index.rst | 18 +- docs/conf.py | 6 +- regolith/fsclient.py | 8 +- regolith/schemas.py | 1377 ++++++++++++++++++++---------------- 5 files changed, 773 insertions(+), 646 deletions(-) diff --git a/.gitignore b/.gitignore index a3a4f1f84..9c5f714dd 100644 --- a/.gitignore +++ b/.gitignore @@ -60,4 +60,12 @@ target/ rever/ # Auto sphinx docs -docs/collections/auto \ No newline at end of file +docs/collections/grades.rst +docs/collections/assignments.rst +docs/collections/grants.rst +docs/collections/abstracts.rst +docs/collections/blog.rst +docs/collections/people.rst +docs/collections/projects.rst +docs/collections/proposals.rst +docs/collections/students.rst diff --git a/docs/collections/index.rst b/docs/collections/index.rst index 74483c10a..953f66474 100644 --- a/docs/collections/index.rst +++ b/docs/collections/index.rst @@ -8,16 +8,16 @@ The following contain the regolith schemas and examples in both YAML and JSON/Mo .. toctree:: :maxdepth: 1 - auto/abstracts - auto/assignments - auto/blog + abstracts + assignments + blog citations courses - auto/grades - auto/grants + grades + grants jobs news - auto/people - auto/projects - auto/proposals - auto/students + people + projects + proposals + students diff --git a/docs/conf.py b/docs/conf.py index 348656ded..6b5c76f71 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -288,14 +288,14 @@ def format_key(schema, key, indent_str=''): def build_schema_doc(key): - fn = 'collections/auto/' + key + '.rst' + fn = 'collections/' + key + '.rst' with open(fn, 'w') as f: s = '' s += key.title() + '\n' s += '=' * len(key) + '\n' - s += SCHEMAS[key]['_description']['description'] + s += SCHEMAS[key]['_description']['description'] + '\n\n' s += 'Schema\n------\nThe following lists key names mapped to its ' \ - 'type and meaning for each entry.\n\n ' + 'type and meaning for each entry.\n\n' schema = SCHEMAS[key] schema_list = list(schema.keys()) schema_list.sort() diff --git a/regolith/fsclient.py b/regolith/fsclient.py index 69d56d24b..fb3f0b705 100644 --- a/regolith/fsclient.py +++ b/regolith/fsclient.py @@ -49,14 +49,14 @@ def dump_yaml(filename, docs, inst=None): """Dumps a dict of documents into a file.""" inst = YAML() if inst is None else inst inst.indent(mapping=2, sequence=4, offset=2) - my_sorted_dict = ruamel.yaml.comments.CommentedMap() + sorted_dict = ruamel.yaml.comments.CommentedMap() for k, doc in docs.items(): _id = doc.pop('_id') - my_sorted_dict[k] = ruamel.yaml.comments.CommentedMap() + sorted_dict[k] = ruamel.yaml.comments.CommentedMap() for kk in sorted(doc.keys()): - my_sorted_dict[k][kk] = doc[kk] + sorted_dict[k][kk] = doc[kk] with open(filename, 'w') as fh: - inst.dump(my_sorted_dict, stream=fh) + inst.dump(sorted_dict, stream=fh) def json_to_yaml(inp, out): diff --git a/regolith/schemas.py b/regolith/schemas.py index 9ba550fa4..e3b543bdb 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -1,636 +1,755 @@ """Database schemas, examples, and tools""" from cerberus import Validator -EXEMPLARS = {'abstracts': {'_id': 'Mouginot.Model', - 'coauthors': 'P.P.H. Wilson', - 'email': 'mouginot@wisc.edu', - 'firstname': 'Baptiste', - 'institution': 'University of Wisconsin-Madison', - 'lastname': 'Mouginot', - 'references': '[1] B. MOUGINOT, “cyCLASS: CLASS models for Cyclus,”, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 (2016).', - 'text': 'The CLASS team has developed high quality predictors based on pre-trained neural network...', - 'timestamp': '5/5/2017 13:15:59', - 'title': 'Model Performance Analysis'}, - 'assignments': {'_id': 'hw01-rx-power', - 'category': 'homework', - 'courses': ['EMCH-558-2016-S', - 'EMCH-758-2016-S'], - 'points': [1, 2, 3], - 'questions': ['1-9', '1-10', '1-12']}, - 'blog': {'_id': 'my-vision', - 'author': 'Anthony Scopatz', - 'day': 18, - 'month': 'September', - 'original': 'https://scopatz.com/my-vision/', - 'post': 'I would like see things move forward. Deep, I know!', - 'title': 'My Vision', - 'year': 2015}, - 'citations': {'_id': 'meurer2016sympy', - 'author': ['Meurer, Aaron', - 'Smith, Christopher P', - 'Paprocki, Mateusz', - "{\\v{C}}ert{\\'\\i}k, Ond{\\v{r}}ej", - 'Rocklin, Matthew', - 'Kumar, AMiT', - 'Ivanov, Sergiu', - 'Moore, Jason K', - 'Singh, Sartaj', - 'Rathnayake, Thilina', - 'Sean Vig', - 'Brian E Granger', - 'Richard P Muller', - 'Francesco Bonazzi', - 'Harsh Gupta', - 'Shivam Vats', - 'Fredrik Johansson', - 'Fabian Pedregosa', - 'Matthew J Curry', - 'Ashutosh Saboo', - 'Isuru Fernando', - 'Sumith Kulal', - 'Robert Cimrman', - 'Anthony Scopatz'], - 'entrytype': 'article', - 'journal': 'PeerJ Computer Science', - 'month': 'Jan', - 'pages': 'e103', - 'publisher': 'PeerJ Inc. San Francisco, USA', - 'title': 'SymPy: Symbolic computing in Python', - 'volume': '4', - 'year': '2017'}, - 'courses': {'_id': 'EMCH-552-2016-F', - 'active': False, - 'department': 'EMCH', - 'number': 552, - 'scale': [[0.875, 'A'], - [0.8125, 'B+'], - [0.75, 'B'], - [0.6875, 'C+'], - [0.625, 'C'], - [0.5625, 'D+'], - [0.5, 'D'], - [-1.0, 'F']], - 'season': 'F', - 'students': ['Human A. Person', 'Human B. Person'], - 'syllabus': 'emch552-2016-f-syllabus.pdf', - 'weights': {'class-notes': 0.15, - 'final': 0.3, - 'homework': 0.35, - 'midterm': 0.2}, - 'year': 2016}, - 'grades': { - '_id': 'Human A. Person-rx-power-hw02-EMCH-758-2017-S', - 'student': 'hap', - 'assignment': '2017-rx-power-hw02', - 'course': 'EMCH-758-2017-S', - 'scores': [1, 1.6, 3]}, - 'grants': {'_id': 'SymPy-1.1', - 'amount': 3000.0, - 'begin_day': 1, - 'begin_month': 'May', - 'begin_year': 2017, - 'call_for_proposals': 'https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ', - 'end_day': 31, - 'end_month': 'December', - 'end_year': 2017, - 'funder': 'NumFOCUS', - 'narrative': 'https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing', - 'program': 'Small Development Grants', - 'team': [ - {'institution': 'University of South Carolina', - 'name': 'Anthony Scopatz', - 'position': 'PI'}, - {'institution': 'University of South Carolina', - 'name': 'Aaron Meurer', - 'position': 'researcher'}], - 'title': 'SymPy 1.1 Release Support'}, - 'jobs': {'_id': '0004', - 'background_fields': ['Data Science', - 'Data Engineering', - 'Computer Engineering', - 'Computer Science', - 'Applied Mathematics', - 'Physics', - 'Nuclear Engineering', - 'Mechanical Engineering', - 'Or similar'], - 'compensation': [ - 'Salary and compensation will be based on prior work experience.'], - 'contact': 'Please send CV or resume to Prof. Scopatz at scopatzATcec.sc.edu.', - 'day': 1, - 'description': '

We are seeking a dedicated individual to help to aid in ...', - 'month': 'July', - 'open': False, - 'positions': ['Scientific Software Developer', - 'Programmer'], - 'start_date': 'ASAP', - 'title': 'Open Source Scientific Software Maintainer', - 'year': 2015}, - 'news': {'_id': '56b4eb6d421aa921504ef2a9', - 'author': 'Anthony Scopatz', - 'body': 'Dr. Robert Flanagan joined ERGS as a post-doctoral scholar.', - 'day': 1, - 'month': 'February', - 'year': 2016}, - 'people': {'_id': 'scopatz', - 'aka': ['Scopatz', - 'Scopatz, A', - 'Scopatz, A.', - 'Scopatz, A M', - 'Anthony Michael Scopatz'], - 'avatar': 'https://avatars1.githubusercontent.com/u/320553?v=3&s=200', - 'bio': 'Anthony Scopatz is currently an Assistant Professor', - 'education': [{'begin_year': 2008, - 'degree': 'Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program', - 'end_year': 2011, - 'institution': 'The University of Texas at Austin', - 'location': 'Austin, TX', - 'other': [ - 'Adviser: Erich A. Schneider', - 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis']}, - {'begin_year': 2006, - 'degree': 'M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program', - 'end_year': 2007, - 'institution': 'The University of Texas at Austin', - 'location': 'Austin, TX', - 'other': [ - 'Adviser: Erich A. Schneider', - 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership']}, - {'begin_year': 2002, - 'degree': 'B.S. Physics', - 'end_year': 2006, - 'institution': 'University of California, Santa Barbara', - 'location': 'Santa Barbara, CA', - 'other': [ - 'Graduated with a Major in Physics and a Minor in Mathematics']}], - 'email': 'scopatz@cec.sc.edu', - 'employment': [{'begin_year': 2015, - 'location': 'Columbia, SC', - 'organization': 'The University of South Carolina', - 'other': [ - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.', - 'PyNE: The Nuclear Engineering Toolkit.', - 'Website: http://www.ergs.sc.edu/'], - 'position': 'Assistant Professor, Mechanical Engineering Department'}, - {'begin_year': 2013, - 'end_year': 2015, - 'location': 'Madison, WI', - 'organization': 'CNERG, The University of Wisconsin-Madison', - 'other': [ - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.', - 'PyNE: The Nuclear Engineering Toolkit.', - 'Website: https://cnerg.github.io/'], - 'position': 'Associate Scientist, Engineering Physics Department'}, - {'begin_month': 'Nov', - 'begin_year': 2011, - 'end_month': 'May', - 'end_year': 2013, - 'location': 'Chicago, IL', - 'organization': 'The FLASH Center, The University of Chicago', - 'other': [ - 'NIF: Simulation of magnetic field generation from neutral plasmas using FLASH.', - 'CosmoB: Simulation of magnetic field generation from neutral plasmas using FLASH.', - 'FLASH4: High-energy density physics capabilities and utilities.', - 'Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from FLASH.', - 'OpacPlot: HDF5-based equation of state and opacity file format.', - 'Website: http://flash.uchicago.edu/site/'], - 'position': 'Research Scientist, Postdoctoral Scholar'}], - 'funding': [ - {'name': "Omega Laser User's Group Travel Award", - 'value': 1100, - 'year': 2013}, - { - 'name': "NIF User's Group Travel Award", - 'value': 1150, - 'year': 2013}], - 'membership': [{'begin_year': 2006, - 'organization': 'American Nuclear Society', - 'position': 'Member'}, - {'begin_year': 2013, - 'organization': 'Python Software Foundation', - 'position': 'Fellow'}], - 'name': 'Anthony Scopatz', - 'position': 'professor', - 'skills': [{'category': 'Programming Languages', - 'level': 'expert', - 'name': 'Python'}, - { - 'category': 'Programming Languages', - 'level': 'expert', - 'name': 'Cython'}], - 'teaching': [{ - 'course': 'EMCH 552: Intro to Nuclear Engineering', - 'description': 'This course is an introduction to nuclear physics.', - 'month': 'August', - 'organization': 'University of South Carolina', - 'position': 'Professor', - 'syllabus': 'https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc', - 'year': 2017}, - { - 'course': 'EMCH 558/758: Reactor Power Systems', - 'description': 'This course covers conventional reactors.', - 'month': 'January', - 'organization': 'University of South Carolina', - 'position': 'Professor', - 'syllabus': 'https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing', - 'year': 2017}], - 'title': 'Dr.'}, - 'projects': {'_id': 'Cyclus', - 'name': 'Cyclus', - 'description': 'Agent-Based Nuclear Fuel Cycle Simulator', - 'logo': 'http://fuelcycle.org/_static/big_c.png', - 'other': [ - 'Discrete facilities with discrete material transactions', - 'Low barrier to entry, rapid payback to adoption'], - 'repo': 'https://github.com/cyclus/cyclus/', - 'team': [{'begin_month': 'June', - 'begin_year': 2013, - 'end_month': 'July', - 'end_year': 2015, - 'name': 'Anthony Scopatz', - 'position': 'Project Lead'}], - 'website': 'http://fuelcycle.org/'}, - 'proposals': {'_id': 'mypropsal', - 'ammount': 1000000.0, - 'authors': ['Anthony Scopatz', - 'Robert Flanagan'], - 'currency': 'USD', - 'day': 18, - 'durration': 3, - 'full': { - 'benefit_of_collaboration': 'http://pdf.com/benefit_of_collaboration', - 'cv': [ - 'http://pdf.com/scopatz-cv', - 'http://pdf.com/flanagan-cv'], - 'narrative': 'http://some.com/pdf'}, - 'month': 'Aug', - 'pi': 'Anthony Scopatz', - 'pre': { - 'benefit_of_collaboration': 'http://pdf.com/benefit_of_collaboration', - 'cv': [ - 'http://pdf.com/scopatz-cv', - 'http://pdf.com/flanagan-cv'], - 'day': 2, - 'month': 'Aug', - 'narrative': 'http://some.com/pdf', - 'year': 1998}, - 'status': 'submitted', - 'title': 'A very fine proposal indeed', - 'year': 1999}, - 'students': {'_id': 'Human A. Person', - 'aka': ['H. A. Person'], - 'email': 'haperson@uni.edu', - 'university_id': 'HAP42'}} +EXEMPLARS = { + 'abstracts': {'_id': 'Mouginot.Model', + 'coauthors': 'P.P.H. Wilson', + 'email': 'mouginot@wisc.edu', + 'firstname': 'Baptiste', + 'institution': 'University of Wisconsin-Madison', + 'lastname': 'Mouginot', + 'references': '[1] B. MOUGINOT, “cyCLASS: CLASS ' + 'models for Cyclus,”, Figshare, ' + 'https://dx.doi.org/10.6084/' + 'm9.figshare.3468671.v2 (2016).', + 'text': 'The CLASS team has developed high ' + 'quality predictors based on pre-trained ' + 'neural network...', + 'timestamp': '5/5/2017 13:15:59', + 'title': 'Model Performance Analysis'}, + 'assignments': {'_id': 'hw01-rx-power', + 'category': 'homework', + 'courses': ['EMCH-558-2016-S', + 'EMCH-758-2016-S'], + 'points': [1, 2, 3], + 'questions': ['1-9', '1-10', '1-12']}, + 'blog': {'_id': 'my-vision', + 'author': 'Anthony Scopatz', + 'day': 18, + 'month': 'September', + 'original': 'https://scopatz.com/my-vision/', + 'post': 'I would like see things move forward. Deep, I know!', + 'title': 'My Vision', + 'year': 2015}, + 'citations': {'_id': 'meurer2016sympy', + 'author': ['Meurer, Aaron', + 'Smith, Christopher P', + 'Paprocki, Mateusz', + "{\\v{C}}ert{\\'\\i}k, Ond{\\v{r}}ej", + 'Rocklin, Matthew', + 'Kumar, AMiT', + 'Ivanov, Sergiu', + 'Moore, Jason K', + 'Singh, Sartaj', + 'Rathnayake, Thilina', + 'Sean Vig', + 'Brian E Granger', + 'Richard P Muller', + 'Francesco Bonazzi', + 'Harsh Gupta', + 'Shivam Vats', + 'Fredrik Johansson', + 'Fabian Pedregosa', + 'Matthew J Curry', + 'Ashutosh Saboo', + 'Isuru Fernando', + 'Sumith Kulal', + 'Robert Cimrman', + 'Anthony Scopatz'], + 'entrytype': 'article', + 'journal': 'PeerJ Computer Science', + 'month': 'Jan', + 'pages': 'e103', + 'publisher': 'PeerJ Inc. San Francisco, USA', + 'title': 'SymPy: Symbolic computing in Python', + 'volume': '4', + 'year': '2017'}, + 'courses': {'_id': 'EMCH-552-2016-F', + 'active': False, + 'department': 'EMCH', + 'number': 552, + 'scale': [[0.875, 'A'], + [0.8125, 'B+'], + [0.75, 'B'], + [0.6875, 'C+'], + [0.625, 'C'], + [0.5625, 'D+'], + [0.5, 'D'], + [-1.0, 'F']], + 'season': 'F', + 'students': ['Human A. Person', 'Human B. Person'], + 'syllabus': 'emch552-2016-f-syllabus.pdf', + 'weights': {'class-notes': 0.15, + 'final': 0.3, + 'homework': 0.35, + 'midterm': 0.2}, + 'year': 2016}, + 'grades': { + '_id': 'Human A. Person-rx-power-hw02-EMCH-758-2017-S', + 'student': 'hap', + 'assignment': '2017-rx-power-hw02', + 'course': 'EMCH-758-2017-S', + 'scores': [1, 1.6, 3]}, + 'grants': {'_id': 'SymPy-1.1', + 'amount': 3000.0, + 'begin_day': 1, + 'begin_month': 'May', + 'begin_year': 2017, + 'call_for_proposals': 'https://groups.google.com/d/msg' + '/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ', + 'end_day': 31, + 'end_month': 'December', + 'end_year': 2017, + 'funder': 'NumFOCUS', + 'narrative': 'https://docs.google.com/document/d/1nZxqoL' + '-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp' + '=sharing', + 'program': 'Small Development Grants', + 'team': [ + {'institution': 'University of South Carolina', + 'name': 'Anthony Scopatz', + 'position': 'PI'}, + {'institution': 'University of South Carolina', + 'name': 'Aaron Meurer', + 'position': 'researcher'}], + 'title': 'SymPy 1.1 Release Support'}, + 'jobs': {'_id': '0004', + 'background_fields': ['Data Science', + 'Data Engineering', + 'Computer Engineering', + 'Computer Science', + 'Applied Mathematics', + 'Physics', + 'Nuclear Engineering', + 'Mechanical Engineering', + 'Or similar'], + 'compensation': [ + 'Salary and compensation will be based on prior work ' + 'experience.'], + 'contact': 'Please send CV or resume to Prof. Scopatz at ' + 'scopatzATcec.sc.edu.', + 'day': 1, + 'description': '

We are seeking a dedicated individual to ' + 'help to aid in ...', + 'month': 'July', + 'open': False, + 'positions': ['Scientific Software Developer', + 'Programmer'], + 'start_date': 'ASAP', + 'title': 'Open Source Scientific Software Maintainer', + 'year': 2015}, + 'news': {'_id': '56b4eb6d421aa921504ef2a9', + 'author': 'Anthony Scopatz', + 'body': 'Dr. Robert Flanagan joined ERGS as a post-doctoral ' + 'scholar.', + 'day': 1, + 'month': 'February', + 'year': 2016}, + 'people': {'_id': 'scopatz', + 'aka': ['Scopatz', + 'Scopatz, A', + 'Scopatz, A.', + 'Scopatz, A M', + 'Anthony Michael Scopatz'], + 'avatar': 'https://avatars1.githubusercontent.com/u/320553?v' + '=3&s=200', + 'bio': 'Anthony Scopatz is currently an Assistant Professor', + 'education': [ + {'begin_year': 2008, + 'degree': 'Ph.D. Mechanical Engineering, ' + 'Nuclear and Radiation Engineering ' + 'Program', + 'end_year': 2011, + 'institution': 'The University of Texas at Austin', + 'location': 'Austin, TX', + 'other': [ + 'Adviser: Erich A. Schneider', + 'Dissertation: Essential Physics for Fuel Cycle ' + 'Modeling & Analysis']}, + {'begin_year': 2006, + 'degree': 'M.S.E. Mechanical Engineering, Nuclear and ' + 'Radiation Engineering Program', + 'end_year': 2007, + 'institution': 'The University of Texas at Austin', + 'location': 'Austin, TX', + 'other': [ + 'Adviser: Erich A. Schneider', + 'Thesis: Recyclable Uranium Options under the Global ' + 'Nuclear Energy Partnership']}, + {'begin_year': 2002, + 'degree': 'B.S. Physics', + 'end_year': 2006, + 'institution': 'University of California, Santa Barbara', + 'location': 'Santa Barbara, CA', + 'other': [ + 'Graduated with a Major in Physics and a Minor in ' + 'Mathematics']}], + 'email': 'scopatz@cec.sc.edu', + 'employment': [ + {'begin_year': 2015, + 'location': 'Columbia, SC', + 'organization': 'The University of South Carolina', + 'other': [ + 'Cyclus: An agent-based, discrete time nuclear fuel ' + 'cycle simulator.', + 'PyNE: The Nuclear Engineering Toolkit.', + 'Website: http://www.ergs.sc.edu/'], + 'position': 'Assistant Professor, Mechanical Engineering ' + 'Department'}, + {'begin_year': 2013, + 'end_year': 2015, + 'location': 'Madison, WI', + 'organization': 'CNERG, The University of ' + 'Wisconsin-Madison', + 'other': [ + 'Cyclus: An agent-based, discrete time nuclear fuel ' + 'cycle simulator.', + 'PyNE: The Nuclear Engineering Toolkit.', + 'Website: https://cnerg.github.io/'], + 'position': 'Associate Scientist, Engineering Physics ' + 'Department'}, + {'begin_month': 'Nov', + 'begin_year': 2011, + 'end_month': 'May', + 'end_year': 2013, + 'location': 'Chicago, IL', + 'organization': 'The FLASH Center, The University of ' + 'Chicago', + 'other': [ + 'NIF: Simulation of magnetic field generation from ' + 'neutral plasmas using FLASH.', + 'CosmoB: Simulation of magnetic field generation ' + 'from neutral plasmas using FLASH.', + 'FLASH4: High-energy density physics capabilities ' + 'and utilities.', + 'Simulated Diagnostics: Schlieren, shadowgraphy, ' + 'Langmuir probes, etc. from FLASH.', + 'OpacPlot: HDF5-based equation of state and opacity ' + 'file format.', + 'Website: http://flash.uchicago.edu/site/'], + 'position': 'Research Scientist, Postdoctoral Scholar'}], + 'funding': [ + {'name': "Omega Laser User's Group Travel Award", + 'value': 1100, + 'year': 2013}, + { + 'name': "NIF User's Group Travel Award", + 'value': 1150, + 'year': 2013}], + 'membership': [{'begin_year': 2006, + 'organization': 'American Nuclear Society', + 'position': 'Member'}, + {'begin_year': 2013, + 'organization': 'Python Software Foundation', + 'position': 'Fellow'}], + 'name': 'Anthony Scopatz', + 'position': 'professor', + 'skills': [{'category': 'Programming Languages', + 'level': 'expert', + 'name': 'Python'}, + { + 'category': 'Programming Languages', + 'level': 'expert', + 'name': 'Cython'}], + 'teaching': [{ + 'course': 'EMCH 552: Intro to Nuclear Engineering', + 'description': 'This course is an introduction to nuclear ' + 'physics.', + 'month': 'August', + 'organization': 'University of South Carolina', + 'position': 'Professor', + 'syllabus': 'https://drive.google.com/open?id' + '=0BxUpd34yizZreDBCMEJNY2FUbnc', + 'year': 2017}, + { + 'course': 'EMCH 558/758: Reactor Power Systems', + 'description': 'This course covers conventional ' + 'reactors.', + 'month': 'January', + 'organization': 'University of South Carolina', + 'position': 'Professor', + 'syllabus': 'https://docs.google.com/document/d' + '/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8' + '-PxiboYdM/edit?usp=sharing', + 'year': 2017}], + 'title': 'Dr.'}, + 'projects': {'_id': 'Cyclus', + 'name': 'Cyclus', + 'description': 'Agent-Based Nuclear Fuel Cycle Simulator', + 'logo': 'http://fuelcycle.org/_static/big_c.png', + 'other': [ + 'Discrete facilities with discrete material transactions', + 'Low barrier to entry, rapid payback to adoption'], + 'repo': 'https://github.com/cyclus/cyclus/', + 'team': [{'begin_month': 'June', + 'begin_year': 2013, + 'end_month': 'July', + 'end_year': 2015, + 'name': 'Anthony Scopatz', + 'position': 'Project Lead'}], + 'website': 'http://fuelcycle.org/'}, + 'proposals': {'_id': 'mypropsal', + 'ammount': 1000000.0, + 'authors': ['Anthony Scopatz', + 'Robert Flanagan'], + 'currency': 'USD', + 'day': 18, + 'durration': 3, + 'full': { + 'benefit_of_collaboration': 'http://pdf.com' + '/benefit_of_collaboration', + 'cv': [ + 'http://pdf.com/scopatz-cv', + 'http://pdf.com/flanagan-cv'], + 'narrative': 'http://some.com/pdf'}, + 'month': 'Aug', + 'pi': 'Anthony Scopatz', + 'pre': { + 'benefit_of_collaboration': 'http://pdf.com' + '/benefit_of_collaboration', + 'cv': [ + 'http://pdf.com/scopatz-cv', + 'http://pdf.com/flanagan-cv'], + 'day': 2, + 'month': 'Aug', + 'narrative': 'http://some.com/pdf', + 'year': 1998}, + 'status': 'submitted', + 'title': 'A very fine proposal indeed', + 'year': 1999}, + 'students': {'_id': 'Human A. Person', + 'aka': ['H. A. Person'], + 'email': 'haperson@uni.edu', + 'university_id': 'HAP42'}} -SCHEMAS = {'abstracts': {'_description': {'description': 'Abstracts for a conference or workshop. This is generally public information\n\n'}, - '_id': {'description': 'Unique identifier for submission. This generally includes the author name and part of the title.', - 'required': True, - 'type': 'string'}, - 'coauthors': {'description': 'names of coauthors', - 'required': False, - 'type': 'string'}, - 'email': {'description': 'contact email for the author.', - 'required': True, - 'type': 'string'}, - 'firstname': {'description': 'first name of the author.', - 'required': True, - 'type': 'string'}, - 'institution': {'description': 'name of the inistitution', - 'required': True, - 'type': 'string'}, - 'lastname': {'description': 'last name of the author.', - 'required': True, - 'type': 'string'}, - 'references': {'description': 'HTML string of reference for the abstract itself', - 'required': False, - 'type': 'string'}, - 'text': {'description': 'HTML string of the abstract.', - 'required': True, - 'type': 'string'}, - 'timestamp': {'description': 'The time when the abstract was submitted.', - 'required': True, - 'type': 'string'}, - 'title': {'description': 'title of the presentation/paper.', - 'required': True, - 'type': 'string'}}, - 'assignments': {'_description': {'description': 'Information about assignments for classes.\n\n'}, - '_id': {'description': 'A unique id for the assignment, such a HW01-EMCH-558-2016-S', - 'required': True, - 'type': 'string'}, - 'category': {'description': "such as 'homework' or 'final'", - 'required': True, - 'type': 'string'}, - 'courses': {'description': 'ids of the courses that have this assignment', - 'required': True, - 'type': 'string'}, - 'file': {'description': 'path to assignment file in store', - 'required': False, - 'type': 'string'}, - 'points': {'description': 'list of number of points possible for each question. Length is the number of questions', - 'required': True, - 'type': ('integer', 'float')}, - 'question': {'description': 'titles for the questions on this assignment', - 'required': False, - 'type': 'string'}, - 'solution': {'description': 'path to solution file in store', - 'required': False, - 'type': 'string'}}, - 'blog': {'_description': {'description': 'This collection represents blog posts written by the members of the research group.\n\n'}, - '_id': {'description': 'short represntation, such as this-is-my-title', - 'required': True, - 'type': 'string'}, - 'author': {'description': 'name or AKA of author', - 'required': True, - 'type': 'string'}, - 'day': {'description': 'Publication day', - 'required': True, - 'type': 'integer'}, - 'month': {'description': 'Publication month', - 'required': True, - 'type': 'string'}, - 'original': {'description': 'URL of original post, if this is a repost', - 'required': False, - 'type': 'string'}, - 'post': {'description': 'actual contents of the post', - 'required': True, - 'type': 'string'}, - 'title': {'description': 'full human readable title', - 'required': True, - 'type': 'string'}, - 'year': {'description': 'Publication year', - 'required': True, - 'type': 'integer'}}, - 'grades': {'_description': {'description': 'The grade for a student on an assignment. This information should be private.\n\n'}, - '_id': {'description': 'unique id, typically the student-assignment-course', - 'required': True, - 'type': 'string'}, - 'assignment': {'description': 'assignment id', - 'required': True, - 'type': 'string'}, - 'course': {'description': 'course id', 'required': True, 'type': 'string'}, - 'filename': {'description': 'path to file in store', - 'required': False, - 'type': 'string'}, - 'scores': {'description': 'the number of points earned on each question', - 'required': True, - 'type': ('integer', 'float')}, - 'student': {'description': 'student id', - 'required': True, - 'type': 'string'}}, - 'grants': {'_description': {'description': 'This collection represents grants that have been awarded to the group.\n\n'}, - '_id': {'description': 'short representation, such as this-is-my-name', - 'required': True, - 'type': ('string', 'integer', 'float')}, - 'amount': {'description': 'value of award', - 'required': True, - 'type': ('integer', 'float')}, - 'begin_day': {'description': 'start day of the grant', - 'required': False, - 'type': 'integer'}, - 'begin_month': {'description': 'start month of the grant', - 'required': True, - 'type': 'string'}, - 'begin_year': {'description': 'start year of the grant', - 'required': True, - 'type': 'integer'}, - 'benefit_of_collaboration': {'description': '', - 'required': False, - 'type': 'string'}, - 'call_for_proposals': {'description': '', - 'required': False, - 'type': 'string'}, - 'currency': {'description': "typically '$' or 'USD'", - 'required': False, - 'type': 'string'}, - 'end_day': {'description': 'end day of the grant', - 'required': False, - 'type': ('string', 'integer')}, - 'end_month"': {'description': 'end month of the grant', - 'required': False, - 'type': 'string'}, - 'end_year': {'description': 'end year of the grant', - 'required': True, - 'type': 'integer'}, - 'funder': {'description': 'the agency funding the work', - 'required': True, - 'type': 'string'}, - 'grant_id': {'description': 'the identfier for this work', - 'required': False, - 'type': 'string'}, - 'narrative': {'description': '', 'required': False, 'type': 'string'}, - 'program': {'description': 'the program the work was funded under', - 'required': True, - 'type': 'string'}, - 'status': {'allowed': ['pending', 'declined', 'accepted', 'in-prep'], - 'description': 'status of the grant', - 'required': True, - 'type': 'string'}, - 'team': {'description': 'information about the team members participating in the grant.', - 'required': True, - 'schema': {'schema': {'cv': {'required': False, 'type': 'string'}, - 'institution': {'required': True, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}, - 'subaward_amount': {'required': False, 'type': ('integer', 'float')}}, - 'type': 'dict'}, - 'type': 'list'}, - 'title': {'description': 'actual title of proposal / grant', - 'required': True, - 'type': 'string'}}, - 'people': {'_description': {'description': 'This collection describes the members of the research group. This is normally public\ndata.\n\n'}, - '_id': {'description': 'unique identifier for the group member', - 'required': True, - 'type': 'string'}, - 'active': {'description': 'If the person is an active member, default True.', - 'required': False, - 'type': 'boolean'}, - 'aka': {'description': 'list of aliases (also-known-as), useful for identifying the group member in citations or elsewhere.', - 'required': True, - 'type': ['string', 'list']}, - 'avatar': {'description': 'URL to avatar', - 'required': True, - 'type': 'string'}, - 'bio': {'description': 'short biographical text', - 'required': True, - 'type': 'string'}, - 'collab': {'description': 'If the person is a collaborator, default False.', - 'required': False, - 'type': 'boolean'}, - 'education': {'description': 'This contains the educational information for the group member.', - 'required': True, - 'schema': {'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'degree': {'required': True, 'type': 'string'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': True, 'type': 'integer'}, - 'gpa': {'required': False, 'type': ('float', 'string')}, - 'institution': {'required': True, 'type': 'string'}, - 'location': {'required': True, 'type': 'string'}, - 'other': {'required': False, 'type': 'string'}}, - 'type': 'list'}, - 'email': {'description': 'email address of the group member', - 'required': False, - 'type': 'string'}, - 'employment': {'description': 'Employment information, similar to educational information.', - 'required': True, - 'schema': {'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'location': {'required': True, 'type': 'string'}, - 'organization': {'required': True, 'type': 'string'}, - 'other': {'required': False, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}}, - 'type': 'list'}, - 'funding': {'description': 'Funding and scholarship that the group member has individually obtained in the past. **WARNING:** this is not to be confused with the **grants** collection', - 'required': False, - 'schema': {'currency': {'required': False, 'type': 'string'}, - 'duration': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'value': {'required': True, 'type': ('float', 'integer')}, - 'year': {'required': True, 'type': 'integer'}}, - 'type': 'list'}, - 'honors': {'description': 'Honors that have been awarded to this group member', - 'required': False, - 'schema': {'description': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'year': {'required': True, 'type': 'integer'}}, - 'type': 'list'}, - 'membership': {'description': 'Profesional organizations this member is a part of', - 'required': False, - 'schema': {'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'description': {'required': False, 'type': 'string'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'organization': {'required': True, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}, - 'website': {'required': False, 'type': 'string'}}, - 'type': 'list'}, - 'name': {'description': 'Full, canonical name for the person', - 'required': True, - 'type': 'string'}, - 'position': {'description': 'such as professor, graduate student, or scientist', - 'required': True, - 'type': 'string'}, - 'service': {'description': 'Service that this group member has provided', - 'required': False, - 'schema': {'description': {'required': False, 'type': 'string'}, - 'duration': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}, - 'year': {'required': True, 'type': 'integer'}}, - 'type': 'list'}, - 'skills': {'description': 'Skill the group member has', - 'required': False, - 'schema': {'category': {'required': True, 'type': 'string'}, - 'level': {'required': True, 'type': 'string'}, - 'name': {'required': True, 'type': 'string'}}, - 'type': 'list'}, - 'teaching': {'description': 'Courses that this group member has taught, if any', - 'required': False, - 'schema': {'course': {'required': True, 'type': 'string'}, - 'description': {'required': False, 'type': 'string'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'materials': {'required': False, 'type': 'string'}, - 'month': {'required': False, 'type': 'string'}, - 'organization': {'required': True, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}, - 'syllabus': {'required': False, 'type': 'string'}, - 'video': {'required': False, 'type': 'string'}, - 'website': {'required': False, 'type': 'string'}, - 'year': {'required': True, 'type': 'integer'}}, - 'type': 'list'}, - 'title': {'description': 'for example, Dr., etc.', - 'required': False, - 'type': 'string'}}, - 'projects': {'_description': {'description': 'This collection describes the research group projects. This is normally public data.\n\n'}, - '_id': {'description': 'Unique project identifier.', - 'required': True, - 'type': 'string'}, - 'description': {'description': 'brief project description.', - 'required': True, - 'type': 'string'}, - 'grant': {'description': 'Grant id if there is a grant supporting this project', - 'type': 'string'}, - 'logo': {'description': 'URL to the project logo', - 'required': False, - 'type': 'string'}, - 'name': {'description': 'name of the project.', - 'required': True, - 'type': 'string'}, - 'other': {'description': 'other information about the project', - 'required': False, - 'type': ['list', 'string']}, - 'repo': {'description': 'URL of the source code repo, if available', - 'required': False, - 'type': 'string'}, - 'team': {'description': 'People who are/have been woking on this project.', - 'required': True, - 'schema': {'begin_month': {'required': False, 'type': 'string'}, - 'begin_year': {'required': True, 'type': 'integer'}, - 'end_month': {'required': False, 'type': 'string'}, - 'end_year': {'required': False, 'type': 'integer'}, - 'name': {'required': True, 'type': 'string'}, - 'position': {'required': True, 'type': 'string'}}, - 'type': 'list'}, - 'website': {'description': 'URL of the website.', - 'required': True, - 'type': 'string'}}, - 'proposals': {'_description': {'description': 'This collection represents proposals that have been submitted by the group.\n\n'}, - '_id': {'description': 'short represntation, such as this-is-my-name', - 'required': True, - 'type': ('string', 'integer', 'float')}, - 'ammount': {'description': 'value of award', - 'required': True, - 'type': ('integer', 'float')}, - 'authors': {'description': 'other investigator names', - 'required': True, - 'type': 'string'}, - 'currency': {'description': "typically '$' or 'USD'", - 'required': True, - 'type': 'string'}, - 'day': {'description': 'day that the proposal is due', - 'required': True, - 'type': 'integer'}, - 'durration': {'description': 'number of years', - 'required': True, - 'type': ('integer', 'float')}, - 'month': {'description': 'month that the proposal is due', - 'required': True, - 'type': 'string'}, - 'pi': {'description': 'principal investigator name', - 'required': True, - 'type': 'string'}, - 'pre': {'description': 'Information about the pre-proposal', - 'required': False, - 'type': 'dict'}, - 'status': {'description': "e.g. 'submitted', 'accepted', 'rejected'", - 'required': True, - 'type': 'string'}, - 'title': {'description': 'actual title of proposal', - 'required': True, - 'type': 'string'}, - 'year': {'description': 'Year that the proposal is due', - 'required': True, - 'type': 'integer'}}, - 'students': {'_description': {'description': 'This is a collection of student names and metadata. This should probably be private.\n\n'}, - '_id': {'description': 'short represntation, such as this-is-my-name', - 'required': True, - 'type': 'string'}, - 'aka': {'description': 'list of aliases', - 'required': False, - 'schema': {'type': 'string'}, - 'type': ('list', 'string')}, - 'email': {'description': 'email address', - 'required': False, - 'type': 'string'}, - 'university_id': {'description': 'The university identifier for the student', - 'required': False, - 'type': 'string'}}} +SCHEMAS = { + 'abstracts': { + '_description': { + 'description': 'Abstracts for a conference or workshop. This is ' + 'generally public information'}, + '_id': { + 'description': 'Unique identifier for submission. This generally ' + 'includes the author name and part of the title.', + 'required': True, + 'type': 'string'}, + 'coauthors': {'description': 'names of coauthors', + 'required': False, + 'type': 'string'}, + 'email': {'description': 'contact email for the author.', + 'required': True, + 'type': 'string'}, + 'firstname': {'description': 'first name of the author.', + 'required': True, + 'type': 'string'}, + 'institution': {'description': 'name of the inistitution', + 'required': True, + 'type': 'string'}, + 'lastname': {'description': 'last name of the author.', + 'required': True, + 'type': 'string'}, + 'references': { + 'description': 'HTML string of reference for the abstract itself', + 'required': False, + 'type': 'string'}, + 'text': {'description': 'HTML string of the abstract.', + 'required': True, + 'type': 'string'}, + 'timestamp': { + 'description': 'The time when the abstract was submitted.', + 'required': True, + 'type': 'string'}, + 'title': {'description': 'title of the presentation/paper.', + 'required': True, + 'type': 'string'}}, + 'assignments': { + '_description': { + 'description': 'Information about assignments for classes.'}, + '_id': { + 'description': 'A unique id for the assignment, such as ' + 'HW01-EMCH-558-2016-S', + 'required': True, + 'type': 'string'}, + 'category': {'description': "such as 'homework' or 'final'", + 'required': True, + 'type': 'string'}, + 'courses': { + 'description': 'ids of the courses that have this assignment', + 'required': True, + 'type': 'string'}, + 'file': {'description': 'path to assignment file in store', + 'required': False, + 'type': 'string'}, + 'points': { + 'description': 'list of number of points possible for each ' + 'question. Length is the number of questions', + 'required': True, + 'type': ('integer', 'float')}, + 'question': { + 'description': 'titles for the questions on this assignment', + 'required': False, + 'type': 'string'}, + 'solution': {'description': 'path to solution file in store', + 'required': False, + 'type': 'string'}}, + 'blog': { + '_description': { + 'description': 'This collection represents blog posts written by ' + 'the members of the research group.'}, + '_id': { + 'description': 'short representation, such as this-is-my-title', + 'required': True, + 'type': 'string'}, + 'author': {'description': 'name or AKA of author', + 'required': True, + 'type': 'string'}, + 'day': {'description': 'Publication day', + 'required': True, + 'type': 'integer'}, + 'month': {'description': 'Publication month', + 'required': True, + 'type': 'string'}, + 'original': { + 'description': 'URL of original post, if this is a repost', + 'required': False, + 'type': 'string'}, + 'post': {'description': 'actual contents of the post', + 'required': True, + 'type': 'string'}, + 'title': {'description': 'full human readable title', + 'required': True, + 'type': 'string'}, + 'year': {'description': 'Publication year', + 'required': True, + 'type': 'integer'}}, + 'grades': { + '_description': { + 'description': 'The grade for a student on an assignment. This ' + 'information should be private.'}, + '_id': { + 'description': 'unique id, typically the ' + 'student-assignment-course', + 'required': True, + 'type': 'string'}, + 'assignment': {'description': 'assignment id', + 'required': True, + 'type': 'string'}, + 'course': { + 'description': 'course id', + 'required': True, + 'type': 'string'}, + 'filename': {'description': 'path to file in store', + 'required': False, + 'type': 'string'}, + 'scores': { + 'description': 'the number of points earned on each question', + 'required': True, + 'type': ('integer', 'float')}, + 'student': {'description': 'student id', + 'required': True, + 'type': 'string'}}, + 'grants': { + '_description': { + 'description': 'This collection represents grants that have been ' + 'awarded to the group.'}, + '_id': {'description': 'short representation, such as this-is-my-name', + 'required': True, + 'type': ('string', 'integer', 'float')}, + 'amount': {'description': 'value of award', + 'required': True, + 'type': ('integer', 'float')}, + 'begin_day': {'description': 'start day of the grant', + 'required': False, + 'type': 'integer'}, + 'begin_month': {'description': 'start month of the grant', + 'required': True, + 'type': 'string'}, + 'begin_year': {'description': 'start year of the grant', + 'required': True, + 'type': 'integer'}, + 'benefit_of_collaboration': {'description': '', + 'required': False, + 'type': 'string'}, + 'call_for_proposals': {'description': '', + 'required': False, + 'type': 'string'}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': False, + 'type': 'string'}, + 'end_day': {'description': 'end day of the grant', + 'required': False, + 'type': ('string', 'integer')}, + 'end_month"': {'description': 'end month of the grant', + 'required': False, + 'type': 'string'}, + 'end_year': {'description': 'end year of the grant', + 'required': True, + 'type': 'integer'}, + 'funder': {'description': 'the agency funding the work', + 'required': True, + 'type': 'string'}, + 'grant_id': {'description': 'the identifier for this work', + 'required': False, + 'type': 'string'}, + 'narrative': {'description': '', 'required': False, 'type': 'string'}, + 'program': {'description': 'the program the work was funded under', + 'required': True, + 'type': 'string'}, + 'status': {'allowed': ['pending', 'declined', 'accepted', 'in-prep'], + 'description': 'status of the grant', + 'required': True, + 'type': 'string'}, + 'team': { + 'description': 'information about the team members participating ' + 'in the grant.', + 'required': True, + 'schema': {'schema': {'cv': {'required': False, 'type': 'string'}, + 'institution': { + 'required': True, + 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'position': { + 'required': True, + 'type': 'string'}, + 'subaward_amount': { + 'required': False, + 'type': ('integer', 'float')}}, + 'type': 'dict'}, + 'type': 'list'}, + 'title': {'description': 'actual title of proposal / grant', + 'required': True, + 'type': 'string'}}, + 'people': { + '_description': { + 'description': 'This collection describes the members of the ' + 'research group. This is normally public data.'}, + '_id': {'description': 'unique identifier for the group member', + 'required': True, + 'type': 'string'}, + 'active': { + 'description': 'If the person is an active member, default True.', + 'required': False, + 'type': 'boolean'}, + 'aka': { + 'description': 'list of aliases (also-known-as), useful for ' + 'identifying the group member in citations or ' + 'elsewhere.', + 'required': True, + 'type': ['string', 'list']}, + 'avatar': {'description': 'URL to avatar', + 'required': True, + 'type': 'string'}, + 'bio': {'description': 'short biographical text', + 'required': True, + 'type': 'string'}, + 'collab': { + 'description': 'If the person is a collaborator, default False.', + 'required': False, + 'type': 'boolean'}, + 'education': { + 'description': 'This contains the educational information for ' + 'the group member.', + 'required': True, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'degree': {'required': True, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': True, 'type': 'integer'}, + 'gpa': {'required': False, 'type': ('float', 'string')}, + 'institution': {'required': True, 'type': 'string'}, + 'location': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}}, + 'type': 'list'}, + 'email': {'description': 'email address of the group member', + 'required': False, + 'type': 'string'}, + 'employment': { + 'description': 'Employment information, similar to educational ' + 'information.', + 'required': True, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'location': {'required': True, 'type': 'string'}, + 'organization': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}}, + 'type': 'list'}, + 'funding': { + 'description': 'Funding and scholarship that the group member ' + 'has individually obtained in the past. ' + '**WARNING:** this is not to be confused with the ' + '**grants** collection', + 'required': False, + 'schema': {'currency': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'value': { + 'required': True, + 'type': ('float', 'integer')}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'honors': { + 'description': 'Honors that have been awarded to this ' + 'group member', + 'required': False, + 'schema': {'description': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'membership': { + 'description': 'Professional organizations this member is ' + 'a part of', + 'required': False, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'description': {'required': False, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'organization': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}}, + 'type': 'list'}, + 'name': {'description': 'Full, canonical name for the person', + 'required': True, + 'type': 'string'}, + 'position': { + 'description': 'such as professor, graduate student, or scientist', + 'required': True, + 'type': 'string'}, + 'service': { + 'description': 'Service that this group member has provided', + 'required': False, + 'schema': {'description': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'skills': {'description': 'Skill the group member has', + 'required': False, + 'schema': {'category': {'required': True, 'type': 'string'}, + 'level': {'required': True, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}}, + 'type': 'list'}, + 'teaching': { + 'description': 'Courses that this group member has taught, if any', + 'required': False, + 'schema': {'course': {'required': True, 'type': 'string'}, + 'description': {'required': False, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'materials': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'organization': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'syllabus': {'required': False, 'type': 'string'}, + 'video': {'required': False, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'title': {'description': 'for example, Dr., etc.', + 'required': False, + 'type': 'string'}}, + 'projects': { + '_description': { + 'description': 'This collection describes the research group ' + 'projects. This is normally public data.'}, + '_id': {'description': 'Unique project identifier.', + 'required': True, + 'type': 'string'}, + 'description': {'description': 'brief project description.', + 'required': True, + 'type': 'string'}, + 'grant': { + 'description': 'Grant id if there is a grant supporting this ' + 'project', + 'type': 'string'}, + 'logo': {'description': 'URL to the project logo', + 'required': False, + 'type': 'string'}, + 'name': {'description': 'name of the project.', + 'required': True, + 'type': 'string'}, + 'other': {'description': 'other information about the project', + 'required': False, + 'type': ['list', 'string']}, + 'repo': {'description': 'URL of the source code repo, if available', + 'required': False, + 'type': 'string'}, + 'team': { + 'description': 'People who are/have been working on this project.', + 'required': True, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'name': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}}, + 'type': 'list'}, + 'website': {'description': 'URL of the website.', + 'required': True, + 'type': 'string'}}, + 'proposals': { + '_description': { + 'description': 'This collection represents proposals that have ' + 'been submitted by the group.'}, + '_id': {'description': 'short representation, such as this-is-my-name', + 'required': True, + 'type': ('string', 'integer', 'float')}, + 'amount': {'description': 'value of award', + 'required': True, + 'type': ('integer', 'float')}, + 'authors': {'description': 'other investigator names', + 'required': True, + 'type': 'string'}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': True, + 'type': 'string'}, + 'day': {'description': 'day that the proposal is due', + 'required': True, + 'type': 'integer'}, + 'duration': {'description': 'number of years', + 'required': True, + 'type': ('integer', 'float')}, + 'month': {'description': 'month that the proposal is due', + 'required': True, + 'type': 'string'}, + 'pi': {'description': 'principal investigator name', + 'required': True, + 'type': 'string'}, + 'pre': {'description': 'Information about the pre-proposal', + 'required': False, + 'type': 'dict'}, + 'status': {'description': "e.g. 'submitted', 'accepted', 'rejected'", + 'required': True, + 'type': 'string'}, + 'title': {'description': 'actual title of proposal', + 'required': True, + 'type': 'string'}, + 'year': {'description': 'Year that the proposal is due', + 'required': True, + 'type': 'integer'}}, + 'students': { + '_description': { + 'description': 'This is a collection of student names and ' + 'metadata. This should probably be private.'}, + '_id': {'description': 'short representation, such as this-is-my-name', + 'required': True, + 'type': 'string'}, + 'aka': {'description': 'list of aliases', + 'required': False, + 'schema': {'type': 'string'}, + 'type': ('list', 'string')}, + 'email': {'description': 'email address', + 'required': False, + 'type': 'string'}, + 'university_id': { + 'description': 'The university identifier for the student', + 'required': False, + 'type': 'string'}}} class NoDescriptionValidator(Validator): From b7d35659829eaf0246f354e15de64a71dcba497a Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 2 Jan 2018 16:31:44 -0500 Subject: [PATCH 26/29] FIX: fix app, move tests --- regolith/app.py | 15 +++++++++------ {regolith/tests => tests}/test_validators.py | 0 2 files changed, 9 insertions(+), 6 deletions(-) rename {regolith/tests => tests}/test_validators.py (100%) diff --git a/regolith/app.py b/regolith/app.py index aea5bb714..e1071cf58 100644 --- a/regolith/app.py +++ b/regolith/app.py @@ -8,6 +8,7 @@ from regolith.schemas import validate + app = Flask('regolith') @@ -57,7 +58,7 @@ def collection_page(dbname, collname): n = os.path.join(td.name, 'regolith.txt') print('Error in json parsing writing text file to {}. ' 'Please try again.'.format(n)) - with open(n) as f: + with open(n, 'w') as f: f.write(form['body']) traceback.print_exc() raise @@ -65,12 +66,14 @@ def collection_page(dbname, collname): if not tv: td = tempfile.TemporaryDirectory() n = os.path.join(td.name, 'regolith.txt') - print(errors) - print('Writing text file to {}. ' - 'Please try again.'.format(n)) - with open(n) as f: + with open(n, 'w') as f: f.write(form['body']) - raise ValueError('Error while validating the record') + raise ValueError('Error while validating the record,' + ' writing text file to {}. ' + 'Please try again.\n\n' + 'Your errors were\n' + '------------------' + '{}'.format(n, errors)) rc.client.update_one(dbname, collname, {'_id': body['_id']}, body) status = 'saved ✓' diff --git a/regolith/tests/test_validators.py b/tests/test_validators.py similarity index 100% rename from regolith/tests/test_validators.py rename to tests/test_validators.py From 1491d88a3492b09471061dddc3c96f974b11029d Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 2 Jan 2018 16:47:13 -0500 Subject: [PATCH 27/29] FIX: move error to raise --- regolith/app.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/regolith/app.py b/regolith/app.py index e1071cf58..069cf592c 100644 --- a/regolith/app.py +++ b/regolith/app.py @@ -95,12 +95,14 @@ def collection_page(dbname, collname): if not tv: td = tempfile.TemporaryDirectory() n = os.path.join(td.name, 'regolith.txt') - print(errors) - print('Writing text file to {}. ' - 'Please try again.'.format(n)) with open(n) as f: f.write(form['body']) - raise ValueError('Error while validating the record') + raise ValueError('Error while validating the record,' + ' writing text file to {}. ' + 'Please try again.\n\n' + 'Your errors were\n' + '------------------' + '{}'.format(n, errors)) try: added = rc.client.insert_one(dbname, collname, body) except Exception: From a990ed91c685c897b0b32dd02927d96abffc0e44 Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 2 Jan 2018 17:52:36 -0500 Subject: [PATCH 28/29] news --- news/schema.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 news/schema.rst diff --git a/news/schema.rst b/news/schema.rst new file mode 100644 index 000000000..080f0e2ad --- /dev/null +++ b/news/schema.rst @@ -0,0 +1,16 @@ +**Added:** + +* Schemas for the tables +* Docs auto generate for collections (if they were documented in the schema) + +**Changed:** + +* App now validates incoming data + +**Deprecated:** None + +**Removed:** None + +**Fixed:** None + +**Security:** None From 5df3883d0965ede2151e5fa72e08c9748a287b6b Mon Sep 17 00:00:00 2001 From: christopher Date: Tue, 2 Jan 2018 19:19:38 -0500 Subject: [PATCH 29/29] elaborate on news --- news/schema.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/news/schema.rst b/news/schema.rst index 080f0e2ad..a4ea2deeb 100644 --- a/news/schema.rst +++ b/news/schema.rst @@ -1,11 +1,17 @@ **Added:** -* Schemas for the tables -* Docs auto generate for collections (if they were documented in the schema) +* Schemas and exemplars for the collections. +Database entries are checked against the schema, making sure that all the +required fields are filled and the values are the same type(s) listed in the +schema. The schema also includes descriptions of the data to be included. +The exemplars are examples which have all the specified fields and are +used to check the validation. + +* Docs auto generate for collections (if they were documented in the schema). **Changed:** -* App now validates incoming data +* App now validates incoming data against schema **Deprecated:** None