diff --git a/.gitignore b/.gitignore index b8aca4966..9c5f714dd 100644 --- a/.gitignore +++ b/.gitignore @@ -57,4 +57,15 @@ docs/_build/ target/ # Rever -rever/ \ No newline at end of file +rever/ + +# Auto sphinx docs +docs/collections/grades.rst +docs/collections/assignments.rst +docs/collections/grants.rst +docs/collections/abstracts.rst +docs/collections/blog.rst +docs/collections/people.rst +docs/collections/projects.rst +docs/collections/proposals.rst +docs/collections/students.rst diff --git a/.travis.yml b/.travis.yml index 24926a997..9d2238bbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,11 +20,14 @@ install: - conda update conda # Install dependencies - - conda install python + - conda create -n test --file requirements/run.txt + - source activate test - python setup.py install script: - set -e + - conda install --file requirements/test.txt + - pytest -vvs . - # Command to build your docs - pip install doctr - conda install --file requirements/doc.txt diff --git a/docs/collections/abstracts.rst b/docs/collections/abstracts.rst deleted file mode 100644 index a6523dd28..000000000 --- a/docs/collections/abstracts.rst +++ /dev/null @@ -1,82 +0,0 @@ -Abstracts -============ -Abstracts for a conference or workshop. This is generally public information - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str, Unique identifier for submission. This generally includes the author name and - part of the title. -:title: str, title of the presentation/paper. -:firstname: str, first name of the author. -:lastname: str, last name of the author. -:coauthors: str, names of coauthors, optional -:email: str, contact email for the author. -:institution: str, name of the inistitution -:references: str, HTML string of reference for the abstract itself, optional -:text: str, HTML string of the abstract. -:timestamp: The time when the abstract was submitted. - - -YAML Example ------------- - -.. code-block:: yaml - - Mouginot.Model: - title: Model Performance Analysis - firstname: Baptiste - lastname: Mouginot - coauthors: P.P.H. Wilson - email: mouginot@wisc.edu - institution: University of Wisconsin-Madison - references: "[1] B. MOUGINOT, “CyCLASS: CLASS models for Cyclus,”, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2\ - \ (2016).\n[2] B. Mouginot, P.P.H. Wilson, R.W. Carlsen, “Impact of Isotope Fidelity\ - \ on Fuel Cycle Calculations”, ANS Winter Conference, Las Vegas, (November 2016)\n\ - [3] B. Leniau, B. Mouginot, N. Thiollière, X. Doligez, A. Bidaud, F. Courtin,\ - \ M. Ernoult and S. David, “A neural network approach for burn-up calculation\ - \ and its application to the dynamic fuel cycle code CLASS,” Annals of Nuclear\ - \ Energy , 81 , 125 – 133 (2015).\n[4] B. Leniau, F. Courtin, B. Mouginot, N.\ - \ Thiollière, X. Doligez, A. Bidaud, “Generation of SFR Physics Models for the\ - \ Nuclear Fuel Cycle Code CLASS” PHYSOR 2016\n" - text: "The CLASS team has developed high quality predictors based on pre-trained\ - \ neural networks, allowing the estimation the evolution different neutronic parameters,\ - \ such as neutron multiplication factor or macroscopic cross sections, along the\ - \ irradiation of the fuel. This allows building various fuel fabrication and depletion\ - \ models for fuel cycle simulators. The cyCLASS package [1] has been developed\ - \ to allow the use of CLASS fabrication and cross section prediction models inside\ - \ Cyclus. cyCLASS provides a reactor facility and a fuel fabrication facility,\ - \ which are able to use any CLASS models to provide/request fuel to the entire\ - \ Cyclus ecosystem. Using cyCLASS, it has been possible to perform fuel cycle\ - \ simulations comparing different levels of archetypes fidelity[2].\n\nThis work\ - \ focuses on the analysis of the performance of some high fidelity models developed\ - \ from [3,4], extending the isotopic validity space from uranium and plutonium\ - \ to the most common transuranic elements for Light Water Reactors (LWR) and Sodium\ - \ Fast Reactors (SFR). Those extended models were required to study a transition\ - \ scenario from the actual US nuclear fleet to a SFR and LWR fleet reprocessing\ - \ the most commun transuranic elements (see “Recipe vs Model” presentation from\ - \ the same author). The present work aims to evaluate the following for each of\ - \ the models:\nthe performance relative to the training sample density,\nthe precision\ - \ topography inside and outside of the validity space,\nthe performance of the\ - \ burnup calculation for the cross section predictors.\nAs a complete set of real\ - \ data is not available to benchmark such models, their relative performances\ - \ will be evaluated with regards to the depletion tool used to train them.\n" - timestamp: 5/5/2017 13:15:59 - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - {"_id": "Mouginot.Model", - "coauthors": "P.P.H. Wilson", - "email": "mouginot@wisc.edu", - "firstname": "Baptiste", - "institution": "University of Wisconsin-Madison", - "lastname": "Mouginot", - "references": "[1] B. MOUGINOT, \u201ccyCLASS: CLASS models for Cyclus,\u201d, Figshare, https://dx.doi.org/10.6084/m9.figshare.3468671.v2 (2016).", - "text": "The CLASS team has developed high quality predictors based on pre-trained neural network...", - "timestamp": "5/5/2017 13:15:59", - "title": "Model Performance Analysis"} diff --git a/docs/collections/assignments.rst b/docs/collections/assignments.rst deleted file mode 100644 index 5e778f6ef..000000000 --- a/docs/collections/assignments.rst +++ /dev/null @@ -1,50 +0,0 @@ -Assignments -============ -Information about assignments for classes. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str, A unique id for the assignment, such a HW01-EMCH-558-2016-S -:courses: list of str, ids of the courses that have this assignment -:category: str, such as 'homework' or 'final' -:file: str, path to assignment file in store, optional -:solution: str, path to solution file in store, optional -:question: list of str, titles for the questions on this assignment, - optional, defaults to 1-indexed ints if absent. -:points: list of int or float, list of number of points possible for each - question. Length is the number of questions - - -YAML Example ------------- - -.. code-block:: yaml - - hw01-rx-power: - category: homework - courses: - - EMCH-558-2016-S - - EMCH-758-2016-S - points: - - 1 - - 2 - - 3 - questions: - - 1-9 - - 1-10 - - 1-12 - - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - {"_id": "hw01-rx-power", - "category": "homework", - "courses": ["EMCH-558-2016-S", "EMCH-758-2016-S"], - "points": [1, 2, 3], - "questions": ["1-9", "1-10", "1-12"]} diff --git a/docs/collections/blank.rst b/docs/collections/blank.rst deleted file mode 100644 index 061df6465..000000000 --- a/docs/collections/blank.rst +++ /dev/null @@ -1,21 +0,0 @@ - -============ - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str, - - -YAML Example ------------- - -.. code-block:: yaml - - -JSON/Mongo Example ------------------- - -.. code-block:: json - diff --git a/docs/collections/blog.rst b/docs/collections/blog.rst deleted file mode 100644 index 9e387cb3e..000000000 --- a/docs/collections/blog.rst +++ /dev/null @@ -1,47 +0,0 @@ -Blog -============ -This collection represents blog posts written by the members of the research group. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str, short represntation, such as this-is-my-title -:title: str, full human readable title -:original: str, URL of original post, if this is a repost, optional -:author: str, name or AKA of author -:year: int, Publication year -:month: str, Publication month -:day: int, Publication day -:post: str, actual contents of the post - - -YAML Example ------------- - -.. code-block:: yaml - - my-vision: - title: My Vision - author: Anthony Scopatz - year: 2015 - month: September - day: 18 - original: https://scopatz.com/my-vision/ - post: | - I would like see things move forward. Deep, I know! - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - {"_id": "my-vision", - "title": "My Vision", - "author": "Anthony Scopatz", - "year": 2015, - "month": "September", - "day": 18, - "original": "https://scopatz.com/my-vision/", - "post": "I would like see things move forward. Deep, I know!"} diff --git a/docs/collections/grades.rst b/docs/collections/grades.rst deleted file mode 100644 index 726af2f50..000000000 --- a/docs/collections/grades.rst +++ /dev/null @@ -1,39 +0,0 @@ -Grades -============ -The grade for a student on an assignment. This information should be private. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str, unique id, typically the student-assignment-course -:student: str, student id -:assignment: str, assignment id -:course: str, course id -:scores: list of int or float, the number of points earned on each question -:filename: str, path to file in store, optional - - -YAML Example ------------- - -.. code-block:: yaml - - Human A. Person-rx-power-hw02-EMCH-758-2017-S: - assignment: 2017-rx-power-hw02 - course: EMCH-758-2017-S - scores: - - 1 - - 1.6 - - 3 - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - {"_id": "Human A. Person-rx-power-hw02-EMCH-758-2017-S", - "assignment": "2017-rx-power-hw02", - "course": "EMCH-758-2017-S", - "scores": [1, 1.6, 3]} diff --git a/docs/collections/grants.rst b/docs/collections/grants.rst deleted file mode 100644 index 802b3e153..000000000 --- a/docs/collections/grants.rst +++ /dev/null @@ -1,93 +0,0 @@ -Grants -============ -This collection represents grants that have been awarded to the group. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str or number, short represntation, such as this-is-my-name -:title: str, actual title of proposal / grant -:funder: str, the agency funding the work -:program: str, the program the work was funded under -:grant_id: str, optional, the identfier for this work, eg #42024 -:call_for_proposals: str, optional, URL to the call for proposals -:narrative: str, optional, URL of document -:benefit_of_collaboration: str, optional, URL of document -:amount: int or float, value of award -:currency: str, typically '$' or 'USD' -:begin_year: int, start year of the grant -:begin_month: str, start month of the grant -:begin_day: int, start day of the grant, optional -:end_year: int, end year of the grant -:end_month": str, end month of the grant -:end_day: str, end day of teh grant, optional -:team: list of dicts, information about the team members participating in the grant. - These dicts have the following form: - - .. code-block:: python - - [{"name": str, # should match a person's name or AKA - "position": str, # PI, Co-PI, Co-I, Researcher, etc. - "institution": str, # The institution of this investigator - "subaward_amount", int or float, # optional - "cv": str, # optional, URL of document - }, - ... - ] - - -YAML Example ------------- - -.. code-block:: yaml - - - SymPy-1.1: - amount: 3000.0 - begin_day: 1 - begin_month: May - begin_year: 2017 - call_for_proposals: https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ - end_day: 31 - end_month: December - end_year: 2017 - funder: NumFOCUS - narrative: https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing - program: Small Development Grants - team: - - institution: University of South Carolina - name: Anthony Scopatz - position: PI - - institution: University of South Carolina - name: Aaron Meurer - position: researcher - title: SymPy 1.1 Release Support - - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - {"_id": "SymPy-1.1", - "amount": 3000.0, - "begin_day": 1, - "begin_month": "May", - "begin_year": 2017, - "call_for_proposals": "https://groups.google.com/d/msg/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ", - "end_day": 31, - "end_month": "December", - "end_year": 2017, - "funder": "NumFOCUS", - "narrative": "https://docs.google.com/document/d/1nZxqoL-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp=sharing", - "program": "Small Development Grants", - "team": [{"institution": "University of South Carolina", - "name": "Anthony Scopatz", - "position": "PI"}, - {"institution": "University of South Carolina", - "name": "Aaron Meurer", - "position": "researcher"} - ], - "title": "SymPy 1.1 Release Support"} diff --git a/docs/collections/people.rst b/docs/collections/people.rst deleted file mode 100644 index eca73904b..000000000 --- a/docs/collections/people.rst +++ /dev/null @@ -1,453 +0,0 @@ -People -============ -This collection describes the members of the research group. This is normally public -data. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str, unique identifier for the group member -:name: str, Full, canonical name for the person -:title: str, for example, Dr., etc. -:position: str, such as professor, graduate student, or scientist -:email: str, email address of the group member, optional. -:aka: list of str, list of aliases (also-known-as), useful for identifying - the group member in citations or elsewhere. -:avatar: str, URL to avatar -:bio: str, short biographical text -:active: bool, If the person is an active member, default True. -:collab: bool, If the person is a collaborator, default False. -:education: list of dictionaries, This contains the educational information for - the group member. These dicts have the following form: - - .. code-block:: python - - [{"institution": str, - "location": str, - "degree": str, - "begin_year": int, - "begin_month": str, # optional - "end_year": int, - "end_month": str, # optional - "gpa": float or str, # optional - "other": [str], # list of strings of other pieces of information - }, - ... - ] - -:employment: list of dicts, Employment information, similar to educational information. - These dicts have the following form: - - .. code-block:: python - - [{"organization": str, - "location": str, - "position": str, - "begin_year": int, - "begin_month": str, # optional - "end_year": int, - "end_month": str, # optional - "other": [str], # list of strings of other pieces of information - }, - ... - ] - -:funding: list of dicts, Funding and scholarship that the group member has - individually obtained in the past (optional). **WARNING:** this is not to be confused - with the **grants** collection, that may refee back to group members. - These dicts have the following form: - - .. code-block:: python - - [{"name": str, - "value": float, - "currency": str, # optional, defaults to "$" - "year": int, - "month": str, # optional - "duration": str or int, # optional length of award - }, - ... - ] - -:service: list of dicts, Service that this group member has provided (optional). - These dicts have the following form: - - .. code-block:: python - - [{"name": str, - "description": str, # optional - "year": int, - "month": str, # optional - "duration": str or int, # optional length of service - }, - ... - ] - -:honors: list of dicts, Honors that have been awarded to this group member (optional). - These dicts have the following form: - - .. code-block:: python - - [{"name": str, - "description": str, # optional - "year": int, - "month": str, # optional - }, - ... - ] - -:teaching: list of dicts, Courses that this group member has taught, if any (optional). - These dicts have the following form: - - .. code-block:: python - - [{"course": str, # name of the course - "organization": str, - "position": str, - "year": int, - "month": str, # optional - "end_year": int, # optional - "end_month": str, # optional - "description": str, # optional - "website": str, # optional URL - "syllabus": str, # optional URL - "video": str, # optional URL - "materials": str, # optional URL - }, - ... - ] - -:membership: list of dicts, Profesional organizations this member is a part of (optional). - These dicts have the following form: - - .. code-block:: python - - [{"organization": str, - "position": str, - "description": str, # optional - "begin_year": int, - "begin_month": str, # optional - "end_year": int, # optional - "end_month": str, # optional - "website": str, # optional URL - }, - ... - ] - -:skills: list of dicts, Skill the group member has (optional) - These dicts have the following form: - - .. code-block:: python - - [{"name": str, - "category": str, - "level": str - }, - ... - ] - - - -YAML Example ------------- - -.. code-block:: yaml - - scopatz: - name: Anthony Scopatz - position: professor - title: Dr. - email: scopatz@cec.sc.edu - aka: - - Scopatz, A. M. - - Scopatz, Anthony - - Scopatz, Anthony M - - Scopatz, Anthony M. - - A. M. Scopatz - - Anthony M Scopatz - - Anthony M. Scopatz - - Anthony Michael Scopatz - avatar: https://avatars1.githubusercontent.com/u/320553?v=3&s=200 - bio: Anthony Scopatz is currently an Assistant Professor at the University of South - Carolina in the Nuclear Engineering program in the Mechanical Engineering Department. - He is a computational physicist and long time Python developer. Anthony holds - his BS in Physics from UC, Santa Barbara and a Ph.D. in Mechanical / Nuclear Engineering - from UT Austin. A former Enthought employee, he spent his post-doctoral studies - at the FLASH Center at the University of Chicago in the Astrophysics Department. - Then he became a Staff Scientist at the University of Wisconsin-Maidson in Engineering - Physics. Anthony's research interests revolve around essential physics modeling - of the nuclear fuel cycle, and information theory & entropy. Anthony is proudly - a fellow of the Python Software Foundation and has published and spoken at numerous - conferences on a variety of science & software development topics. - education: - - begin_year: 2008 - degree: Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program - end_year: 2011 - institution: The University of Texas at Austin - location: Austin, TX - other: - - 'Adviser: Erich A. Schneider' - - 'Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis' - - begin_year: 2006 - degree: M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program - end_year: 2007 - institution: The University of Texas at Austin - location: Austin, TX - other: - - 'Adviser: Erich A. Schneider' - - 'Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership' - - begin_year: 2002 - degree: B.S. Physics - end_year: 2006 - institution: University of California, Santa Barbara - location: Santa Barbara, CA - other: - - Graduated with a Major in Physics and a Minor in Mathematics - employment: - - begin_year: 2015 - location: Columbia, SC - organization: The University of South Carolina - other: - - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' - - 'PyNE: The Nuclear Engineering Toolkit.' - - 'Website: http://www.ergs.sc.edu/' - position: Assistant Professor, Mechanical Engineering Department - - begin_year: 2013 - end_year: 2015 - location: Madison, WI - organization: CNERG, The University of Wisconsin-Madison - other: - - 'Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.' - - 'PyNE: The Nuclear Engineering Toolkit.' - - 'Website: https://cnerg.github.io/' - position: Associate Scientist, Engineering Physics Department - - begin_month: Nov - begin_year: 2011 - end_month: May - end_year: 2013 - location: Chicago, IL - organization: The FLASH Center, The University of Chicago - other: - - 'NIF: Simulation of magnetic field generation from neutral plasmas using - FLASH.' - - 'CosmoB: Simulation of magnetic field generation from neutral plasmas using - FLASH.' - - 'FLASH4: High-energy density physics capabilities and utilities.' - - 'Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from - FLASH.' - - 'OpacPlot: HDF5-based equation of state and opacity file format.' - - 'Website: http://flash.uchicago.edu/site/' - position: Research Scientist, Postdoctoral Scholar, Astronomy & Astrophysics - Dept. - - begin_month: May - begin_year: 2010 - end_month: October - end_year: 2011 - location: Austin, TX - organization: Enthought, Inc. - other: - - 'PlotTool: Time series data visualization for J.P. Morgan Chase.' - - 'PivotTable: Data cube configuration, management, and visualization for - J.P. Morgan Chase.' - - 'WIPP-PA: Waste Isolation Pilot Project Performance Assessment tool for - Sandia Nat’l Labs.' - - 'EasyGuide2: Dental implant imaging software for Keystone Dental.' - position: Scientific Software Developer - - begin_year: 2002 - end_year: 2005 - location: Santa Barbara, CA - organization: University of California, Santa Barbara - other: - - Terahertz Continuous Wave Spectroscopy - - Studied the effect of Terahertz waves on biomacromolecules in water. - position: Undergraduate Research Assistant - - begin_month: June - begin_year: 2004 - end_month: August - end_year: 2004 - location: Los Alamos, NM - organization: Los Alamos National Laboratory - other: - - Terahertz Pump-Probe Spectroscopy - - Looked at the effect of Terahertz waves on Erbium-Arsenide on a Gallium-Arsenide - substrate. - position: Summer Internship/Visiting Scientist - funding: - - name: Omega Laser User's Group Travel Award - value: 1100 - year: 2013 - - name: NIF User's Group Travel Award - value: 1150 - year: 2013 - - name: Innovations in Fuel Cycle Research, 2nd place Systems Engineering and - Analysis - value: 2500 - year: 2010 - - name: 'MCNPX Bug #100' - value: 20 - year: 2009 - membership: - - begin_year: 2006 - organization: American Nuclear Society - position: Member - - begin_year: 2013 - organization: Python Software Foundation - position: Fellow - - begin_year: 2011 - end_year: 2014 - organization: NumFOCUS - position: Founding Treasurer & Board Member - website: http://numfocus.org - service: - - name: SciPy 2014 Communications Chair - year: 2014 - - name: SciPy 2013 Communications Chair - year: 2013 - - name: SciPy 2012 Open Spaces Chair - year: 2012 - - name: SciPy 2011 Chair of Python & Core Technologies Track - year: 2011 - - name: PyCon 2010 Program Committee Member - year: 2010 - - name: Peer Reviewer for Nuclear Engineering & Design (NED-D-09-00256) - year: 2009 - skills: - - category: Programming Languages - level: expert - name: Python - - category: Programming Languages - level: expert - name: Cython - - category: Programming Languages - level: expert - name: C - - category: Programming Languages - level: expert - name: C++ - - category: Programming Languages - level: expert - name: Bash - - category: Specialized Software - level: expert - name: Linux - - category: Specialized Software - level: intermediate - name: MongoDB - teaching: - - course: 'EMCH 552: Intro to Nuclear Engineering' - description: This course is an introduction to nuclear physics and engineering, - covering the important underlying science and mathematics to nuclear power - generation. Unlike previous sememsters, this course was taught in a flipped - style. - month: August - organization: University of South Carolina - position: Professor - syllabus: https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc - year: 2017 - - course: 'EMCH 558/758: Reactor Power Systems' - description: This course covers conventional Pressurized Water Reactors (PWR) - and Boiling Water Reactors (BWR) system energy designs for transport, containment, - and accident prevention and mitigation. Near term, developmental, and proposed - enhanced capability reactor system designs. Cross listed as an undergraduate - and graduate course. - month: January - organization: University of South Carolina - position: Professor - syllabus: https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing - year: 2017 - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - {"_id": "scopatz", - "name": "Anthony Scopatz", - "position": "professor", - "title": "Dr.", - "aka": ["Scopatz", "Scopatz, A", "Scopatz, A.", "Scopatz, A M", "Anthony Michael Scopatz"], - "avatar": "https://avatars1.githubusercontent.com/u/320553?v=3&s=200", - "email": "scopatz@cec.sc.edu", - "bio": "Anthony Scopatz is currently an Assistant Professor", - "education": [{"begin_year": 2008, - "degree": "Ph.D. Mechanical Engineering, Nuclear and Radiation Engineering Program", - "end_year": 2011, - "institution": "The University of Texas at Austin", - "location": "Austin, TX", - "other": ["Adviser: Erich A. Schneider", - "Dissertation: Essential Physics for Fuel Cycle Modeling & Analysis"]}, - {"begin_year": 2006, - "degree": "M.S.E. Mechanical Engineering, Nuclear and Radiation Engineering Program", - "end_year": 2007, - "institution": "The University of Texas at Austin", - "location": "Austin, TX", - "other": ["Adviser: Erich A. Schneider", - "Thesis: Recyclable Uranium Options under the Global Nuclear Energy Partnership"]}, - {"begin_year": 2002, - "degree": "B.S. Physics", - "end_year": 2006, - "institution": "University of California, Santa Barbara", - "location": "Santa Barbara, CA", - "other": ["Graduated with a Major in Physics and a Minor in Mathematics"]}], - "employment": [{"begin_year": 2015, - "location": "Columbia, SC", - "organization": "The University of South Carolina", - "other": ["Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.", - "PyNE: The Nuclear Engineering Toolkit.", - "Website: http://www.ergs.sc.edu/"], - "position": "Assistant Professor, Mechanical Engineering Department"}, - {"begin_year": 2013, - "end_year": 2015, - "location": "Madison, WI", - "organization": "CNERG, The University of Wisconsin-Madison", - "other": ["Cyclus: An agent-based, discrete time nuclear fuel cycle simulator.", - "PyNE: The Nuclear Engineering Toolkit.", - "Website: https://cnerg.github.io/"], - "position": "Associate Scientist, Engineering Physics Department"}, - {"begin_month": "Nov", - "begin_year": 2011, - "end_month": "May", - "end_year": 2013, - "location": "Chicago, IL", - "organization": "The FLASH Center, The University of Chicago", - "other": ["NIF: Simulation of magnetic field generation from neutral plasmas using FLASH.", "CosmoB: Simulation of magnetic field generation from neutral plasmas using FLASH.", - "FLASH4: High-energy density physics capabilities and utilities.", - "Simulated Diagnostics: Schlieren, shadowgraphy, Langmuir probes, etc. from FLASH.", - "OpacPlot: HDF5-based equation of state and opacity file format.", - "Website: http://flash.uchicago.edu/site/"], - "position": "Research Scientist, Postdoctoral Scholar"}], - "funding": [{"name": "Omega Laser User's Group Travel Award", - "value": 1100, - "year": 2013}, - {"name": "NIF User's Group Travel Award", - "value": 1150, - "year": 2013}], - "membership": [{"begin_year": 2006, - "organization": "American Nuclear Society", - "position": "Member"}, - {"begin_year": 2013, - "organization": "Python Software Foundation", - "position": "Fellow"}], - "skills": [{"category": "Programming Languages", - "level": "expert", - "name": "Python"}, - {"category": "Programming Languages", - "level": "expert", - "name": "Cython"}], - "teaching": [{"course": "EMCH 552: Intro to Nuclear Engineering", - "description": "This course is an introduction to nuclear physics.", - "month": "August", - "organization": "University of South Carolina", - "position": "Professor", - "syllabus": "https://drive.google.com/open?id=0BxUpd34yizZreDBCMEJNY2FUbnc", - "year": 2017}, - {"course": "EMCH 558/758: Reactor Power Systems", - "description": "This course covers conventional reactors.", - "month": "January", - "organization": "University of South Carolina", - "position": "Professor", - "syllabus": "https://docs.google.com/document/d/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8-PxiboYdM/edit?usp=sharing", - "year": 2017}] - } diff --git a/docs/collections/projects.rst b/docs/collections/projects.rst deleted file mode 100644 index 2494119b3..000000000 --- a/docs/collections/projects.rst +++ /dev/null @@ -1,71 +0,0 @@ -Projects -============ -This collection describes the research group projects. This is normally public data. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str, Unique project identifier. -:name: str, name of the project. -:description: str, brief project description. -:website: str, URL of the website. -:repo: str, URL of the source code repo, if available, optional. -:logo: str, URL to the project logo, optional. -:other: list of str, other information about the project, optional/ -:team: list of dict, People who are/have been woking on this project. - These dicts have the following structure: - - .. code-block:: python - - [{"name": str, # should match a person's name or AKA - "position": str, - "begin_year": int, - "begin_month": str, # optional - "end_year": int, # optional - "end_month": str, # optional - }, - ... - ] - - -YAML Example ------------- - -.. code-block:: yaml - - Cyclus: - description: Agent-Based Nuclear Fuel Cycle Simulator - logo: http://fuelcycle.org/_static/big_c.png - other: - - Discrete facilities with discrete material transactions - - Low barrier to entry, rapid payback to adoption - repo: https://github.com/cyclus/cyclus/ - team: - - begin_month: June - begin_year: 2013 - end_month: July - end_year: 2015 - name: Anthony Scopatz - position: Project Lead - website: http://fuelcycle.org/ - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - {"_id": "Cyclus", - "description": "Agent-Based Nuclear Fuel Cycle Simulator", - "logo": "http://fuelcycle.org/_static/big_c.png", - "other": ["Discrete facilities with discrete material transactions", - "Low barrier to entry, rapid payback to adoption"], - "repo": "https://github.com/cyclus/cyclus/", - "team": [{"begin_month": "June", - "begin_year": 2013, - "end_month": "July", - "end_year": 2015, - "name": "Anthony Scopatz", - "position": "Project Lead"}], - "website": "http://fuelcycle.org/"} diff --git a/docs/collections/proposals.rst b/docs/collections/proposals.rst deleted file mode 100644 index f69c4985c..000000000 --- a/docs/collections/proposals.rst +++ /dev/null @@ -1,108 +0,0 @@ -Proposals -============ -This collection represents proposals that have been submitted by the group. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str or number, short represntation, such as this-is-my-name -:title: str, actual title of proposal -:pi: str, principal investigator name -:authors: list of str, other investigator names -:status: str, e.g. 'submitted', 'accepted', 'rejected' -:ammount: int or float, value of award -:currency: str, typically '$' or 'USD' -:durration: int or float, number of years -:year: int, Year that the proposal is due -:month: str, month that the proposal is due -:day: int, day that the proposal is due -:pre: dict, Information about the pre-proposal, optional. - This dict has the following form: - - .. code-block:: python - - {"year": int, - "month": str, - "day": int, - "narrative": str, # URL of document - "benefit_of_collaboration": str, # URL of document - "cv": list of str, # URL to documents - } - -:full: Information about the pre-proposal, optional - This dict has the following form: - - .. code-block:: python - - {"narrative": str, # URL of document - "benefit_of_collaboration": str, # URL of document - "cv": list of str, # URL to documents - } - - -YAML Example ------------- - -.. code-block:: yaml - - mypropsal: - title: A very fine proposal indeed - pi: Anthony Scopatz - authors: - - Anthony Scopatz - - Robert Flanagan - status: submitted - ammount: 1000000.0 - currency: USD - durration: 3 - year: 1999 - month: Aug - day: 18 - pre: - year: 1998 - month: Aug - day: 2 - narrative: http://some.com/pdf - benefit_of_collaboration: http://pdf.com/benefit_of_collaboration - cv: - - http://pdf.com/scopatz-cv - - http://pdf.com/flanagan-cv - full: - narrative: http://some.com/pdf - benefit_of_collaboration: http://pdf.com/benefit_of_collaboration - cv: - - http://pdf.com/scopatz-cv - - http://pdf.com/flanagan-cv - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - {"_id": "mypropsal", - "title": "A very fine proposal indeed", - "pi": "Anthony Scopatz", - "authors": ["Anthony Scopatz", - "Robert Flanagan"], - "status": "submitted", - "ammount": 1000000.0, - "currency": "USD", - "durration": 3, - "year": 1999, - "month": "Aug", - "day": 18, - "pre": {"year": 1998, - "month": "Aug", - "day": 2, - "narrative": "http://some.com/pdf", - "benefit_of_collaboration": "http://pdf.com/benefit_of_collaboration", - "cv": ["http://pdf.com/scopatz-cv", - "http://pdf.com/flanagan-cv"]}, - "full": {"narrative": "http://some.com/pdf" - "benefit_of_collaboration": "http://pdf.com/benefit_of_collaboration", - "cv": ["http://pdf.com/scopatz-cv", - "http://pdf.com/flanagan-cv"]} - } - diff --git a/docs/collections/students.rst b/docs/collections/students.rst deleted file mode 100644 index 7ea2562e3..000000000 --- a/docs/collections/students.rst +++ /dev/null @@ -1,34 +0,0 @@ -Students -============ -This is a collection of student names and metadata. This should probably be private. - -Schema ------- -The following lists key names mapped to its type and meaning for each entry. - -:_id: str, short represntation, such as this-is-my-name -:aka: list of str, list of aliases, optional -:email: str, email address, optional -:university_id: str, The university identifier for the student, optional - -YAML Example ------------- - -.. code-block:: yaml - - Human A. Person: - aka: - - H. A. Person - email: haperson@uni.edu - university_id: HAP42 - - -JSON/Mongo Example ------------------- - -.. code-block:: json - - {"_id": "Human A. Person", - "aka": ["H. A. Person"], - "email": "haperson@uni.edu", - "university_id": "HAP42"} diff --git a/docs/conf.py b/docs/conf.py index 79d263fe6..6b5c76f71 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,9 +11,15 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import json +import tempfile +from textwrap import indent + import cloud_sptheme as csp + from regolith import __version__ as REGOLITH_VERSION +from regolith.fsclient import json_to_yaml +from regolith.schemas import SCHEMAS, EXEMPLARS # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -73,7 +79,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build', 'collections/blank.rst'] +exclude_patterns = ['_build', ] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None @@ -102,7 +108,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'redcloud' -#html_theme = 'blackcloud' +# html_theme = 'blackcloud' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -253,3 +259,68 @@ # How to display URL addresses: 'footnote', 'no', or 'inline'. # texinfo_show_urls = 'footnote' + + +def format_key(schema, key, indent_str=''): + s = '' + line_format = ':{key}: {type}, {description}, required\n' + line_format_o = ':{key}: {type}, {description}, optional\n' + if not schema.get('required', False): + lf = line_format_o + else: + lf = line_format + if 'type' in schema and 'description' in schema: + s += indent(lf.format(key=key, + description=schema.get('description', ''), + type=schema.get('type', '')), indent_str) + elif 'type' in schema[key]: + s += indent(lf.format(key=key, + description=schema[key].get('description', ''), + type=schema[key].get('type', '')), indent_str) + s = s.replace(', , ', ', ') + if schema.get('schema', False): + s += '\n' + for inner_key in schema.get('schema', ()): + s += format_key(schema['schema'], inner_key, + indent_str=indent_str + '\t') + + return s + + +def build_schema_doc(key): + fn = 'collections/' + key + '.rst' + with open(fn, 'w') as f: + s = '' + s += key.title() + '\n' + s += '=' * len(key) + '\n' + s += SCHEMAS[key]['_description']['description'] + '\n\n' + s += 'Schema\n------\nThe following lists key names mapped to its ' \ + 'type and meaning for each entry.\n\n' + schema = SCHEMAS[key] + schema_list = list(schema.keys()) + schema_list.sort() + for k in schema_list: + if k not in ['_description']: + s += format_key(schema[k], key=k) + s += '\n\n' + s += 'YAML Example\n------------\n\n' + s += '.. code-block:: yaml\n\n' + temp = tempfile.NamedTemporaryFile() + temp2 = tempfile.NamedTemporaryFile() + with open(temp.name, 'w') as ff: + json.dump(EXEMPLARS[key], ff, sort_keys=True) + jd = json.dumps(EXEMPLARS[key], sort_keys=True, + indent=4, separators=(',', ': ')) + json_to_yaml(temp.name, temp2.name) + with open(temp2.name, 'r') as ff: + s += indent(ff.read(), '\t') + s += '\n\n' + s += 'JSON/Mongo Example\n------------------\n\n' + s += '.. code-block:: json\n\n' + s += indent(jd, '\t') + s += '\n' + f.write(s) + + +for k in SCHEMAS: + build_schema_doc(k) diff --git a/news/schema.rst b/news/schema.rst new file mode 100644 index 000000000..a4ea2deeb --- /dev/null +++ b/news/schema.rst @@ -0,0 +1,22 @@ +**Added:** + +* Schemas and exemplars for the collections. +Database entries are checked against the schema, making sure that all the +required fields are filled and the values are the same type(s) listed in the +schema. The schema also includes descriptions of the data to be included. +The exemplars are examples which have all the specified fields and are +used to check the validation. + +* Docs auto generate for collections (if they were documented in the schema). + +**Changed:** + +* App now validates incoming data against schema + +**Deprecated:** None + +**Removed:** None + +**Fixed:** None + +**Security:** None diff --git a/regolith/app.py b/regolith/app.py index 135a122c3..069cf592c 100644 --- a/regolith/app.py +++ b/regolith/app.py @@ -6,6 +6,9 @@ from flask import Flask, abort, request, render_template, redirect, url_for +from regolith.schemas import validate + + app = Flask('regolith') @@ -55,10 +58,23 @@ def collection_page(dbname, collname): n = os.path.join(td.name, 'regolith.txt') print('Error in json parsing writing text file to {}. ' 'Please try again.'.format(n)) - with open(n) as f: + with open(n, 'w') as f: f.write(form['body']) traceback.print_exc() raise + tv, errors = validate(dbname, body) + if not tv: + td = tempfile.TemporaryDirectory() + n = os.path.join(td.name, 'regolith.txt') + with open(n, 'w') as f: + f.write(form['body']) + raise ValueError('Error while validating the record,' + ' writing text file to {}. ' + 'Please try again.\n\n' + 'Your errors were\n' + '------------------' + '{}'.format(n, errors)) + rc.client.update_one(dbname, collname, {'_id': body['_id']}, body) status = 'saved ✓' status_id = str(body['_id']) @@ -75,6 +91,18 @@ def collection_page(dbname, collname): f.write(form['body']) traceback.print_exc() raise + tv, errors = validate(dbname, body) + if not tv: + td = tempfile.TemporaryDirectory() + n = os.path.join(td.name, 'regolith.txt') + with open(n) as f: + f.write(form['body']) + raise ValueError('Error while validating the record,' + ' writing text file to {}. ' + 'Please try again.\n\n' + 'Your errors were\n' + '------------------' + '{}'.format(n, errors)) try: added = rc.client.insert_one(dbname, collname, body) except Exception: diff --git a/regolith/fsclient.py b/regolith/fsclient.py index ba4639ca4..fb3f0b705 100644 --- a/regolith/fsclient.py +++ b/regolith/fsclient.py @@ -6,6 +6,7 @@ from collections import defaultdict from ruamel.yaml import YAML +import ruamel.yaml from regolith.tools import dbdirname, dbpathname @@ -48,10 +49,14 @@ def dump_yaml(filename, docs, inst=None): """Dumps a dict of documents into a file.""" inst = YAML() if inst is None else inst inst.indent(mapping=2, sequence=4, offset=2) - for doc in docs.values(): + sorted_dict = ruamel.yaml.comments.CommentedMap() + for k, doc in docs.items(): _id = doc.pop('_id') + sorted_dict[k] = ruamel.yaml.comments.CommentedMap() + for kk in sorted(doc.keys()): + sorted_dict[k][kk] = doc[kk] with open(filename, 'w') as fh: - inst.dump(docs, stream=fh) + inst.dump(sorted_dict, stream=fh) def json_to_yaml(inp, out): diff --git a/regolith/schemas.py b/regolith/schemas.py new file mode 100644 index 000000000..e3b543bdb --- /dev/null +++ b/regolith/schemas.py @@ -0,0 +1,781 @@ +"""Database schemas, examples, and tools""" +from cerberus import Validator + +EXEMPLARS = { + 'abstracts': {'_id': 'Mouginot.Model', + 'coauthors': 'P.P.H. Wilson', + 'email': 'mouginot@wisc.edu', + 'firstname': 'Baptiste', + 'institution': 'University of Wisconsin-Madison', + 'lastname': 'Mouginot', + 'references': '[1] B. MOUGINOT, “cyCLASS: CLASS ' + 'models for Cyclus,”, Figshare, ' + 'https://dx.doi.org/10.6084/' + 'm9.figshare.3468671.v2 (2016).', + 'text': 'The CLASS team has developed high ' + 'quality predictors based on pre-trained ' + 'neural network...', + 'timestamp': '5/5/2017 13:15:59', + 'title': 'Model Performance Analysis'}, + 'assignments': {'_id': 'hw01-rx-power', + 'category': 'homework', + 'courses': ['EMCH-558-2016-S', + 'EMCH-758-2016-S'], + 'points': [1, 2, 3], + 'questions': ['1-9', '1-10', '1-12']}, + 'blog': {'_id': 'my-vision', + 'author': 'Anthony Scopatz', + 'day': 18, + 'month': 'September', + 'original': 'https://scopatz.com/my-vision/', + 'post': 'I would like see things move forward. Deep, I know!', + 'title': 'My Vision', + 'year': 2015}, + 'citations': {'_id': 'meurer2016sympy', + 'author': ['Meurer, Aaron', + 'Smith, Christopher P', + 'Paprocki, Mateusz', + "{\\v{C}}ert{\\'\\i}k, Ond{\\v{r}}ej", + 'Rocklin, Matthew', + 'Kumar, AMiT', + 'Ivanov, Sergiu', + 'Moore, Jason K', + 'Singh, Sartaj', + 'Rathnayake, Thilina', + 'Sean Vig', + 'Brian E Granger', + 'Richard P Muller', + 'Francesco Bonazzi', + 'Harsh Gupta', + 'Shivam Vats', + 'Fredrik Johansson', + 'Fabian Pedregosa', + 'Matthew J Curry', + 'Ashutosh Saboo', + 'Isuru Fernando', + 'Sumith Kulal', + 'Robert Cimrman', + 'Anthony Scopatz'], + 'entrytype': 'article', + 'journal': 'PeerJ Computer Science', + 'month': 'Jan', + 'pages': 'e103', + 'publisher': 'PeerJ Inc. San Francisco, USA', + 'title': 'SymPy: Symbolic computing in Python', + 'volume': '4', + 'year': '2017'}, + 'courses': {'_id': 'EMCH-552-2016-F', + 'active': False, + 'department': 'EMCH', + 'number': 552, + 'scale': [[0.875, 'A'], + [0.8125, 'B+'], + [0.75, 'B'], + [0.6875, 'C+'], + [0.625, 'C'], + [0.5625, 'D+'], + [0.5, 'D'], + [-1.0, 'F']], + 'season': 'F', + 'students': ['Human A. Person', 'Human B. Person'], + 'syllabus': 'emch552-2016-f-syllabus.pdf', + 'weights': {'class-notes': 0.15, + 'final': 0.3, + 'homework': 0.35, + 'midterm': 0.2}, + 'year': 2016}, + 'grades': { + '_id': 'Human A. Person-rx-power-hw02-EMCH-758-2017-S', + 'student': 'hap', + 'assignment': '2017-rx-power-hw02', + 'course': 'EMCH-758-2017-S', + 'scores': [1, 1.6, 3]}, + 'grants': {'_id': 'SymPy-1.1', + 'amount': 3000.0, + 'begin_day': 1, + 'begin_month': 'May', + 'begin_year': 2017, + 'call_for_proposals': 'https://groups.google.com/d/msg' + '/numfocus/wPjhdm8NJiA/S8JL1_NZDQAJ', + 'end_day': 31, + 'end_month': 'December', + 'end_year': 2017, + 'funder': 'NumFOCUS', + 'narrative': 'https://docs.google.com/document/d/1nZxqoL' + '-Ucni_aXLWmXtRDd3IWqW0mZBO65CEvDrsXZM/edit?usp' + '=sharing', + 'program': 'Small Development Grants', + 'team': [ + {'institution': 'University of South Carolina', + 'name': 'Anthony Scopatz', + 'position': 'PI'}, + {'institution': 'University of South Carolina', + 'name': 'Aaron Meurer', + 'position': 'researcher'}], + 'title': 'SymPy 1.1 Release Support'}, + 'jobs': {'_id': '0004', + 'background_fields': ['Data Science', + 'Data Engineering', + 'Computer Engineering', + 'Computer Science', + 'Applied Mathematics', + 'Physics', + 'Nuclear Engineering', + 'Mechanical Engineering', + 'Or similar'], + 'compensation': [ + 'Salary and compensation will be based on prior work ' + 'experience.'], + 'contact': 'Please send CV or resume to Prof. Scopatz at ' + 'scopatzATcec.sc.edu.', + 'day': 1, + 'description': '

We are seeking a dedicated individual to ' + 'help to aid in ...', + 'month': 'July', + 'open': False, + 'positions': ['Scientific Software Developer', + 'Programmer'], + 'start_date': 'ASAP', + 'title': 'Open Source Scientific Software Maintainer', + 'year': 2015}, + 'news': {'_id': '56b4eb6d421aa921504ef2a9', + 'author': 'Anthony Scopatz', + 'body': 'Dr. Robert Flanagan joined ERGS as a post-doctoral ' + 'scholar.', + 'day': 1, + 'month': 'February', + 'year': 2016}, + 'people': {'_id': 'scopatz', + 'aka': ['Scopatz', + 'Scopatz, A', + 'Scopatz, A.', + 'Scopatz, A M', + 'Anthony Michael Scopatz'], + 'avatar': 'https://avatars1.githubusercontent.com/u/320553?v' + '=3&s=200', + 'bio': 'Anthony Scopatz is currently an Assistant Professor', + 'education': [ + {'begin_year': 2008, + 'degree': 'Ph.D. Mechanical Engineering, ' + 'Nuclear and Radiation Engineering ' + 'Program', + 'end_year': 2011, + 'institution': 'The University of Texas at Austin', + 'location': 'Austin, TX', + 'other': [ + 'Adviser: Erich A. Schneider', + 'Dissertation: Essential Physics for Fuel Cycle ' + 'Modeling & Analysis']}, + {'begin_year': 2006, + 'degree': 'M.S.E. Mechanical Engineering, Nuclear and ' + 'Radiation Engineering Program', + 'end_year': 2007, + 'institution': 'The University of Texas at Austin', + 'location': 'Austin, TX', + 'other': [ + 'Adviser: Erich A. Schneider', + 'Thesis: Recyclable Uranium Options under the Global ' + 'Nuclear Energy Partnership']}, + {'begin_year': 2002, + 'degree': 'B.S. Physics', + 'end_year': 2006, + 'institution': 'University of California, Santa Barbara', + 'location': 'Santa Barbara, CA', + 'other': [ + 'Graduated with a Major in Physics and a Minor in ' + 'Mathematics']}], + 'email': 'scopatz@cec.sc.edu', + 'employment': [ + {'begin_year': 2015, + 'location': 'Columbia, SC', + 'organization': 'The University of South Carolina', + 'other': [ + 'Cyclus: An agent-based, discrete time nuclear fuel ' + 'cycle simulator.', + 'PyNE: The Nuclear Engineering Toolkit.', + 'Website: http://www.ergs.sc.edu/'], + 'position': 'Assistant Professor, Mechanical Engineering ' + 'Department'}, + {'begin_year': 2013, + 'end_year': 2015, + 'location': 'Madison, WI', + 'organization': 'CNERG, The University of ' + 'Wisconsin-Madison', + 'other': [ + 'Cyclus: An agent-based, discrete time nuclear fuel ' + 'cycle simulator.', + 'PyNE: The Nuclear Engineering Toolkit.', + 'Website: https://cnerg.github.io/'], + 'position': 'Associate Scientist, Engineering Physics ' + 'Department'}, + {'begin_month': 'Nov', + 'begin_year': 2011, + 'end_month': 'May', + 'end_year': 2013, + 'location': 'Chicago, IL', + 'organization': 'The FLASH Center, The University of ' + 'Chicago', + 'other': [ + 'NIF: Simulation of magnetic field generation from ' + 'neutral plasmas using FLASH.', + 'CosmoB: Simulation of magnetic field generation ' + 'from neutral plasmas using FLASH.', + 'FLASH4: High-energy density physics capabilities ' + 'and utilities.', + 'Simulated Diagnostics: Schlieren, shadowgraphy, ' + 'Langmuir probes, etc. from FLASH.', + 'OpacPlot: HDF5-based equation of state and opacity ' + 'file format.', + 'Website: http://flash.uchicago.edu/site/'], + 'position': 'Research Scientist, Postdoctoral Scholar'}], + 'funding': [ + {'name': "Omega Laser User's Group Travel Award", + 'value': 1100, + 'year': 2013}, + { + 'name': "NIF User's Group Travel Award", + 'value': 1150, + 'year': 2013}], + 'membership': [{'begin_year': 2006, + 'organization': 'American Nuclear Society', + 'position': 'Member'}, + {'begin_year': 2013, + 'organization': 'Python Software Foundation', + 'position': 'Fellow'}], + 'name': 'Anthony Scopatz', + 'position': 'professor', + 'skills': [{'category': 'Programming Languages', + 'level': 'expert', + 'name': 'Python'}, + { + 'category': 'Programming Languages', + 'level': 'expert', + 'name': 'Cython'}], + 'teaching': [{ + 'course': 'EMCH 552: Intro to Nuclear Engineering', + 'description': 'This course is an introduction to nuclear ' + 'physics.', + 'month': 'August', + 'organization': 'University of South Carolina', + 'position': 'Professor', + 'syllabus': 'https://drive.google.com/open?id' + '=0BxUpd34yizZreDBCMEJNY2FUbnc', + 'year': 2017}, + { + 'course': 'EMCH 558/758: Reactor Power Systems', + 'description': 'This course covers conventional ' + 'reactors.', + 'month': 'January', + 'organization': 'University of South Carolina', + 'position': 'Professor', + 'syllabus': 'https://docs.google.com/document/d' + '/1uMAx_KFZK9ugYyF6wWtLLWgITVhaTBkAf8' + '-PxiboYdM/edit?usp=sharing', + 'year': 2017}], + 'title': 'Dr.'}, + 'projects': {'_id': 'Cyclus', + 'name': 'Cyclus', + 'description': 'Agent-Based Nuclear Fuel Cycle Simulator', + 'logo': 'http://fuelcycle.org/_static/big_c.png', + 'other': [ + 'Discrete facilities with discrete material transactions', + 'Low barrier to entry, rapid payback to adoption'], + 'repo': 'https://github.com/cyclus/cyclus/', + 'team': [{'begin_month': 'June', + 'begin_year': 2013, + 'end_month': 'July', + 'end_year': 2015, + 'name': 'Anthony Scopatz', + 'position': 'Project Lead'}], + 'website': 'http://fuelcycle.org/'}, + 'proposals': {'_id': 'mypropsal', + 'ammount': 1000000.0, + 'authors': ['Anthony Scopatz', + 'Robert Flanagan'], + 'currency': 'USD', + 'day': 18, + 'durration': 3, + 'full': { + 'benefit_of_collaboration': 'http://pdf.com' + '/benefit_of_collaboration', + 'cv': [ + 'http://pdf.com/scopatz-cv', + 'http://pdf.com/flanagan-cv'], + 'narrative': 'http://some.com/pdf'}, + 'month': 'Aug', + 'pi': 'Anthony Scopatz', + 'pre': { + 'benefit_of_collaboration': 'http://pdf.com' + '/benefit_of_collaboration', + 'cv': [ + 'http://pdf.com/scopatz-cv', + 'http://pdf.com/flanagan-cv'], + 'day': 2, + 'month': 'Aug', + 'narrative': 'http://some.com/pdf', + 'year': 1998}, + 'status': 'submitted', + 'title': 'A very fine proposal indeed', + 'year': 1999}, + 'students': {'_id': 'Human A. Person', + 'aka': ['H. A. Person'], + 'email': 'haperson@uni.edu', + 'university_id': 'HAP42'}} + +SCHEMAS = { + 'abstracts': { + '_description': { + 'description': 'Abstracts for a conference or workshop. This is ' + 'generally public information'}, + '_id': { + 'description': 'Unique identifier for submission. This generally ' + 'includes the author name and part of the title.', + 'required': True, + 'type': 'string'}, + 'coauthors': {'description': 'names of coauthors', + 'required': False, + 'type': 'string'}, + 'email': {'description': 'contact email for the author.', + 'required': True, + 'type': 'string'}, + 'firstname': {'description': 'first name of the author.', + 'required': True, + 'type': 'string'}, + 'institution': {'description': 'name of the inistitution', + 'required': True, + 'type': 'string'}, + 'lastname': {'description': 'last name of the author.', + 'required': True, + 'type': 'string'}, + 'references': { + 'description': 'HTML string of reference for the abstract itself', + 'required': False, + 'type': 'string'}, + 'text': {'description': 'HTML string of the abstract.', + 'required': True, + 'type': 'string'}, + 'timestamp': { + 'description': 'The time when the abstract was submitted.', + 'required': True, + 'type': 'string'}, + 'title': {'description': 'title of the presentation/paper.', + 'required': True, + 'type': 'string'}}, + 'assignments': { + '_description': { + 'description': 'Information about assignments for classes.'}, + '_id': { + 'description': 'A unique id for the assignment, such as ' + 'HW01-EMCH-558-2016-S', + 'required': True, + 'type': 'string'}, + 'category': {'description': "such as 'homework' or 'final'", + 'required': True, + 'type': 'string'}, + 'courses': { + 'description': 'ids of the courses that have this assignment', + 'required': True, + 'type': 'string'}, + 'file': {'description': 'path to assignment file in store', + 'required': False, + 'type': 'string'}, + 'points': { + 'description': 'list of number of points possible for each ' + 'question. Length is the number of questions', + 'required': True, + 'type': ('integer', 'float')}, + 'question': { + 'description': 'titles for the questions on this assignment', + 'required': False, + 'type': 'string'}, + 'solution': {'description': 'path to solution file in store', + 'required': False, + 'type': 'string'}}, + 'blog': { + '_description': { + 'description': 'This collection represents blog posts written by ' + 'the members of the research group.'}, + '_id': { + 'description': 'short representation, such as this-is-my-title', + 'required': True, + 'type': 'string'}, + 'author': {'description': 'name or AKA of author', + 'required': True, + 'type': 'string'}, + 'day': {'description': 'Publication day', + 'required': True, + 'type': 'integer'}, + 'month': {'description': 'Publication month', + 'required': True, + 'type': 'string'}, + 'original': { + 'description': 'URL of original post, if this is a repost', + 'required': False, + 'type': 'string'}, + 'post': {'description': 'actual contents of the post', + 'required': True, + 'type': 'string'}, + 'title': {'description': 'full human readable title', + 'required': True, + 'type': 'string'}, + 'year': {'description': 'Publication year', + 'required': True, + 'type': 'integer'}}, + 'grades': { + '_description': { + 'description': 'The grade for a student on an assignment. This ' + 'information should be private.'}, + '_id': { + 'description': 'unique id, typically the ' + 'student-assignment-course', + 'required': True, + 'type': 'string'}, + 'assignment': {'description': 'assignment id', + 'required': True, + 'type': 'string'}, + 'course': { + 'description': 'course id', + 'required': True, + 'type': 'string'}, + 'filename': {'description': 'path to file in store', + 'required': False, + 'type': 'string'}, + 'scores': { + 'description': 'the number of points earned on each question', + 'required': True, + 'type': ('integer', 'float')}, + 'student': {'description': 'student id', + 'required': True, + 'type': 'string'}}, + 'grants': { + '_description': { + 'description': 'This collection represents grants that have been ' + 'awarded to the group.'}, + '_id': {'description': 'short representation, such as this-is-my-name', + 'required': True, + 'type': ('string', 'integer', 'float')}, + 'amount': {'description': 'value of award', + 'required': True, + 'type': ('integer', 'float')}, + 'begin_day': {'description': 'start day of the grant', + 'required': False, + 'type': 'integer'}, + 'begin_month': {'description': 'start month of the grant', + 'required': True, + 'type': 'string'}, + 'begin_year': {'description': 'start year of the grant', + 'required': True, + 'type': 'integer'}, + 'benefit_of_collaboration': {'description': '', + 'required': False, + 'type': 'string'}, + 'call_for_proposals': {'description': '', + 'required': False, + 'type': 'string'}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': False, + 'type': 'string'}, + 'end_day': {'description': 'end day of the grant', + 'required': False, + 'type': ('string', 'integer')}, + 'end_month"': {'description': 'end month of the grant', + 'required': False, + 'type': 'string'}, + 'end_year': {'description': 'end year of the grant', + 'required': True, + 'type': 'integer'}, + 'funder': {'description': 'the agency funding the work', + 'required': True, + 'type': 'string'}, + 'grant_id': {'description': 'the identifier for this work', + 'required': False, + 'type': 'string'}, + 'narrative': {'description': '', 'required': False, 'type': 'string'}, + 'program': {'description': 'the program the work was funded under', + 'required': True, + 'type': 'string'}, + 'status': {'allowed': ['pending', 'declined', 'accepted', 'in-prep'], + 'description': 'status of the grant', + 'required': True, + 'type': 'string'}, + 'team': { + 'description': 'information about the team members participating ' + 'in the grant.', + 'required': True, + 'schema': {'schema': {'cv': {'required': False, 'type': 'string'}, + 'institution': { + 'required': True, + 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'position': { + 'required': True, + 'type': 'string'}, + 'subaward_amount': { + 'required': False, + 'type': ('integer', 'float')}}, + 'type': 'dict'}, + 'type': 'list'}, + 'title': {'description': 'actual title of proposal / grant', + 'required': True, + 'type': 'string'}}, + 'people': { + '_description': { + 'description': 'This collection describes the members of the ' + 'research group. This is normally public data.'}, + '_id': {'description': 'unique identifier for the group member', + 'required': True, + 'type': 'string'}, + 'active': { + 'description': 'If the person is an active member, default True.', + 'required': False, + 'type': 'boolean'}, + 'aka': { + 'description': 'list of aliases (also-known-as), useful for ' + 'identifying the group member in citations or ' + 'elsewhere.', + 'required': True, + 'type': ['string', 'list']}, + 'avatar': {'description': 'URL to avatar', + 'required': True, + 'type': 'string'}, + 'bio': {'description': 'short biographical text', + 'required': True, + 'type': 'string'}, + 'collab': { + 'description': 'If the person is a collaborator, default False.', + 'required': False, + 'type': 'boolean'}, + 'education': { + 'description': 'This contains the educational information for ' + 'the group member.', + 'required': True, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'degree': {'required': True, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': True, 'type': 'integer'}, + 'gpa': {'required': False, 'type': ('float', 'string')}, + 'institution': {'required': True, 'type': 'string'}, + 'location': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}}, + 'type': 'list'}, + 'email': {'description': 'email address of the group member', + 'required': False, + 'type': 'string'}, + 'employment': { + 'description': 'Employment information, similar to educational ' + 'information.', + 'required': True, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'location': {'required': True, 'type': 'string'}, + 'organization': {'required': True, 'type': 'string'}, + 'other': {'required': False, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}}, + 'type': 'list'}, + 'funding': { + 'description': 'Funding and scholarship that the group member ' + 'has individually obtained in the past. ' + '**WARNING:** this is not to be confused with the ' + '**grants** collection', + 'required': False, + 'schema': {'currency': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'value': { + 'required': True, + 'type': ('float', 'integer')}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'honors': { + 'description': 'Honors that have been awarded to this ' + 'group member', + 'required': False, + 'schema': {'description': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'membership': { + 'description': 'Professional organizations this member is ' + 'a part of', + 'required': False, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'description': {'required': False, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'organization': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}}, + 'type': 'list'}, + 'name': {'description': 'Full, canonical name for the person', + 'required': True, + 'type': 'string'}, + 'position': { + 'description': 'such as professor, graduate student, or scientist', + 'required': True, + 'type': 'string'}, + 'service': { + 'description': 'Service that this group member has provided', + 'required': False, + 'schema': {'description': {'required': False, 'type': 'string'}, + 'duration': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'skills': {'description': 'Skill the group member has', + 'required': False, + 'schema': {'category': {'required': True, 'type': 'string'}, + 'level': {'required': True, 'type': 'string'}, + 'name': {'required': True, 'type': 'string'}}, + 'type': 'list'}, + 'teaching': { + 'description': 'Courses that this group member has taught, if any', + 'required': False, + 'schema': {'course': {'required': True, 'type': 'string'}, + 'description': {'required': False, 'type': 'string'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'materials': {'required': False, 'type': 'string'}, + 'month': {'required': False, 'type': 'string'}, + 'organization': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}, + 'syllabus': {'required': False, 'type': 'string'}, + 'video': {'required': False, 'type': 'string'}, + 'website': {'required': False, 'type': 'string'}, + 'year': {'required': True, 'type': 'integer'}}, + 'type': 'list'}, + 'title': {'description': 'for example, Dr., etc.', + 'required': False, + 'type': 'string'}}, + 'projects': { + '_description': { + 'description': 'This collection describes the research group ' + 'projects. This is normally public data.'}, + '_id': {'description': 'Unique project identifier.', + 'required': True, + 'type': 'string'}, + 'description': {'description': 'brief project description.', + 'required': True, + 'type': 'string'}, + 'grant': { + 'description': 'Grant id if there is a grant supporting this ' + 'project', + 'type': 'string'}, + 'logo': {'description': 'URL to the project logo', + 'required': False, + 'type': 'string'}, + 'name': {'description': 'name of the project.', + 'required': True, + 'type': 'string'}, + 'other': {'description': 'other information about the project', + 'required': False, + 'type': ['list', 'string']}, + 'repo': {'description': 'URL of the source code repo, if available', + 'required': False, + 'type': 'string'}, + 'team': { + 'description': 'People who are/have been working on this project.', + 'required': True, + 'schema': {'begin_month': {'required': False, 'type': 'string'}, + 'begin_year': {'required': True, 'type': 'integer'}, + 'end_month': {'required': False, 'type': 'string'}, + 'end_year': {'required': False, 'type': 'integer'}, + 'name': {'required': True, 'type': 'string'}, + 'position': {'required': True, 'type': 'string'}}, + 'type': 'list'}, + 'website': {'description': 'URL of the website.', + 'required': True, + 'type': 'string'}}, + 'proposals': { + '_description': { + 'description': 'This collection represents proposals that have ' + 'been submitted by the group.'}, + '_id': {'description': 'short representation, such as this-is-my-name', + 'required': True, + 'type': ('string', 'integer', 'float')}, + 'amount': {'description': 'value of award', + 'required': True, + 'type': ('integer', 'float')}, + 'authors': {'description': 'other investigator names', + 'required': True, + 'type': 'string'}, + 'currency': {'description': "typically '$' or 'USD'", + 'required': True, + 'type': 'string'}, + 'day': {'description': 'day that the proposal is due', + 'required': True, + 'type': 'integer'}, + 'duration': {'description': 'number of years', + 'required': True, + 'type': ('integer', 'float')}, + 'month': {'description': 'month that the proposal is due', + 'required': True, + 'type': 'string'}, + 'pi': {'description': 'principal investigator name', + 'required': True, + 'type': 'string'}, + 'pre': {'description': 'Information about the pre-proposal', + 'required': False, + 'type': 'dict'}, + 'status': {'description': "e.g. 'submitted', 'accepted', 'rejected'", + 'required': True, + 'type': 'string'}, + 'title': {'description': 'actual title of proposal', + 'required': True, + 'type': 'string'}, + 'year': {'description': 'Year that the proposal is due', + 'required': True, + 'type': 'integer'}}, + 'students': { + '_description': { + 'description': 'This is a collection of student names and ' + 'metadata. This should probably be private.'}, + '_id': {'description': 'short representation, such as this-is-my-name', + 'required': True, + 'type': 'string'}, + 'aka': {'description': 'list of aliases', + 'required': False, + 'schema': {'type': 'string'}, + 'type': ('list', 'string')}, + 'email': {'description': 'email address', + 'required': False, + 'type': 'string'}, + 'university_id': { + 'description': 'The university identifier for the student', + 'required': False, + 'type': 'string'}}} + + +class NoDescriptionValidator(Validator): + def _validate_description(self, description, field, value): + if False: + pass + + +def validate(db, record): + """Validate a record for a given db + + Parameters + ---------- + db : str + The name of the db in question + record : dict + The record to be validated + + Returns + ------- + rtn : bool + True is valid + errors: dict + The errors encountered (if any) + + """ + schema = SCHEMAS[db] + v = NoDescriptionValidator(schema) + return v.validate(record), v.errors diff --git a/regolith/validators.py b/regolith/validators.py index 64030873d..fe527d890 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -1,10 +1,10 @@ -"""Validators and convertors for regolith input.""" +"""Validators and converters for regolith input.""" import os -import re from getpass import getpass from regolith.tools import string_types + def noop(x): """Does nothing, just returns the input.""" return x @@ -29,6 +29,7 @@ def is_bool(x): """Tests if something is a boolean""" return isinstance(x, bool) + def is_string(x): """Tests if something is a string""" return isinstance(x, string_types) @@ -54,6 +55,7 @@ def ensure_string(x): else: return str(x) + def ensure_database(db): db['name'] = ensure_string(db['name']) db['url'] = ensure_string(db['url']) @@ -109,4 +111,4 @@ def ensure_email(email): 'databases': (always_false, ensure_databases), 'stores': (always_false, ensure_stores), 'email': (always_false, ensure_email), - } +} diff --git a/requirements/run.txt b/requirements/run.txt index 4c0f633d2..3fce50576 100644 --- a/requirements/run.txt +++ b/requirements/run.txt @@ -1 +1,4 @@ -ruamel.yaml \ No newline at end of file +python +ruamel.yaml +jinja2 +cerberus \ No newline at end of file diff --git a/requirements/test.txt b/requirements/test.txt index 4c0f633d2..24137e07a 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1 +1,2 @@ -ruamel.yaml \ No newline at end of file +ruamel.yaml +pytest \ No newline at end of file diff --git a/tests/test_validators.py b/tests/test_validators.py new file mode 100644 index 000000000..85e8c4590 --- /dev/null +++ b/tests/test_validators.py @@ -0,0 +1,8 @@ +import pytest + +from regolith.schemas import SCHEMAS, validate, EXEMPLARS + + +@pytest.mark.parametrize('key', SCHEMAS.keys()) +def test_validation(key): + validate(key, EXEMPLARS[key])