dtcenter · bikegeek · Nov 12, 2024 · Oct 21, 2024 · Oct 21, 2024 · Oct 21, 2024
diff --git a/METdbLoad/conftest.py b/METdbLoad/conftest.py
@@ -134,8 +134,8 @@ def mock_logger():
 @pytest.fixture
 def get_specified_xml_loadfile( ) -> XmlLoadFile:
     """
-       Retrieve the specified XML load specification filee. This is useful for using different XML
-       specification file for  validating against recursive payloads, large payloads, etc.
+       Retrieve the specified XML load specification file. This is useful for using different XML
+       specification file for validating against recursive payloads, large payloads, etc.
 
        Args:
            xml_filename: The name of the XML file of interest

diff --git a/METdbLoad/docs/Users_Guide/load.rst b/METdbLoad/docs/Users_Guide/load.rst
@@ -79,18 +79,18 @@ increase load time.
 **<drop_indexes>:** **TRUE** or **FALSE**, this option indicates whether
 database indexes should be dropped prior to loading new data.
 
-**<load_indexes>:** **TRUE** or **FALSE**, this option indicates whether
+**<apply_indexes>:** **TRUE** or **FALSE**, this option indicates whether
 database indexes should be created after loading new data.
 
 **<group>:** The name of the group for the user interface.
 
 **<description>:** A short description of the database.
 
-| **<load_files>:** A list structure containing individual MET output files to load into the database.
-|
-| **</load_files>:** Follows the list of files after the previous tag, to end the list.
+|**<load_files>:** A list structure containing individual MET output files to load into the database.
+|   **<file>:** Contains a single MET output file to load.
+|**</load_files>:** Follows the list of files after the previous tag, to end the list.
 
-| **<file>:** Contains a single MET output file to load.
+|
 
 **<folder_tmpl>:** A template string describing the file structure of
 the input MET files, which is populated with values specified in

diff --git a/METdbLoad/test/Examples/example_load_specific_files.xml b/METdbLoad/test/Examples/example_load_specific_files.xml
@@ -0,0 +1,29 @@
+<load_spec>
+  <connection>
+    <host>yourhost:1234</host>
+    <database>your_database_name</database>
+    <user>your_db_username</user>
+    <password>your_password</password>
+  </connection>
+
+  <verbose>true</verbose>
+  <insert_size>1</insert_size>
+  <stat_header_db_check>False</stat_header_db_check>
+  <mode_header_db_check>false</mode_header_db_check>
+  <drop_indexes>FALSE</drop_indexes>
+  <apply_indexes>true</apply_indexes>
+  <load_stat>True</load_stat>
+  <load_mode>false</load_mode>
+  <load_mpr>false</load_mpr>
+  <group>Regional Ensemble</group>
+
+  <load_files>
+    <file>/met_out/mode/mode_MASK_POLY_300000L_20120410_180000V_060000A_cts.txt</file>
+    <file>/met_out/rhist/ensemble_stat_RRFS_GEFS_GF.SPP.SPPT_RETOP_MRMS_20220507_120000V.stat</file>
+  </load_files>
+
+
+  <load_xml>true</load_xml>
+  <load_note>Load MODE and RHIST for load_files.</load_note>
+
+</load_spec>
diff --git a/METdbLoad/test/Examples/example_load_specification.xml b/METdbLoad/test/Examples/example_load_specification.xml
@@ -0,0 +1,73 @@
+<load_spec>
+  <connection>
+    <host>yourhost:1234</host>
+    <database>your_database_name</database>
+    <user>your_db_username</user>
+    <password>your_password</password>
+  </connection>
+
+  <date_list name="folder_dates">
+    <start>2022050100</start>
+    <end>2022051200</end>
+    <inc>86400</inc>
+    <format>yyyyMMddHH</format>
+  </date_list>
+   <date_list name="valid_dates">
+    <start>2022050100</start>
+    <end>2022051200</end>
+    <inc>0600</inc>
+    <format>yyyyMMddHH</format>
+  </date_list>
+
+  <verbose>false</verbose>
+  <insert_size>1</insert_size>
+  <stat_header_db_check>False</stat_header_db_check>
+  <mode_header_db_check>false</mode_header_db_check>
+  <drop_indexes>FALSE</drop_indexes>
+  <apply_indexes>true</apply_indexes>
+  <load_stat>True</load_stat>
+  <load_mode>false</load_mode>
+  <load_mpr>false</load_mpr>
+  <group>Regional Ensemble</group>
+
+  <folder_tmpl>/var/autofs/mnt/hostmachine/projects/RRFS/prototype/met_out/{config}/{fcst_init}/{mem}/{valid_times}/metprd/{met_out}/</folder_tmpl>
+
+  <load_val>
+    <field name="config">
+      <val>HREF_lag_offset</val>
+      <val>RTPS</val>
+    </field>
+    <field name="fcst_init">
+      <date_list name="folder_dates"/>
+    </field>
+       <field name="mem">
+      <val>mem01</val>
+      <val>mem02</val>
+      <val>mem03</val>
+      <val>mem04</val>
+      <val>mem05</val>
+      <val>mem06</val>
+      <val>mem07</val>
+      <val>mem08</val>
+      <val>mem09</val>
+      <val>mem10</val>
+    </field>
+    <field name="valid_times">
+      <date_list name="valid_dates"/>
+    </field>
+    <field name="met_out">
+      <val>grid_stat_cmn</val>
+      <val>point_stat_cmn</val>
+    </field>
+    <line_type>
+      <val>CNT</val>
+      <val>ORANK</val>
+      <val>SEEPS_MPR</val>
+      <val>ME</val>
+    </line_type>
+  </load_val>
+
+  <load_xml>true</load_xml>
+  <load_note>Load HREF and RTPS data for Spring 2022.</load_note>
+
+</load_spec>
diff --git a/METdbLoad/test/example_load_specific_files.xml b/METdbLoad/test/example_load_specific_files.xml
@@ -0,0 +1,29 @@
+<load_spec>
+  <connection>
+    <host>yourhost:1234</host>
+    <database>your_database_name</database>
+    <user>your_db_username</user>
+    <password>your_password</password>
+  </connection>
+
+  <verbose>true</verbose>
+  <insert_size>1</insert_size>
+  <stat_header_db_check>False</stat_header_db_check>
+  <mode_header_db_check>false</mode_header_db_check>
+  <drop_indexes>FALSE</drop_indexes>
+  <apply_indexes>true</apply_indexes>
+  <load_stat>True</load_stat>
+  <load_mode>false</load_mode>
+  <load_mpr>false</load_mpr>
+  <group>Regional Ensemble</group>
+
+  <load_files>
+    <file>/met_out/mode/mode_MASK_POLY_300000L_20120410_180000V_060000A_cts.txt</file>
+    <file>/met_out/rhist/ensemble_stat_RRFS_GEFS_GF.SPP.SPPT_RETOP_MRMS_20220507_120000V.stat</file>
+  </load_files>
+
+
+  <load_xml>true</load_xml>
+  <load_note>Load MODE and RHIST for load_files.</load_note>
+
+</load_spec>
diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py
@@ -133,11 +133,11 @@ def test_tmp_xml(get_specified_xml_loadfile):
                f"schema {utils.LOAD_SPECIFICATION_SCHEMA}")
         pytest.fail(msg)
 
-@pytest.mark.parametrize("xml_config", ["load_met_gha_prod.xml", "load_met_gha_new.xml"])
+@pytest.mark.parametrize("xml_config", ["load_met_gha_prod.xml", "load_met_gha_new.xml", "example_load_specific_files.xml"])
 def test_db_xml(get_specified_xml_loadfile, xml_config):
     """
-       Test validation against an XML specification file that is used in another test that compares two
-       databases.
+       Test validation against XML specification files used in another test that compares two
+       databases and an XML specification file that explicitly lists specific files to load.
        ValueError should be NOT be raised for
        the load_met_gha_prod/new specification files which are used on real data.
     """

diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd
@@ -136,9 +136,18 @@
 
                 <xs:element type="limitedstringtype" name="group" minOccurs="0"/>
                 <xs:element type="limitedstringtype" name="description" minOccurs="0"/>
-                <xs:element type="foldertmpl" name="folder_tmpl"/>
 
-                <xs:element name="load_val" minOccurs="1">
+                <xs:element name="load_files" minOccurs="0">
+                   <xs:complexType>
+                       <xs:sequence>
+                          <xs:element type="xs:string" name="file" minOccurs="1" maxOccurs="200"/>
+                       </xs:sequence>
+                   </xs:complexType>
+                </xs:element>
+
+                <xs:element type="foldertmpl" name="folder_tmpl" minOccurs="0"/>
+
+                <xs:element name="load_val" minOccurs="0">
                     <xs:complexType>
                         <xs:sequence>
                             <xs:element name="field" minOccurs="0" maxOccurs="20">

diff --git a/METreformat/test/eclv_mpr_pct_vl1l2_vcnt.xml b/METreformat/test/eclv_mpr_pct_vl1l2_vcnt.xml
diff --git a/METreformat/test/ens_stat_ecnt.xml b/METreformat/test/ens_stat_ecnt.xml
diff --git a/METreformat/test/mctc_mcts.xml b/METreformat/test/mctc_mcts.xml
diff --git a/METreformat/test/point_stat.xml b/METreformat/test/point_stat.xml
diff --git a/docs/Users_Guide/common.rst b/docs/Users_Guide/common.rst
diff --git a/docs/Users_Guide/index.rst b/docs/Users_Guide/index.rst
@@ -68,15 +68,15 @@ Foundation (NSF).
 
 .. toctree::
    :titlesonly:
-   :numbered: 4
+   :numbered: 6
 
    overview
    release-notes
    installation
-   common
    load_data
    read_nc
    reformat_stat_data
+
 
 
 Indices and tables

diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst
@@ -45,32 +45,13 @@ Installation
 Download
 --------
 
-Download METdataio into an installation directory.  Programs can be run from
-the ush subdirectory.
+Download the latest release:
 
 .. code-block:: ini
 
-  git clone https://github.com/dtcenter/METdataio [install]/METdataio
-  cd [install]/METdataio/METdbLoad/ush
+  git clone https://github.com/dtcenter/METdataio
 
-Create Database
----------------
+.. note::
 
-Data must be loaded into a database which has the prefix \'\mv_\'\,
-e.g. mv_met_data. This database must be structured with the METviewer
-mv_mysql.sql schema:
+  Programs can be run from the METdbLoad/ush subdirectory.
 
-.. code-block:: ini
-
-  cd [install]/metviewer
-  mysql -u[db_username] -p[db_password] -e'create database [db_name];'
-  mysql -u[db_username] -p[db_password] [db_name] < sql/mv_mysql.sql
-
-
-Create an XML load specification document which contains information about your
-MET data. Run METdbload using the XML load specification as input (called
-[load_xml] here) and monitor progress:
-
-.. code-block:: ini
-
-  python met_db_load.py [load_xml] [optional redirection of output, e.g. &> log/load_[date].log &]'