Update build_erddap_catalog.py (#290)

* Update build_erddap_catalog.py * Use ioos-qc library in glider_qc.py * Update requirements.txt added an extension in the requirement file to pull in the ioos-qc library package * Update qc_config.yml * Update glider_qc.py * Update build_erddap_catalog.py Fixed the flag name attribute to match ioos-qc definitions Fixed the manual link Added the min and max values for a flag * Update glider_qc.py Modified the script to create qartod variables and add the qartod test results to each qartod variable. * Update build_erddap_catalog.py - Changed the required variables to a list of qartod variables. - Made changes to few functions to work accordingly. * Update glider_qc.py fixed spelling detected by the code checks * Update build_erddap_catalog.py rectified the omitted quotes in line 611 * Update glider_qc.py added 'nc_path' to the run_qc function * Update glider_qc.py commented out an old version of the get_unmasked function * Update glider_qc.py fixed syntax errors added qartod_status attribute * Update glider_qc.py fixed log.info() and log.exception() syntax errors * Update glider_qc.py it did not like the addition log.exception() added to the "except OSError:" when checking os.getxattr() in the previous commit. * Update glider_qc.py extended qartod variables' attributes to include more verbose explanation * Update glider_qc.py added more to log.info for debugging purposes * Update requirements.txt commented out the ioos_qartod git repo that has been deprecated * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
ioos · Jan 5, 2024 · 5b756aa · 5b756aa
1 parent d69e4b4
commit 5b756aa
Show file tree

Hide file tree

Showing 4 changed files with 535 additions and 301 deletions.
diff --git a/data/qc_config.yml b/data/qc_config.yml
@@ -1,67 +1,131 @@
-sea_water_temperature: # deg_C
-  flat_line:
-    low_reps: 4
-    high_reps: 8
-    eps: 1.1920929e-07
-  gross_range:
-    sensor_span:
-      - -5.
-      - 45.
-  spike: {}
-  rate_of_change: {}
-sea_water_electrical_conductivity: # S m-1
-  flat_line:
-    low_reps: 4
-    high_reps: 8
-    eps: 1.1920929e-07
-  gross_range:
-    sensor_span:
-      - 0
-      - 7
-  spike: {}
-  rate_of_change: {}
-sea_water_practical_salinity: # unitless
-  flat_line:
-    low_reps: 4
-    high_reps: 8
-    eps: 1.1920929e-07
-  gross_range:
-    sensor_span:
-      - 0
-      - 50
-  spike: {}
-  rate_of_change: {}
-sea_water_salinity: # unitless
-  flat_line:
-    low_reps: 4
-    high_reps: 8
-    eps: 1.1920929e-07
-  gross_range:
-    sensor_span:
-      - 0
-      - 50
-  spike: {}
-  rate_of_change: {}
-sea_water_density: # kg m-3
-  flat_line:
-    low_reps: 4
-    high_reps: 8
-    eps: 1.1920929e-07
-  gross_range:
-    sensor_span:
-      - 900
-      - 1050
-  spike: {}
-  rate_of_change: {}
-sea_water_pressure: # `
-  flat_line:
-    low_reps: 4
-    high_reps: 8
-    eps: 1.1920929e-07
-  gross_range:
-    sensor_span:
-      - 0
-      - 11000
-  pressure: {}
-  rate_of_change: {}
-  spike: {}
+contexts:
+   - streams:
+       conductivity:
+         qartod:
+             gross_range_test:
+               suspect_span: [0, 6]
+               fail_span: [0, 9]
+             spike_test:
+               suspect_threshold:
+               fail_threshold:
+             rate_of_change_test:
+               threshold: 0.1
+       temperature:
+         qartod:
+             gross_range_test:
+               suspect_span: [0, 35]
+               fail_span: [-2, 40]
+             spike_test:
+               suspect_threshold:
+               fail_threshold:
+             rate_of_change_test:
+               threshold: 0.1
+#             flat_line_test:
+#               tolerance: 1
+#               suspect_threshold: 3600
+#               fail_threshold: 9000
+       pressure:
+         qartod:
+             gross_range_test:
+               suspect_span: [0, 1000]
+               fail_span: [0, 6000]
+             spike_test:
+               suspect_threshold:
+               fail_threshold:
+             rate_of_change_test:
+               threshold: 0.1
+#             flat_line_test:
+#               tolerance: 1
+#               suspect_threshold: 3600
+#               fail_threshold: 9000
+       salinity:
+         qartod:
+             gross_range_test:
+               fail_span: [0, 42]
+               suspect_span: [10, 38]
+             spike_test:
+               suspect_threshold:
+               fail_threshold:
+             rate_of_change_test:
+               threshold: 0.1
+       density:
+         qartod:
+             gross_range_test:
+               fail_span: [1000, 1100]
+             spike_test:
+               suspect_threshold:
+               fail_threshold:
+             rate_of_change_test:
+               threshold: 0.1
+#             flat_line_test:
+#               tolerance: 1
+#               suspect_threshold: 3600
+#               fail_threshold: 9000
+
+# sea_water_temperature: # deg_C
+#   flat_line:
+#     low_reps: 4
+#     high_reps: 8
+#     eps: 1.1920929e-07
+#   gross_range:
+#     sensor_span:
+#       - -5.
+#       - 45.
+#   spike: {}
+#   rate_of_change: {}
+# sea_water_electrical_conductivity: # S m-1
+#   flat_line:
+#     low_reps: 4
+#     high_reps: 8
+#     eps: 1.1920929e-07
+#   gross_range:
+#     sensor_span:
+#       - 0
+#       - 7
+#   spike: {}
+#   rate_of_change: {}
+# sea_water_practical_salinity: # unitless
+#   flat_line:
+#     low_reps: 4
+#     high_reps: 8
+#     eps: 1.1920929e-07
+#   gross_range:
+#     sensor_span:
+#       - 0
+#       - 50
+#   spike: {}
+#   rate_of_change: {}
+# sea_water_salinity: # unitless
+#   flat_line:
+#     low_reps: 4
+#     high_reps: 8
+#     eps: 1.1920929e-07
+#   gross_range:
+#     sensor_span:
+#       - 0
+#       - 50
+#   spike: {}
+#   rate_of_change: {}
+# sea_water_density: # kg m-3
+#   flat_line:
+#     low_reps: 4
+#     high_reps: 8
+#     eps: 1.1920929e-07
+#   gross_range:
+#     sensor_span:
+#       - 900
+#       - 1050
+#   spike: {}
+#   rate_of_change: {}
+# sea_water_pressure: # `
+#   flat_line:
+#     low_reps: 4
+#     high_reps: 8
+#     eps: 1.1920929e-07
+#   gross_range:
+#     sensor_span:
+#       - 0
+#       - 11000
+#   pressure: {}
+#   rate_of_change: {}
+#   spike: {}