Merge branch 'develop' into feature/mtg2

sebvi · Feb 9, 2024 · 26d9a36 · 26d9a36
2 parents 202829b + 2849a10
commit 26d9a36
Show file tree

Hide file tree

Showing 58 changed files with 15,935 additions and 5,813 deletions.
diff --git a/.github/ci-config.yml b/.github/ci-config.yml
@@ -0,0 +1,7 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@master
+  ecmwf/eccodes
+  ecmwf/eckit
+dependency_branch: develop
+parallelism_factor: 8
diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml
@@ -0,0 +1,8 @@
+build:
+  modules:
+    - ninja
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/eccodes@develop
+    - ecmwf/eckit@develop
+  parallel: 64
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -0,0 +1,11 @@
+name: cd
+
+on:
+  push:
+    tags:
+      - '**'
+
+jobs:
+  deploy:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/create-package.yml@v2
+    secrets: inherit
diff --git a/.github/workflows/check-release-version.yml b/.github/workflows/check-release-version.yml
@@ -0,0 +1,11 @@
+name: Check VERSION file
+
+on:
+  push:
+    branches:
+      - "release/**"
+      - "hotfix/**"
+
+jobs:
+  check_version:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/check-release-version.yml@v2
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -27,6 +27,7 @@ jobs:
     uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
     with:
       metkit: ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
     secrets: inherit
 
   # Run CI of private downstream packages on self-hosted runners
@@ -73,12 +74,3 @@ jobs:
           repository: private-downstream-ci
           event_type: downstream-ci-hpc
           payload: '{"metkit": "ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}"}'
-
-  codecov:
-    name: code-coverage
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ./.github/workflows/reusable-ci.yml
-    with:
-      metkit: ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}
-      codecov: true
-    secrets: inherit
diff --git a/.github/workflows/reusable-ci-hpc.yml b/.github/workflows/reusable-ci-hpc.yml
diff --git a/.github/workflows/reusable-ci.yml b/.github/workflows/reusable-ci.yml
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,3 @@
-.vscode
 .atom-build.yaml
 .tags*
 CMakeLists.txt.user*

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,138 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        },
+        {
+            "name": "make params MARS yaml (deprecated method)",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/git/metkit/share/metkit/make-params-yaml.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "ECFLOW_DIR": "/usr/local/apps/ecflow/5.8.1" // "${env:ECFLOW_DIR}"
+            },
+            "args": [""],
+            "cwd":"${env:HOME}/git/metkit/share/metkit/"
+        },
+        {
+            "name": "make paramIds yaml - 0001",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/git/metkit/share/metkit/make-paramids-yaml.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "ECFLOW_DIR": "/usr/local/apps/ecflow/5.8.1" // "${env:ECFLOW_DIR}"
+            },
+            "args": [""],
+            "cwd":"${env:HOME}/git/metkit/share/metkit/"
+        },
+        {
+            "name": "make paramIds yaml - 0078",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/git/metkit/share/metkit/make-paramids-yaml-esuite.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "ECFLOW_DIR": "/usr/local/apps/ecflow/5.8.1" // "${env:ECFLOW_DIR}"
+            },
+            "args": [""],
+            "cwd":"${env:HOME}/git/metkit/share/metkit/"
+        },
+        {
+            "name": "make params FDB yaml - 0001",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/git/metkit/share/metkit/make-params-yaml-from-fields-catalogue.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "ECFLOW_DIR": "/usr/local/apps/ecflow/5.8.1", // "${env:ECFLOW_DIR}"
+                "PRODUCT_CATALOGUE_HOST": "bol-db-products-prod-01",
+                "PRODUCT_CATALOGUE_DB": "products",
+                "PRODUCT_CATALOGUE_USER":     "ecmwf_ro",
+                "PRODUCT_CATALOGUE_PASSWORD": "ecmwf_ro",
+                "PRODUCT_CATALOGUE_PORT": "5432",
+                "VALIDATION_SERVICE_HOST":    "https://products.ecmwf.int/requirements/"
+            },
+            "args": [""],
+            "cwd":"${env:HOME}/git/metkit/share/metkit/"
+        },
+        {            
+            "name": "make params FDB yaml - 0078 - preprod",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/git/metkit/share/metkit/make-params-yaml-from-fields-catalogue.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "ECFLOW_DIR": "/usr/local/apps/ecflow/5.8.1", // "${env:ECFLOW_DIR}"
+                "PRODUCT_CATALOGUE_HOST": "k8s-bol-webapps-test-worker-016",
+                "PRODUCT_CATALOGUE_DB": "products",
+                "PRODUCT_CATALOGUE_USER": "products",
+                "PRODUCT_CATALOGUE_PASSWORD": "products",
+                "PRODUCT_CATALOGUE_PORT": "30544",
+                "VALIDATION_SERVICE_HOST": "https://products-test.ecmwf.int/esuite/requirements/"
+            },
+            "args": [""],
+            "cwd":"${env:HOME}/git/metkit/share/metkit/"
+        },
+        {            
+            "name": "make params FDB yaml - 0078 - prod",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/git/metkit/share/metkit/make-params-yaml-from-fields-catalogue.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "ECFLOW_DIR": "/usr/local/apps/ecflow/5.8.1", // "${env:ECFLOW_DIR}"
+                "PRODUCT_CATALOGUE_HOST": "k8s-bol-webapps-prod-worker-012",
+                "PRODUCT_CATALOGUE_DB": "products",
+                "PRODUCT_CATALOGUE_USER": "products",
+                "PRODUCT_CATALOGUE_PASSWORD": "products",
+                "PRODUCT_CATALOGUE_PORT": "30545",
+                "VALIDATION_SERVICE_HOST": "https://products.ecmwf.int/esuite/requirements/"
+            },
+            "args": [""],
+            "cwd":"${env:HOME}/git/metkit/share/metkit/"
+        },     
+        {
+            "name": "make fields DB yaml - 0001",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/git/metkit/share/metkit/fieldsdb.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "ECFLOW_DIR": "/usr/local/apps/ecflow/5.8.1", // "${env:ECFLOW_DIR}"
+            },
+            "args": [""],
+            "cwd":"${env:HOME}/git/metkit/share/metkit/"
+        },
+        {
+            "name": "make fields DB yaml - 0078",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/git/metkit/share/metkit/fieldsdb-esuite.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "ECFLOW_DIR": "/usr/local/apps/ecflow/5.8.1", // "${env:ECFLOW_DIR}"
+            },
+            "args": [""],
+            "cwd":"${env:HOME}/git/metkit/share/metkit/"
+        },
+    ]
+}
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -14,7 +14,7 @@ find_package( ecbuild 3.7.2 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_C
 
 project( metkit LANGUAGES CXX )
 
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 ########################################################################################################################

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.10.6
+1.11.5
diff --git a/share/metkit/axis.yaml b/share/metkit/axis.yaml
@@ -40,3 +40,12 @@ axes:
     - direction
     - channel
     - reportype
+    - dataset
+    - activity
+    - experiment
+    - generation
+    - model
+    - realization
+    - resolution
+    - year
+    - month
diff --git a/share/metkit/fieldsdb-esuite.py b/share/metkit/fieldsdb-esuite.py
@@ -0,0 +1,39 @@
+from sqlalchemy import create_engine, text
+import yaml
+import os
+
+db = create_engine("postgresql://products:[email protected]:30544/products")
+TYPES = text("select distinct stream, type, levtype from fields")
+PARAMS = text(
+    "select distinct param from fields where stream=:stream and type=:type and levtype=:levtype"
+)
+
+P = {}
+
+for stream, type, levtype in db.execute(TYPES):
+    kind = dict(stream=stream, type=type, levtype=levtype)
+    key = (stream, type, levtype)
+    # print('-'.join([stream, type, levtype]))
+    params = list(db.execute(PARAMS, kind))
+
+    try:
+        P[key] = tuple(sorted([int(x[0]) for x in params]))
+    except:
+        pass
+        # print("  %s" % params,)
+        # P[key] = (129, 999)
+
+
+Y = []
+for k, v in sorted(P.items()):
+    if k[2]:
+        Y.append([dict(stream=k[0], type=k[1], levtype=k[2]), v])
+    else:
+        Y.append([dict(stream=k[0], type=k[1]), v])
+
+with open("fieldsdb.yaml", "w") as f:
+    f.write(
+        "# File automatically generated by %s\n# Do not edit\n\n"
+        % (os.path.basename(__file__))
+    )
+    f.write(yaml.safe_dump(Y, default_flow_style=False))
diff --git a/share/metkit/fieldsdb.py b/share/metkit/fieldsdb.py
@@ -2,7 +2,8 @@
 import yaml
 import os
 
-db = create_engine("postgres://ecmwf_ro:[email protected]/products")
+db = create_engine("postgresql://ecmwf_ro:[email protected]:5432/products")
+
 TYPES = text("select distinct stream, type, levtype from fields")
 PARAMS = text(
     "select distinct param from fields where stream=:stream and type=:type and levtype=:levtype"