Update automapper capability and add testing (#956)

.gencode_hash.txt

@@ -7,14 +7,14 @@ fb876b8f7bfdccf156d751efddf2660a7a7ab5585e89be0f82c59f7a61e68d4c  gencode/docs/c
 6a097f1f87ab7b9a82e4d3aa6f6cedf69499a95742863a7f53bb5fdf53fd6ff0  gencode/docs/configuration_pod.html
 4d327975ae4d48a5dd920f298931fc5056a7239d677e61a9f4d129068316dd0a  gencode/docs/configuration_pubber.html
 f24d15e549f1b143b50d6011f2f7fd03286e1cd128a3e25591195b25f8efc472  gencode/docs/data_template.html
-012fa1275063d526db96ee4e02f8a3fd7fdf4aa5d5b9bcc7267cd22b3adc935d  gencode/docs/events.html
-59cc4bfb2f0523d87c29cc8a475049432a9ac374728d60486897cbea6ca71701  gencode/docs/events_discovery.html
+45d39c61ce39afc0d14f3dc99a93947587d63999cd373cb44b63ce8f576c6e7e  gencode/docs/events.html
+45f60d2b63951448dded8c9c170913af428e50e8b4cff2143c97304332077df6  gencode/docs/events_discovery.html
 65858e981eaad27e41fd7f64aae7d6a82d4519e798145eb10898e59cead517c3  gencode/docs/events_mapping.html
 c3bae844432d172033bc416e623fecee7608efd01d916b7eaee96252932c552a  gencode/docs/events_pointset.html
 eed6ae125d94cf1986de96c210b0937b9c7b199724839f43fce3b357b057f8be  gencode/docs/events_system.html
 e59e52110ec12d7b82351abfc28c31255c57e2af637adf3fa5cbb7e3b26b49fe  gencode/docs/events_udmi.html
 e3683cb4561b6dda5703cb659fd6a8f12242725de389709a12eb45f146cbb295  gencode/docs/events_validation.html
-fd8e63cdfe293f4e8a2c667200aec77c7c4856ea1e9436c88fda00a6bef77c5b  gencode/docs/metadata.html
+3e02bab5ef46438c99f20049f30f8dd4428530b6036aa20f29a8a3964ac2b580  gencode/docs/metadata.html
 b110edfb73182782c7539adef417a970959432c92d4c9f4a0bf7e328c0e8427d  gencode/docs/monitoring.html
 c8a40993b2810dffb9119469f83592461d817c7148072203f157b89922187e7c  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
@@ -44,7 +44,7 @@ fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56  gencode/java/u
 0c133f64013d5f2c4be203708def7ae11be631e90d5992222d6b4c97eef2573e  gencode/java/udmi/schema/CapabilityValidationState.java
 ec164962f2f00924ecb41ae07c2e01d6bf027951ccc605e0210988370b987973  gencode/java/udmi/schema/Category.java
 a010037f8ad570060c6a03e5aa13bc6d5261a61bf70eb71e0d00e124036decea  gencode/java/udmi/schema/CloudConfigModel.java
-65328b8a88d357ad3205e3fa8eef6fe9148a86b955790022be3c7804cab145fb  gencode/java/udmi/schema/CloudModel.java
+eec1119dd068464fd74fbf2b12d0cd39e80b8dc88c6a1b5b08678ed5c78ff6b4  gencode/java/udmi/schema/CloudModel.java
 06c8c3131f111e49e0d3e518603a3f66349d5dee1aee64a98659cb8703a8baa6  gencode/java/udmi/schema/CloudQuery.java
 9b0fe553c2270b541f11acb25bfd18b8857aaac4e4b5d1ff09f4ca0f28121729  gencode/java/udmi/schema/Common.java
 377eb78de936317676faaf7ec5a32ad17f2c2c54a3280df87f5fcc2d36c7014d  gencode/java/udmi/schema/Config.java
@@ -192,7 +192,7 @@ a61368a737743f63365d1ec4c49ddc84c0e9a09452c6d73d4a4cd013e4bd015f  gencode/python
 8f4ec5b4d717a0c497e914a2e15c72d96bfd3bbb119fa3b1e21ac96243c195bd  gencode/python/udmi/schema/events_validation.py
 3707a9a5a07b7cf80e4ce6b0ca81584de74a9d5fe361214a4d3b6f22dd30cad8  gencode/python/udmi/schema/events_validation_device.py
 7476dc629261c39851a6b3a4e76bda2f3db3d57b8a87fb5e020f3833b572cc9d  gencode/python/udmi/schema/metadata.py
-547e0cbdb212fec4e2daac9d5c5bcbe86f6657a2e9ad967c4354107eef691514  gencode/python/udmi/schema/model_cloud.py
+8a7736e5869c810113208cd97a14401d0fb0a3c07acc6e20fc877f49dbb961a2  gencode/python/udmi/schema/model_cloud.py
 2ec91522178789a3d5ec27ad8ed13a94b8619139359dc3dcad9601b9adc1582d  gencode/python/udmi/schema/model_cloud_config.py
 387e3f68dad9ddd6211a4e10d25c35026a4c0083b606819fdb4e338be9e135ca  gencode/python/udmi/schema/model_discovery.py
 cead43bbefcc2d957bf8316560edf849d74df9270f9473d87c2a3bdf61f08332  gencode/python/udmi/schema/model_discovery_family.py

.github/workflows/testing.yml

@@ -222,12 +222,50 @@ jobs:
           name: udmi-support_${{ github.run_id }}-l
           path: '*_udmi-support_*.tgz'
 
+  automapping:
+    name: Automapping capability
+    runs-on: ubuntu-latest
+    needs: baseline
+    timeout-minutes: 10
+    if: ${{ vars.TARGET_PROJECT != '' && !cancelled() }}
+    env:
+      TARGET_PROJECT: ${{ vars.TARGET_PROJECT }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: base setup
+        run: |
+          bin/setup_base
+          bin/clone_model
+      - name: local setup
+        run: bin/start_local sites/udmi_site_model $TARGET_PROJECT
+      - name: bin/test_automapper
+        run: bin/test_automapper $TARGET_PROJECT
+      - name: udmis log
+        if: ${{ !cancelled() }}
+        run: cat /tmp/udmis.log
+      - name: pubber log
+        if: ${{ !cancelled() }}
+        run: cat out/pubber.log.GAT-123
+      - name: support bundle
+        if: ${{ !cancelled() }}
+        run: bin/support ${{ github.repository_owner }}_${{ github.job }}_
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          if-no-files-found: error
+          name: udmi-support_${{ github.run_id }}-m
+          path: '*_udmi-support_*.tgz'
+
   redirect:
     name: Endpoint Redirection
     runs-on: ubuntu-latest
-    needs: baseline
+    needs: automapping
     timeout-minutes: 15
-    if: vars.TARGET_PROJECT != ''
+    if: ${{ vars.TARGET_PROJECT != '' && !cancelled() }}
     env:
       TARGET_PROJECT: ${{ vars.TARGET_PROJECT }}
       UDMI_ALT_REGISTRY: ZZ-REDIRECT-NA

bin/mapper

@@ -3,14 +3,17 @@
 UDMI_ROOT=$(dirname $0)/..
 source $UDMI_ROOT/etc/shell_common.sh
 
-in_file=/tmp/registrar_config.json
-out_file=/tmp/mapper_config.json
+in_file=out/registrar_conf.json
+out_file=out/mapper_conf.json
+
+if [[ $# != 2 ]]; then
+    usage device_id command
+fi
 
 device_id=$1
 command=$2
-shift 2 || fail $0 device_id command
+shift 2
 
 jq .device_id=\"$device_id\" $in_file > $out_file
 
-
 java -cp $UDMI_JAR com.google.daq.mqtt.mapping.MappingAgent $out_file $command

bin/test_automapper

@@ -0,0 +1,49 @@
+#!/bin/bash -e
+
+UDMI_ROOT=$(dirname $0)/..
+cd $UDMI_ROOT
+
+source etc/shell_common.sh
+
+if [[ $# != 1 ]]; then
+   usage PROJECT_SPEC
+fi
+
+project_spec=$1
+shift
+
+site_path=sites/udmi_site_model
+
+bin/registrar $site_path $project_spec -x -d
+
+# Initialize the system to have no proxied devices present
+metadata=$site_path/devices/GAT-123/metadata.json
+jq '.gateway.proxy_ids = []' $metadata | sponge $metadata
+
+bin/registrar $site_path $project_spec GAT-123
+
+entries=$(wc -l < $site_path/out/registration_summary.csv)
+[[ $entries == 2 ]] || fail Unexpected registered entries, found $entries
+
+pubber_bg GAT-123
+
+bin/mapper GAT-123 provision
+
+bin/mapper GAT-123 discover
+
+echo Waiting for discovery...
+sleep 20
+
+echo Extracting results at $(date -u -Is)
+bin/registrar $site_path $project_spec
+
+status=$(fgrep discovered_vendor-281 $site_path/out/registration_summary.csv | awk '{print $3}') || true
+echo vendor device status is $status
+[[ $status == BLOCK, ]] || fail Vendor device status should be BLOCK
+
+type=$(jq -r .resource_type sites/udmi_site_model/extras/discovered_vendor-20231/cloud_model.json) || true
+echo vendor extracted device type $type
+[[ $type == DEVICE ]] || fail Vendor device type should be DEVICE
+
+echo
+echo Done with automapper test

bin/test_itemized

@@ -45,6 +45,8 @@ done
 
 test_index=1
 while read -u 7 action test_name pubber_opts; do
+    echo
+    echo =============================================================
     test_marker=$(printf %02d $test_index)
     ((test_index++)) || true
 

common/src/main/java/com/google/udmi/util/Common.java

@@ -56,6 +56,7 @@ public abstract class Common {
   public static final String CATEGORY_PROPERTY_KEY = "category";
   public static final Pattern DEVICE_ID_ALLOWABLE = Pattern.compile("^[-_a-zA-Z0-9]+$");
   public static final Pattern POINT_NAME_ALLOWABLE = DEVICE_ID_ALLOWABLE;
+  public static final int SEC_TO_MS = 1000;
 
   private static final String PREFIX_SEPARATOR = "~";
   private static final String UDMI_VERSION_ENV = "UDMI_TOOLS";

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -99,8 +99,8 @@ public static String changedLines(List<DiffEntry> nullableChanges) {
    * the target class is "final" but the fields themselves need to be updated.
    *
    * @param from source object
-   * @param to   target object
-   * @param <T>  type of object
+   * @param to target object
+   * @param <T> type of object
    */
   public static <T> void copyFields(T from, T to, boolean includeNull) {
     Field[] fields = from.getClass().getDeclaredFields();
@@ -283,13 +283,18 @@ public static Map<String, Object> getSubMapDefault(Map<String, Object> input, St
   }
 
   public static <T, V> V ifNotNullGet(T value, Function<T, V> converter) {
-    return ifNotNullGet(value, converter, (V) null);
+    return ifNotNullGet(value, converter, null);
   }
 
   public static <T, V> V ifNotNullGet(T value, Function<T, V> converter, V elseResult) {
     return value == null ? elseResult : converter.apply(value);
   }
 
+  public static <T, V> V ifNotNullGetElse(T value, Function<T, V> converter,
+      Supplier<V> elseResult) {
+    return value == null ? elseResult.get() : converter.apply(value);
+  }
+
   public static <T, V> V ifNotNullGet(T value, Supplier<V> converter) {
     return value == null ? null : converter.get();
   }

common/src/main/java/com/google/udmi/util/JsonUtil.java

@@ -439,6 +439,18 @@ public static Map<String, String> toStringMap(String message) {
     return map;
   }
 
+  /**
+   * Convert the pojo to a mapped representation of strings only.
+   *
+   * @param message input object to convert
+   * @return object-as-map
+   */
+  public static Map<String, String> toStringMapStr(String message) {
+    @SuppressWarnings("unchecked")
+    Map<String, String> map = fromString(TreeMap.class, message);
+    return map;
+  }
+
   /**
    * Extract the underlying string representation from a JSON encoded message.
    */

common/src/main/java/com/google/udmi/util/MetadataMapKeys.java

@@ -8,6 +8,7 @@ public class MetadataMapKeys {
   public static final String UDMI_GENERATION = "udmi_generation";
   public static final String UDMI_UPDATED = "udmi_updated";
   public static final String UDMI_PROVISION_GENERATION = "udmi_provision_generation";
+  public static final String UDMI_PROVISION_ENABLE = "udmi_provisioning_enabled";
   public static final String UDMI_DISCOVERED_FROM = "udmi_discovered_from";
   public static final String UDMI_DISCOVERED_WITH = "udmi_discovered_with";
   public static final String KEY_BYTES_KEY = "key_bytes";

etc/validator.out

@@ -276,19 +276,12 @@ sites/udmi_site_model/out/devices/AHU-22/state.out
   "sub_folder" : "update",
   "sub_type" : "state",
   "status" : {
-    "message" : "missing pointset subblock",
-    "detail" : "state_update: missing pointset subblock",
-    "category" : "validation.device.schema",
+    "message" : "Successful validation",
+    "category" : "validation.device.receive",
     "timestamp" : "REDACTED_TIMESTAMP",
-    "level" : 500
+    "level" : 200
   },
-  "errors" : [ {
-    "message" : "missing pointset subblock",
-    "detail" : "state_update: missing pointset subblock",
-    "category" : "validation.device.schema",
-    "timestamp" : "REDACTED_TIMESTAMP",
-    "level" : 500
-  } ]
+  "errors" : [ ]
 }
 ::::::::::::::
 sites/udmi_site_model/out/devices/AHU-22/state_gateway.out
@@ -667,15 +660,15 @@ sites/udmi_site_model/out/devices/SNS-4/state.out
   "sub_folder" : "update",
   "sub_type" : "state",
   "status" : {
-    "message" : "missing pointset subblock",
-    "detail" : "state_update: missing pointset subblock",
+    "message" : "Device has missing points: split_threshold, triangulating_axis",
+    "detail" : "state_update: Device has missing points: split_threshold, triangulating_axis",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "missing pointset subblock",
-    "detail" : "state_update: missing pointset subblock",
+    "message" : "Device has missing points: split_threshold, triangulating_axis",
+    "detail" : "state_update: Device has missing points: split_threshold, triangulating_axis",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500