Discovery node enhancements and testing

.gencode_hash.txt

@@ -7,14 +7,14 @@ fb876b8f7bfdccf156d751efddf2660a7a7ab5585e89be0f82c59f7a61e68d4c  gencode/docs/c
 6a097f1f87ab7b9a82e4d3aa6f6cedf69499a95742863a7f53bb5fdf53fd6ff0  gencode/docs/configuration_pod.html
 4d327975ae4d48a5dd920f298931fc5056a7239d677e61a9f4d129068316dd0a  gencode/docs/configuration_pubber.html
 f24d15e549f1b143b50d6011f2f7fd03286e1cd128a3e25591195b25f8efc472  gencode/docs/data_template.html
-45d39c61ce39afc0d14f3dc99a93947587d63999cd373cb44b63ce8f576c6e7e  gencode/docs/events.html
-45f60d2b63951448dded8c9c170913af428e50e8b4cff2143c97304332077df6  gencode/docs/events_discovery.html
+012fa1275063d526db96ee4e02f8a3fd7fdf4aa5d5b9bcc7267cd22b3adc935d  gencode/docs/events.html
+59cc4bfb2f0523d87c29cc8a475049432a9ac374728d60486897cbea6ca71701  gencode/docs/events_discovery.html
 65858e981eaad27e41fd7f64aae7d6a82d4519e798145eb10898e59cead517c3  gencode/docs/events_mapping.html
 c3bae844432d172033bc416e623fecee7608efd01d916b7eaee96252932c552a  gencode/docs/events_pointset.html
 eed6ae125d94cf1986de96c210b0937b9c7b199724839f43fce3b357b057f8be  gencode/docs/events_system.html
 e59e52110ec12d7b82351abfc28c31255c57e2af637adf3fa5cbb7e3b26b49fe  gencode/docs/events_udmi.html
 e3683cb4561b6dda5703cb659fd6a8f12242725de389709a12eb45f146cbb295  gencode/docs/events_validation.html
-3e02bab5ef46438c99f20049f30f8dd4428530b6036aa20f29a8a3964ac2b580  gencode/docs/metadata.html
+264b6eef8595d766d1e148f6a9e0281027fc8c6e252e3236c0312b38ecd8a076  gencode/docs/metadata.html
 b110edfb73182782c7539adef417a970959432c92d4c9f4a0bf7e328c0e8427d  gencode/docs/monitoring.html
 c8a40993b2810dffb9119469f83592461d817c7148072203f157b89922187e7c  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
@@ -44,7 +44,7 @@ fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56  gencode/java/u
 0c133f64013d5f2c4be203708def7ae11be631e90d5992222d6b4c97eef2573e  gencode/java/udmi/schema/CapabilityValidationState.java
 ec164962f2f00924ecb41ae07c2e01d6bf027951ccc605e0210988370b987973  gencode/java/udmi/schema/Category.java
 a010037f8ad570060c6a03e5aa13bc6d5261a61bf70eb71e0d00e124036decea  gencode/java/udmi/schema/CloudConfigModel.java
-eec1119dd068464fd74fbf2b12d0cd39e80b8dc88c6a1b5b08678ed5c78ff6b4  gencode/java/udmi/schema/CloudModel.java
+65328b8a88d357ad3205e3fa8eef6fe9148a86b955790022be3c7804cab145fb  gencode/java/udmi/schema/CloudModel.java
 06c8c3131f111e49e0d3e518603a3f66349d5dee1aee64a98659cb8703a8baa6  gencode/java/udmi/schema/CloudQuery.java
 9b0fe553c2270b541f11acb25bfd18b8857aaac4e4b5d1ff09f4ca0f28121729  gencode/java/udmi/schema/Common.java
 377eb78de936317676faaf7ec5a32ad17f2c2c54a3280df87f5fcc2d36c7014d  gencode/java/udmi/schema/Config.java
@@ -86,22 +86,21 @@ b0d4bff14a65ebddc2dd253c996708a4cce99592e82978f057f32a6d9c7768a1  gencode/java/u
 5a0cc53317592a868f7f23a67a7493bd571054bb25d9e28de51000ad6473dfeb  gencode/java/udmi/schema/Level.java
 bfa27f6c1804ec9a3d115249c64b3bdf7d3b0b58d28e3cb4e69f70eddf3a6efb  gencode/java/udmi/schema/Links.java
 6faf591c87776c170d4314a6b9a9eaf86b99c15b7b5c7a738827a945082c4570  gencode/java/udmi/schema/LocalnetConfig.java
-3bf878f6f049aa0e3aee6748174188bd7ddf010e27b9133fd49950452568f952  gencode/java/udmi/schema/LocalnetModel.java
+910c68183db7703b00bcb81146ad73e6fe0d4bbc4caec4dc9c621f3cc2e5eee5  gencode/java/udmi/schema/LocalnetModel.java
 f6dd2fff0e100160f84a5d62c137c618ed616c82f8722a0530f5393bfc82e281  gencode/java/udmi/schema/LocalnetState.java
-fd76f99a242320a331a5d4f052e9e0a4e7e5974a2d55cbd12223700245d34dbb  gencode/java/udmi/schema/Location.java
+17556111117940a6e780801b0c7125bc048d6100493f2b018659cd7e5f8cd654  gencode/java/udmi/schema/Location.java
 7f71faa644ac86f3dcaf832d39a0deecfe443fc6a3bff912c0a66c09243c4cd9  gencode/java/udmi/schema/MappingCommand.java
 0f6f85a4aaffb90735acc2565e8e4afa349d3ded5781cb1dce1ebbaa237feaa7  gencode/java/udmi/schema/MappingConfig.java
 2786ae25faad6bd0d9f20985e3576ffbe15c1b98c16033f7459a00df3573c44b  gencode/java/udmi/schema/MappingEventEntity.java
 9ba718b4f74dd3c2022ca57ca4d1527a2c69c8358f02356392bfb9a0a4e9619d  gencode/java/udmi/schema/MappingEvents.java
 84fc81b687d42c689b6b9794716e34bdfd06daf3ab3640a5cf47baf74c185cee  gencode/java/udmi/schema/MappingState.java
 73c28ed2d7b7f8ee9abdab9fcb9ac7b261fc2c4f03e37cb5122c535e31d38fdd  gencode/java/udmi/schema/MessageTemplateData.java
-8f5f3e92d495387857c02362d3f1f569d998aa4f2fa7231e5e1e9ffa8add639f  gencode/java/udmi/schema/Metadata.java
+cdd5ab56904d434e40bfbad17f3697696e938b979d7ab6cef2e280d7a5cfe44f  gencode/java/udmi/schema/Metadata.java
 df64e4ddbf543ac70e7c2af9d3fbc20ffe3dff68c6718aa9ceadab7f64d3d171  gencode/java/udmi/schema/Metrics.java
 14722df90406cdcd29c7c946e5fcdf6a3e513eb879a7fbb493801fc0d9093504  gencode/java/udmi/schema/Monitoring.java
 31dab2c11cfcfdd346b1fe83e92a86abc652de05d85634a22a2c5c3f17c741aa  gencode/java/udmi/schema/MonitoringMetric.java
 4e7fb33911cf11845d494a07502577647cb96e1196cb7b76553190d14b6ed099  gencode/java/udmi/schema/Operation.java
 626ba7dd9251b8b949632d9a7da0b5519a7e86be3df0dae153f6d59578044a78  gencode/java/udmi/schema/Parameters.java
-fd3326c10c26fee4a930351c86318ae69661a427b0815602a5094c8685835123  gencode/java/udmi/schema/Parent.java
 5e1c5411fae4d7c47391ceb5d19ae864fcd484df75ac6b6db39fd2d12647dec8  gencode/java/udmi/schema/Physical_tag.java
 9cc6f769b6da904d59e161f4809d9bee2d0e3dd137f863af2955c468905c23ad  gencode/java/udmi/schema/PodConfiguration.java
 468c2c95e9dcbaf4ff706fad7f168c14321eaa35b7fb190848ea8bfd86d981d7  gencode/java/udmi/schema/PointPointsetConfig.java
@@ -130,7 +129,7 @@ c9b920d355e5f6350fa6eb978afbfe55dd1438df171d2ea215845d5ac0b3e7e9  gencode/java/u
 0797914b73ef17e5d7a54f7f27ab1838c9002ffe41d7145f6da54482bbd6e879  gencode/java/udmi/schema/SystemDiscoveryData.java
 605e285545247dbd1e37bfaf3cf3af7cda1b45d0bf0098e1a224297ba5b55a43  gencode/java/udmi/schema/SystemEvents.java
 2cf23174ef4e2876511fb471d3f9fcb5cefe2fde324db844c2d0d505fd2c8844  gencode/java/udmi/schema/SystemHardware.java
-749bd1dee5b957b35f052b5940a8d77ac48b8b4d47d168f2c44760dcd615f8e5  gencode/java/udmi/schema/SystemModel.java
+f055daf2ce49e634367f59d7fdf5c81bf62c6f69c49e51bdda8bd0fcb994aa66  gencode/java/udmi/schema/SystemModel.java
 51e073ccda479c77b6cc3d826c3c69a12bd12f1159137a9aa479b4eb71f156ba  gencode/java/udmi/schema/SystemState.java
 7d6dd13e368e7f073738fee69c15e18652a9b7d7ac63bde0a200f747e3aa1b1d  gencode/java/udmi/schema/TargetTestingModel.java
 7db9435bd72c03d4bc4bff720d6cee1b65c7af5d10687579e44d96d2d1683813  gencode/java/udmi/schema/TestingModel.java
@@ -191,18 +190,18 @@ a61368a737743f63365d1ec4c49ddc84c0e9a09452c6d73d4a4cd013e4bd015f  gencode/python
 0423bfd2158a71a1ce3961054aceacbe6603ad1f65a707b586d17bfabb2a99cb  gencode/python/udmi/schema/events_udmi.py
 8f4ec5b4d717a0c497e914a2e15c72d96bfd3bbb119fa3b1e21ac96243c195bd  gencode/python/udmi/schema/events_validation.py
 3707a9a5a07b7cf80e4ce6b0ca81584de74a9d5fe361214a4d3b6f22dd30cad8  gencode/python/udmi/schema/events_validation_device.py
-7476dc629261c39851a6b3a4e76bda2f3db3d57b8a87fb5e020f3833b572cc9d  gencode/python/udmi/schema/metadata.py
-8a7736e5869c810113208cd97a14401d0fb0a3c07acc6e20fc877f49dbb961a2  gencode/python/udmi/schema/model_cloud.py
+9aabddaa4ef5a27f61a3b064fcd3a5e0cf3f4269945f15f51609724837c5d47c  gencode/python/udmi/schema/metadata.py
+547e0cbdb212fec4e2daac9d5c5bcbe86f6657a2e9ad967c4354107eef691514  gencode/python/udmi/schema/model_cloud.py
 2ec91522178789a3d5ec27ad8ed13a94b8619139359dc3dcad9601b9adc1582d  gencode/python/udmi/schema/model_cloud_config.py
 387e3f68dad9ddd6211a4e10d25c35026a4c0083b606819fdb4e338be9e135ca  gencode/python/udmi/schema/model_discovery.py
 cead43bbefcc2d957bf8316560edf849d74df9270f9473d87c2a3bdf61f08332  gencode/python/udmi/schema/model_discovery_family.py
 bef91798566798ec70b2fe10ae52c909090de9095b92e2567da3cbdcec29cd2c  gencode/python/udmi/schema/model_features.py
 72d17ba53c9d537720bb0ddf24d615b838fc2b9bb2a5c288742a8d82c77bd85d  gencode/python/udmi/schema/model_gateway.py
-3e2f7eea6bc5f56b416d883f1da7336f479ab2f88ae47d174d22b7941ac9cfc3  gencode/python/udmi/schema/model_localnet.py
+ae6a7bc93b520cdc90cd158f876a58e98a3e000402ebb1ca632f01ee492d408a  gencode/python/udmi/schema/model_localnet.py
 083afb4579592a702876e579bdc3986d85b803f6e0b3da70729ef928502ccad8  gencode/python/udmi/schema/model_localnet_family.py
 cd4be21b66fb3e28dff96cfd39a878791ae8543f2fe67269b66c2c360d3f2513  gencode/python/udmi/schema/model_pointset.py
 1378f80c1b968d13d546fc7b2125df772f97573a703955e6eb55950dd621d9e7  gencode/python/udmi/schema/model_pointset_point.py
-f579e5c4fd867fb7878554e7f963ca053891169076ddf444b09add7f22518fd4  gencode/python/udmi/schema/model_system.py
+77809d848cb5ebe1baaedf7dd1dded338ffcf0b68a28651c46e5b5e0b2909fc1  gencode/python/udmi/schema/model_system.py
 15b349141ebae651c6c3c5c313b197d49c8b2b44e8ff1b0639848ad42e5c4e63  gencode/python/udmi/schema/model_system_hardware.py
 b48ae013d203eb31cc388a084c0cc6f93d4d5534336d4da2ecbccd84a085ea2b  gencode/python/udmi/schema/model_testing.py
 5c50847e136a033ea511209238bb570499b43fbee6189dae06603132dcb9f01f  gencode/python/udmi/schema/model_testing_target.py

.github/workflows/testing.yml

@@ -222,50 +222,12 @@ jobs:
           name: udmi-support_${{ github.run_id }}-l
           path: '*_udmi-support_*.tgz'
 
-  automapping:
-    name: Automapping capability
-    runs-on: ubuntu-latest
-    needs: baseline
-    timeout-minutes: 10
-    if: ${{ vars.TARGET_PROJECT != '' && !cancelled() }}
-    env:
-      TARGET_PROJECT: ${{ vars.TARGET_PROJECT }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-java@v4
-        with:
-          distribution: 'temurin'
-          java-version: '17'
-      - name: base setup
-        run: |
-          bin/setup_base
-          bin/clone_model
-      - name: local setup
-        run: bin/start_local sites/udmi_site_model $TARGET_PROJECT
-      - name: bin/test_automapper
-        run: bin/test_automapper $TARGET_PROJECT
-      - name: udmis log
-        if: ${{ !cancelled() }}
-        run: cat /tmp/udmis.log
-      - name: pubber log
-        if: ${{ !cancelled() }}
-        run: cat out/pubber.log.GAT-123
-      - name: support bundle
-        if: ${{ !cancelled() }}
-        run: bin/support ${{ github.repository_owner }}_${{ github.job }}_
-      - uses: actions/upload-artifact@v4
-        if: ${{ !cancelled() }}
-        with:
-          if-no-files-found: error
-          name: udmi-support_${{ github.run_id }}-m
-          path: '*_udmi-support_*.tgz'
-
   redirect:
     name: Endpoint Redirection
     runs-on: ubuntu-latest
-    needs: automapping
+    needs: baseline
     timeout-minutes: 15
-    if: ${{ vars.TARGET_PROJECT != '' && !cancelled() }}
+    if: vars.TARGET_PROJECT != ''
     env:
       TARGET_PROJECT: ${{ vars.TARGET_PROJECT }}
       UDMI_ALT_REGISTRY: ZZ-REDIRECT-NA
@@ -340,3 +302,26 @@ jobs:
       - name: Itemized validation
         if: ${{ !cancelled() }}
         run: bin/test_itemcheck
+
+  discoverynode:
+    name: Discovery Node
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: base setup
+        run: |
+          bin/setup_base
+          bin/clone_model
+          bin/registrar sites/udmi_site_model
+      - name: integration test
+        run: |
+          misc/discoverynode/testing/integration/integration_test.sh
+      - name: sequencer test
+        if: always()
+        run: |
+          misc/discoverynode/testing/sequencer/sequencer_test.sh   

bin/mapper

@@ -3,17 +3,14 @@
 UDMI_ROOT=$(dirname $0)/..
 source $UDMI_ROOT/etc/shell_common.sh
 
-in_file=out/registrar_conf.json
-out_file=out/mapper_conf.json
-
-if [[ $# != 2 ]]; then
-    usage device_id command
-fi
+in_file=/tmp/registrar_config.json
+out_file=/tmp/mapper_config.json
 
 device_id=$1
 command=$2
-shift 2
+shift 2 || fail $0 device_id command
 
 jq .device_id=\"$device_id\" $in_file > $out_file
 
+
 java -cp $UDMI_JAR com.google.daq.mqtt.mapping.MappingAgent $out_file $command

bin/test_itemized

@@ -45,8 +45,6 @@ done
 
 test_index=1
 while read -u 7 action test_name pubber_opts; do
-    echo
-    echo =============================================================
     test_marker=$(printf %02d $test_index)
     ((test_index++)) || true
 

bin/test_mapping

@@ -0,0 +1,26 @@
+#!/bin/bash -e
+
+UDMI_ROOT=$(realpath $(dirname $0)/..)
+source $UDMI_ROOT/etc/shell_common.sh
+
+PROJECT_SPEC=$1
+shift || fail Missing project_spec
+
+cd $UDMI_ROOT
+rm -rf tmp/discovery
+cp -a tests/sites/discovery/ tmp/sites/
+
+cd tmp/sites/
+
+./cleanup_site
+
+rm -rf reflector devices/AHU-1 devices/AHU-22
+cp -a $UDMI_ROOT/sites/udmi_site_model/reflector/ .
+cp -a $UDMI_ROOT/sites/udmi_site_model/devices/GAT-123/ec* devices/GAT-123/
+
+jq .gateway.proxy_ids=[] devices/GAT-123/metadata.json | sponge devices/GAT-123/metadata.json
+
+$UDMI_ROOT/bin/registrar . $PROJECT_SPEC -x -d
+$UDMI_ROOT/bin/registrar . $PROJECT_SPEC
+
+echo Done with mapping test setup.

common/src/main/java/com/google/udmi/util/Common.java

@@ -56,7 +56,6 @@ public abstract class Common {
   public static final String CATEGORY_PROPERTY_KEY = "category";
   public static final Pattern DEVICE_ID_ALLOWABLE = Pattern.compile("^[-_a-zA-Z0-9]+$");
   public static final Pattern POINT_NAME_ALLOWABLE = DEVICE_ID_ALLOWABLE;
-  public static final int SEC_TO_MS = 1000;
 
   private static final String PREFIX_SEPARATOR = "~";
   private static final String UDMI_VERSION_ENV = "UDMI_TOOLS";

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -99,8 +99,8 @@ public static String changedLines(List<DiffEntry> nullableChanges) {
    * the target class is "final" but the fields themselves need to be updated.
    *
    * @param from source object
-   * @param to target object
-   * @param <T> type of object
+   * @param to   target object
+   * @param <T>  type of object
    */
   public static <T> void copyFields(T from, T to, boolean includeNull) {
     Field[] fields = from.getClass().getDeclaredFields();
@@ -283,18 +283,13 @@ public static Map<String, Object> getSubMapDefault(Map<String, Object> input, St
   }
 
   public static <T, V> V ifNotNullGet(T value, Function<T, V> converter) {
-    return ifNotNullGet(value, converter, null);
+    return ifNotNullGet(value, converter, (V) null);
   }
 
   public static <T, V> V ifNotNullGet(T value, Function<T, V> converter, V elseResult) {
     return value == null ? elseResult : converter.apply(value);
   }
 
-  public static <T, V> V ifNotNullGetElse(T value, Function<T, V> converter,
-      Supplier<V> elseResult) {
-    return value == null ? elseResult.get() : converter.apply(value);
-  }
-
   public static <T, V> V ifNotNullGet(T value, Supplier<V> converter) {
     return value == null ? null : converter.get();
   }

common/src/main/java/com/google/udmi/util/JsonUtil.java

@@ -439,18 +439,6 @@ public static Map<String, String> toStringMap(String message) {
     return map;
   }
 
-  /**
-   * Convert the pojo to a mapped representation of strings only.
-   *
-   * @param message input object to convert
-   * @return object-as-map
-   */
-  public static Map<String, String> toStringMapStr(String message) {
-    @SuppressWarnings("unchecked")
-    Map<String, String> map = fromString(TreeMap.class, message);
-    return map;
-  }
-
   /**
    * Extract the underlying string representation from a JSON encoded message.
    */

common/src/main/java/com/google/udmi/util/MetadataMapKeys.java

@@ -8,7 +8,6 @@ public class MetadataMapKeys {
   public static final String UDMI_GENERATION = "udmi_generation";
   public static final String UDMI_UPDATED = "udmi_updated";
   public static final String UDMI_PROVISION_GENERATION = "udmi_provision_generation";
-  public static final String UDMI_PROVISION_ENABLE = "udmi_provisioning_enabled";
   public static final String UDMI_DISCOVERED_FROM = "udmi_discovered_from";
   public static final String UDMI_DISCOVERED_WITH = "udmi_discovered_with";
   public static final String KEY_BYTES_KEY = "key_bytes";

etc/validator.out

@@ -276,12 +276,19 @@ sites/udmi_site_model/out/devices/AHU-22/state.out
   "sub_folder" : "update",
   "sub_type" : "state",
   "status" : {
-    "message" : "Successful validation",
-    "category" : "validation.device.receive",
+    "message" : "missing pointset subblock",
+    "detail" : "state_update: missing pointset subblock",
+    "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
-    "level" : 200
+    "level" : 500
   },
-  "errors" : [ ]
+  "errors" : [ {
+    "message" : "missing pointset subblock",
+    "detail" : "state_update: missing pointset subblock",
+    "category" : "validation.device.schema",
+    "timestamp" : "REDACTED_TIMESTAMP",
+    "level" : 500
+  } ]
 }
 ::::::::::::::
 sites/udmi_site_model/out/devices/AHU-22/state_gateway.out
@@ -660,15 +667,15 @@ sites/udmi_site_model/out/devices/SNS-4/state.out
   "sub_folder" : "update",
   "sub_type" : "state",
   "status" : {
-    "message" : "Device has missing points: split_threshold, triangulating_axis",
-    "detail" : "state_update: Device has missing points: split_threshold, triangulating_axis",
+    "message" : "missing pointset subblock",
+    "detail" : "state_update: missing pointset subblock",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "Device has missing points: split_threshold, triangulating_axis",
-    "detail" : "state_update: Device has missing points: split_threshold, triangulating_axis",
+    "message" : "missing pointset subblock",
+    "detail" : "state_update: missing pointset subblock",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500