diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 88b77d574e..07bd49c9e3 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -7,14 +7,14 @@ fb876b8f7bfdccf156d751efddf2660a7a7ab5585e89be0f82c59f7a61e68d4c gencode/docs/c 6a097f1f87ab7b9a82e4d3aa6f6cedf69499a95742863a7f53bb5fdf53fd6ff0 gencode/docs/configuration_pod.html 4d327975ae4d48a5dd920f298931fc5056a7239d677e61a9f4d129068316dd0a gencode/docs/configuration_pubber.html f24d15e549f1b143b50d6011f2f7fd03286e1cd128a3e25591195b25f8efc472 gencode/docs/data_template.html -012fa1275063d526db96ee4e02f8a3fd7fdf4aa5d5b9bcc7267cd22b3adc935d gencode/docs/events.html -59cc4bfb2f0523d87c29cc8a475049432a9ac374728d60486897cbea6ca71701 gencode/docs/events_discovery.html +45d39c61ce39afc0d14f3dc99a93947587d63999cd373cb44b63ce8f576c6e7e gencode/docs/events.html +45f60d2b63951448dded8c9c170913af428e50e8b4cff2143c97304332077df6 gencode/docs/events_discovery.html 65858e981eaad27e41fd7f64aae7d6a82d4519e798145eb10898e59cead517c3 gencode/docs/events_mapping.html c3bae844432d172033bc416e623fecee7608efd01d916b7eaee96252932c552a gencode/docs/events_pointset.html eed6ae125d94cf1986de96c210b0937b9c7b199724839f43fce3b357b057f8be gencode/docs/events_system.html e59e52110ec12d7b82351abfc28c31255c57e2af637adf3fa5cbb7e3b26b49fe gencode/docs/events_udmi.html e3683cb4561b6dda5703cb659fd6a8f12242725de389709a12eb45f146cbb295 gencode/docs/events_validation.html -fd8e63cdfe293f4e8a2c667200aec77c7c4856ea1e9436c88fda00a6bef77c5b gencode/docs/metadata.html +3e02bab5ef46438c99f20049f30f8dd4428530b6036aa20f29a8a3964ac2b580 gencode/docs/metadata.html b110edfb73182782c7539adef417a970959432c92d4c9f4a0bf7e328c0e8427d gencode/docs/monitoring.html c8a40993b2810dffb9119469f83592461d817c7148072203f157b89922187e7c gencode/docs/persistent_device.html 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d gencode/docs/properties.html @@ -44,7 +44,7 @@ fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56 gencode/java/u 0c133f64013d5f2c4be203708def7ae11be631e90d5992222d6b4c97eef2573e gencode/java/udmi/schema/CapabilityValidationState.java ec164962f2f00924ecb41ae07c2e01d6bf027951ccc605e0210988370b987973 gencode/java/udmi/schema/Category.java a010037f8ad570060c6a03e5aa13bc6d5261a61bf70eb71e0d00e124036decea gencode/java/udmi/schema/CloudConfigModel.java -65328b8a88d357ad3205e3fa8eef6fe9148a86b955790022be3c7804cab145fb gencode/java/udmi/schema/CloudModel.java +eec1119dd068464fd74fbf2b12d0cd39e80b8dc88c6a1b5b08678ed5c78ff6b4 gencode/java/udmi/schema/CloudModel.java 06c8c3131f111e49e0d3e518603a3f66349d5dee1aee64a98659cb8703a8baa6 gencode/java/udmi/schema/CloudQuery.java 9b0fe553c2270b541f11acb25bfd18b8857aaac4e4b5d1ff09f4ca0f28121729 gencode/java/udmi/schema/Common.java 377eb78de936317676faaf7ec5a32ad17f2c2c54a3280df87f5fcc2d36c7014d gencode/java/udmi/schema/Config.java @@ -192,7 +192,7 @@ a61368a737743f63365d1ec4c49ddc84c0e9a09452c6d73d4a4cd013e4bd015f gencode/python 8f4ec5b4d717a0c497e914a2e15c72d96bfd3bbb119fa3b1e21ac96243c195bd gencode/python/udmi/schema/events_validation.py 3707a9a5a07b7cf80e4ce6b0ca81584de74a9d5fe361214a4d3b6f22dd30cad8 gencode/python/udmi/schema/events_validation_device.py 7476dc629261c39851a6b3a4e76bda2f3db3d57b8a87fb5e020f3833b572cc9d gencode/python/udmi/schema/metadata.py -547e0cbdb212fec4e2daac9d5c5bcbe86f6657a2e9ad967c4354107eef691514 gencode/python/udmi/schema/model_cloud.py +8a7736e5869c810113208cd97a14401d0fb0a3c07acc6e20fc877f49dbb961a2 gencode/python/udmi/schema/model_cloud.py 2ec91522178789a3d5ec27ad8ed13a94b8619139359dc3dcad9601b9adc1582d gencode/python/udmi/schema/model_cloud_config.py 387e3f68dad9ddd6211a4e10d25c35026a4c0083b606819fdb4e338be9e135ca gencode/python/udmi/schema/model_discovery.py cead43bbefcc2d957bf8316560edf849d74df9270f9473d87c2a3bdf61f08332 gencode/python/udmi/schema/model_discovery_family.py diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ce6b8803fe..796411b3ec 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -222,12 +222,50 @@ jobs: name: udmi-support_${{ github.run_id }}-l path: '*_udmi-support_*.tgz' + automapping: + name: Automapping capability + runs-on: ubuntu-latest + needs: baseline + timeout-minutes: 10 + if: ${{ vars.TARGET_PROJECT != '' && !cancelled() }} + env: + TARGET_PROJECT: ${{ vars.TARGET_PROJECT }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + - name: base setup + run: | + bin/setup_base + bin/clone_model + - name: local setup + run: bin/start_local sites/udmi_site_model $TARGET_PROJECT + - name: bin/test_automapper + run: bin/test_automapper $TARGET_PROJECT + - name: udmis log + if: ${{ !cancelled() }} + run: cat /tmp/udmis.log + - name: pubber log + if: ${{ !cancelled() }} + run: cat out/pubber.log.GAT-123 + - name: support bundle + if: ${{ !cancelled() }} + run: bin/support ${{ github.repository_owner }}_${{ github.job }}_ + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + if-no-files-found: error + name: udmi-support_${{ github.run_id }}-m + path: '*_udmi-support_*.tgz' + redirect: name: Endpoint Redirection runs-on: ubuntu-latest - needs: baseline + needs: automapping timeout-minutes: 15 - if: vars.TARGET_PROJECT != '' + if: ${{ vars.TARGET_PROJECT != '' && !cancelled() }} env: TARGET_PROJECT: ${{ vars.TARGET_PROJECT }} UDMI_ALT_REGISTRY: ZZ-REDIRECT-NA diff --git a/bin/mapper b/bin/mapper index e0f5b50b29..4a479111ea 100755 --- a/bin/mapper +++ b/bin/mapper @@ -3,14 +3,17 @@ UDMI_ROOT=$(dirname $0)/.. source $UDMI_ROOT/etc/shell_common.sh -in_file=/tmp/registrar_config.json -out_file=/tmp/mapper_config.json +in_file=out/registrar_conf.json +out_file=out/mapper_conf.json + +if [[ $# != 2 ]]; then + usage device_id command +fi device_id=$1 command=$2 -shift 2 || fail $0 device_id command +shift 2 jq .device_id=\"$device_id\" $in_file > $out_file - java -cp $UDMI_JAR com.google.daq.mqtt.mapping.MappingAgent $out_file $command diff --git a/bin/test_automapper b/bin/test_automapper new file mode 100755 index 0000000000..459365cf19 --- /dev/null +++ b/bin/test_automapper @@ -0,0 +1,49 @@ +#!/bin/bash -e + +UDMI_ROOT=$(dirname $0)/.. +cd $UDMI_ROOT + +source etc/shell_common.sh + +if [[ $# != 1 ]]; then + usage PROJECT_SPEC +fi + +project_spec=$1 +shift + +site_path=sites/udmi_site_model + +bin/registrar $site_path $project_spec -x -d + +# Initialize the system to have no proxied devices present +metadata=$site_path/devices/GAT-123/metadata.json +jq '.gateway.proxy_ids = []' $metadata | sponge $metadata + +bin/registrar $site_path $project_spec GAT-123 + +entries=$(wc -l < $site_path/out/registration_summary.csv) +[[ $entries == 2 ]] || fail Unexpected registered entries, found $entries + +pubber_bg GAT-123 + +bin/mapper GAT-123 provision + +bin/mapper GAT-123 discover + +echo Waiting for discovery... +sleep 20 + +echo Extracting results at $(date -u -Is) +bin/registrar $site_path $project_spec + +status=$(fgrep discovered_vendor-281 $site_path/out/registration_summary.csv | awk '{print $3}') || true +echo vendor device status is $status +[[ $status == BLOCK, ]] || fail Vendor device status should be BLOCK + +type=$(jq -r .resource_type sites/udmi_site_model/extras/discovered_vendor-20231/cloud_model.json) || true +echo vendor extracted device type $type +[[ $type == DEVICE ]] || fail Vendor device type should be DEVICE + +echo +echo Done with automapper test diff --git a/bin/test_itemized b/bin/test_itemized index 46a241ba21..4f68d7d60f 100755 --- a/bin/test_itemized +++ b/bin/test_itemized @@ -45,6 +45,8 @@ done test_index=1 while read -u 7 action test_name pubber_opts; do + echo + echo ============================================================= test_marker=$(printf %02d $test_index) ((test_index++)) || true diff --git a/bin/test_mapping b/bin/test_mapping deleted file mode 100755 index bef4a747d6..0000000000 --- a/bin/test_mapping +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -e - -UDMI_ROOT=$(realpath $(dirname $0)/..) -source $UDMI_ROOT/etc/shell_common.sh - -PROJECT_SPEC=$1 -shift || fail Missing project_spec - -cd $UDMI_ROOT -rm -rf tmp/discovery -cp -a tests/sites/discovery/ tmp/sites/ - -cd tmp/sites/ - -./cleanup_site - -rm -rf reflector devices/AHU-1 devices/AHU-22 -cp -a $UDMI_ROOT/sites/udmi_site_model/reflector/ . -cp -a $UDMI_ROOT/sites/udmi_site_model/devices/GAT-123/ec* devices/GAT-123/ - -jq .gateway.proxy_ids=[] devices/GAT-123/metadata.json | sponge devices/GAT-123/metadata.json - -$UDMI_ROOT/bin/registrar . $PROJECT_SPEC -x -d -$UDMI_ROOT/bin/registrar . $PROJECT_SPEC - -echo Done with mapping test setup. diff --git a/common/src/main/java/com/google/udmi/util/Common.java b/common/src/main/java/com/google/udmi/util/Common.java index 1d43f9033d..a4319bdfb8 100644 --- a/common/src/main/java/com/google/udmi/util/Common.java +++ b/common/src/main/java/com/google/udmi/util/Common.java @@ -56,6 +56,7 @@ public abstract class Common { public static final String CATEGORY_PROPERTY_KEY = "category"; public static final Pattern DEVICE_ID_ALLOWABLE = Pattern.compile("^[-_a-zA-Z0-9]+$"); public static final Pattern POINT_NAME_ALLOWABLE = DEVICE_ID_ALLOWABLE; + public static final int SEC_TO_MS = 1000; private static final String PREFIX_SEPARATOR = "~"; private static final String UDMI_VERSION_ENV = "UDMI_TOOLS"; diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index 10cefc68b0..67982a5b01 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -99,8 +99,8 @@ public static String changedLines(List nullableChanges) { * the target class is "final" but the fields themselves need to be updated. * * @param from source object - * @param to target object - * @param type of object + * @param to target object + * @param type of object */ public static void copyFields(T from, T to, boolean includeNull) { Field[] fields = from.getClass().getDeclaredFields(); @@ -283,13 +283,18 @@ public static Map getSubMapDefault(Map input, St } public static V ifNotNullGet(T value, Function converter) { - return ifNotNullGet(value, converter, (V) null); + return ifNotNullGet(value, converter, null); } public static V ifNotNullGet(T value, Function converter, V elseResult) { return value == null ? elseResult : converter.apply(value); } + public static V ifNotNullGetElse(T value, Function converter, + Supplier elseResult) { + return value == null ? elseResult.get() : converter.apply(value); + } + public static V ifNotNullGet(T value, Supplier converter) { return value == null ? null : converter.get(); } diff --git a/common/src/main/java/com/google/udmi/util/JsonUtil.java b/common/src/main/java/com/google/udmi/util/JsonUtil.java index a5fe16c485..fed7fedf74 100644 --- a/common/src/main/java/com/google/udmi/util/JsonUtil.java +++ b/common/src/main/java/com/google/udmi/util/JsonUtil.java @@ -439,6 +439,18 @@ public static Map toStringMap(String message) { return map; } + /** + * Convert the pojo to a mapped representation of strings only. + * + * @param message input object to convert + * @return object-as-map + */ + public static Map toStringMapStr(String message) { + @SuppressWarnings("unchecked") + Map map = fromString(TreeMap.class, message); + return map; + } + /** * Extract the underlying string representation from a JSON encoded message. */ diff --git a/common/src/main/java/com/google/udmi/util/MetadataMapKeys.java b/common/src/main/java/com/google/udmi/util/MetadataMapKeys.java index 610bdc0a96..0811c152d0 100644 --- a/common/src/main/java/com/google/udmi/util/MetadataMapKeys.java +++ b/common/src/main/java/com/google/udmi/util/MetadataMapKeys.java @@ -8,6 +8,7 @@ public class MetadataMapKeys { public static final String UDMI_GENERATION = "udmi_generation"; public static final String UDMI_UPDATED = "udmi_updated"; public static final String UDMI_PROVISION_GENERATION = "udmi_provision_generation"; + public static final String UDMI_PROVISION_ENABLE = "udmi_provisioning_enabled"; public static final String UDMI_DISCOVERED_FROM = "udmi_discovered_from"; public static final String UDMI_DISCOVERED_WITH = "udmi_discovered_with"; public static final String KEY_BYTES_KEY = "key_bytes"; diff --git a/etc/validator.out b/etc/validator.out index d32ed564ec..76f67ca4cf 100644 --- a/etc/validator.out +++ b/etc/validator.out @@ -276,19 +276,12 @@ sites/udmi_site_model/out/devices/AHU-22/state.out "sub_folder" : "update", "sub_type" : "state", "status" : { - "message" : "missing pointset subblock", - "detail" : "state_update: missing pointset subblock", - "category" : "validation.device.schema", + "message" : "Successful validation", + "category" : "validation.device.receive", "timestamp" : "REDACTED_TIMESTAMP", - "level" : 500 + "level" : 200 }, - "errors" : [ { - "message" : "missing pointset subblock", - "detail" : "state_update: missing pointset subblock", - "category" : "validation.device.schema", - "timestamp" : "REDACTED_TIMESTAMP", - "level" : 500 - } ] + "errors" : [ ] } :::::::::::::: sites/udmi_site_model/out/devices/AHU-22/state_gateway.out @@ -667,15 +660,15 @@ sites/udmi_site_model/out/devices/SNS-4/state.out "sub_folder" : "update", "sub_type" : "state", "status" : { - "message" : "missing pointset subblock", - "detail" : "state_update: missing pointset subblock", + "message" : "Device has missing points: split_threshold, triangulating_axis", + "detail" : "state_update: Device has missing points: split_threshold, triangulating_axis", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, "errors" : [ { - "message" : "missing pointset subblock", - "detail" : "state_update: missing pointset subblock", + "message" : "Device has missing points: split_threshold, triangulating_axis", + "detail" : "state_update: Device has missing points: split_threshold, triangulating_axis", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 diff --git a/gencode/docs/events.html b/gencode/docs/events.html index 2815d2a056..c311da1009 100644 --- a/gencode/docs/events.html +++ b/gencode/docs/events.html @@ -6951,6 +6951,61 @@

Must be one of:

+ + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + discovery + + + + registries + + + + Cloud Model + + + + metadata_str
Type: string
+

Strigified version of the metadata object, used for internal backend processing

+ + + + + + +
@@ -7951,6 +8006,61 @@

Must be one of:

+
+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + discovery + + + + devices + + + + Cloud Model + + + + metadata_str
Type: string
+

Strigified version of the metadata object, used for internal backend processing

+ + + + + + +
@@ -12371,6 +12481,54 @@

Must be one of:

+
+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + discovery + + + + cloud_model + + + + metadata_str
Type: string
+

Strigified version of the metadata object, used for internal backend processing

+ + + + + + +
diff --git a/gencode/docs/events_discovery.html b/gencode/docs/events_discovery.html index 5fe270f913..206328f2f7 100644 --- a/gencode/docs/events_discovery.html +++ b/gencode/docs/events_discovery.html @@ -3012,6 +3012,54 @@

Must be one of:

+
+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + registries + + + + Cloud Model + + + + metadata_str
Type: string
+

Strigified version of the metadata object, used for internal backend processing

+ + + + + + +
@@ -3886,6 +3934,54 @@

Must be one of:

+
+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + devices + + + + Cloud Model + + + + metadata_str
Type: string
+

Strigified version of the metadata object, used for internal backend processing

+ + + + + + +
@@ -7788,6 +7884,47 @@

Must be one of:

+
+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + cloud_model + + + + metadata_str
Type: string
+

Strigified version of the metadata object, used for internal backend processing

+ + + + + + +
diff --git a/gencode/docs/metadata.html b/gencode/docs/metadata.html index e275f5305e..6445574582 100644 --- a/gencode/docs/metadata.html +++ b/gencode/docs/metadata.html @@ -784,6 +784,47 @@

Must be one of:

+
+ + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + cloud + + + + metadata_str
Type: string
+

Strigified version of the metadata object, used for internal backend processing

+ + + + + + +
diff --git a/gencode/java/udmi/schema/CloudModel.java b/gencode/java/udmi/schema/CloudModel.java index 88f98c6cca..924b2dfd4c 100644 --- a/gencode/java/udmi/schema/CloudModel.java +++ b/gencode/java/udmi/schema/CloudModel.java @@ -42,6 +42,7 @@ "num_id", "operation", "metadata", + "metadata_str", "device_ids" }) @Generated("jsonschema2pojo") @@ -122,6 +123,13 @@ public class CloudModel { public CloudModel.Operation operation; @JsonProperty("metadata") public Map metadata; + /** + * Strigified version of the metadata object, used for internal backend processing + * + */ + @JsonProperty("metadata_str") + @JsonPropertyDescription("Strigified version of the metadata object, used for internal backend processing") + public java.lang.String metadata_str; /** * If operating on the entire registry, then this manifests as a map of devices not just one device. * @@ -147,6 +155,7 @@ public int hashCode() { result = ((result* 31)+((this.version == null)? 0 :this.version.hashCode())); result = ((result* 31)+((this.blocked == null)? 0 :this.blocked.hashCode())); result = ((result* 31)+((this.last_error_time == null)? 0 :this.last_error_time.hashCode())); + result = ((result* 31)+((this.metadata_str == null)? 0 :this.metadata_str.hashCode())); result = ((result* 31)+((this.detail == null)? 0 :this.detail.hashCode())); result = ((result* 31)+((this.device_ids == null)? 0 :this.device_ids.hashCode())); result = ((result* 31)+((this.config == null)? 0 :this.config.hashCode())); @@ -165,7 +174,7 @@ public boolean equals(Object other) { return false; } CloudModel rhs = ((CloudModel) other); - return (((((((((((((((((((((this.updated_time == rhs.updated_time)||((this.updated_time!= null)&&this.updated_time.equals(rhs.updated_time)))&&((this.auth_type == rhs.auth_type)||((this.auth_type!= null)&&this.auth_type.equals(rhs.auth_type))))&&((this.device_key == rhs.device_key)||((this.device_key!= null)&&this.device_key.equals(rhs.device_key))))&&((this.metadata == rhs.metadata)||((this.metadata!= null)&&this.metadata.equals(rhs.metadata))))&&((this.connection_type == rhs.connection_type)||((this.connection_type!= null)&&this.connection_type.equals(rhs.connection_type))))&&((this.last_event_time == rhs.last_event_time)||((this.last_event_time!= null)&&this.last_event_time.equals(rhs.last_event_time))))&&((this.last_config_time == rhs.last_config_time)||((this.last_config_time!= null)&&this.last_config_time.equals(rhs.last_config_time))))&&((this.credentials == rhs.credentials)||((this.credentials!= null)&&this.credentials.equals(rhs.credentials))))&&((this.last_state_time == rhs.last_state_time)||((this.last_state_time!= null)&&this.last_state_time.equals(rhs.last_state_time))))&&((this.resource_type == rhs.resource_type)||((this.resource_type!= null)&&this.resource_type.equals(rhs.resource_type))))&&((this.num_id == rhs.num_id)||((this.num_id!= null)&&this.num_id.equals(rhs.num_id))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.blocked == rhs.blocked)||((this.blocked!= null)&&this.blocked.equals(rhs.blocked))))&&((this.last_error_time == rhs.last_error_time)||((this.last_error_time!= null)&&this.last_error_time.equals(rhs.last_error_time))))&&((this.detail == rhs.detail)||((this.detail!= null)&&this.detail.equals(rhs.detail))))&&((this.device_ids == rhs.device_ids)||((this.device_ids!= null)&&this.device_ids.equals(rhs.device_ids))))&&((this.config == rhs.config)||((this.config!= null)&&this.config.equals(rhs.config))))&&((this.last_config_ack == rhs.last_config_ack)||((this.last_config_ack!= null)&&this.last_config_ack.equals(rhs.last_config_ack))))&&((this.operation == rhs.operation)||((this.operation!= null)&&this.operation.equals(rhs.operation))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); + return ((((((((((((((((((((((this.updated_time == rhs.updated_time)||((this.updated_time!= null)&&this.updated_time.equals(rhs.updated_time)))&&((this.auth_type == rhs.auth_type)||((this.auth_type!= null)&&this.auth_type.equals(rhs.auth_type))))&&((this.device_key == rhs.device_key)||((this.device_key!= null)&&this.device_key.equals(rhs.device_key))))&&((this.metadata == rhs.metadata)||((this.metadata!= null)&&this.metadata.equals(rhs.metadata))))&&((this.connection_type == rhs.connection_type)||((this.connection_type!= null)&&this.connection_type.equals(rhs.connection_type))))&&((this.last_event_time == rhs.last_event_time)||((this.last_event_time!= null)&&this.last_event_time.equals(rhs.last_event_time))))&&((this.last_config_time == rhs.last_config_time)||((this.last_config_time!= null)&&this.last_config_time.equals(rhs.last_config_time))))&&((this.credentials == rhs.credentials)||((this.credentials!= null)&&this.credentials.equals(rhs.credentials))))&&((this.last_state_time == rhs.last_state_time)||((this.last_state_time!= null)&&this.last_state_time.equals(rhs.last_state_time))))&&((this.resource_type == rhs.resource_type)||((this.resource_type!= null)&&this.resource_type.equals(rhs.resource_type))))&&((this.num_id == rhs.num_id)||((this.num_id!= null)&&this.num_id.equals(rhs.num_id))))&&((this.version == rhs.version)||((this.version!= null)&&this.version.equals(rhs.version))))&&((this.blocked == rhs.blocked)||((this.blocked!= null)&&this.blocked.equals(rhs.blocked))))&&((this.last_error_time == rhs.last_error_time)||((this.last_error_time!= null)&&this.last_error_time.equals(rhs.last_error_time))))&&((this.metadata_str == rhs.metadata_str)||((this.metadata_str!= null)&&this.metadata_str.equals(rhs.metadata_str))))&&((this.detail == rhs.detail)||((this.detail!= null)&&this.detail.equals(rhs.detail))))&&((this.device_ids == rhs.device_ids)||((this.device_ids!= null)&&this.device_ids.equals(rhs.device_ids))))&&((this.config == rhs.config)||((this.config!= null)&&this.config.equals(rhs.config))))&&((this.last_config_ack == rhs.last_config_ack)||((this.last_config_ack!= null)&&this.last_config_ack.equals(rhs.last_config_ack))))&&((this.operation == rhs.operation)||((this.operation!= null)&&this.operation.equals(rhs.operation))))&&((this.timestamp == rhs.timestamp)||((this.timestamp!= null)&&this.timestamp.equals(rhs.timestamp)))); } diff --git a/gencode/python/udmi/schema/model_cloud.py b/gencode/python/udmi/schema/model_cloud.py index 84d8c7c92b..1b384c5ca9 100644 --- a/gencode/python/udmi/schema/model_cloud.py +++ b/gencode/python/udmi/schema/model_cloud.py @@ -59,6 +59,7 @@ def __init__(self): self.num_id = None self.operation = None self.metadata = None + self.metadata_str = None self.device_ids = None @staticmethod @@ -85,6 +86,7 @@ def from_dict(source): result.num_id = source.get('num_id') result.operation = source.get('operation') result.metadata = source.get('metadata') + result.metadata_str = source.get('metadata_str') result.device_ids = Object18ECC5EE.map_from(source.get('device_ids')) return result @@ -144,6 +146,8 @@ def to_dict(self): result['operation'] = self.operation # 5 if self.metadata: result['metadata'] = self.metadata # 1 + if self.metadata_str: + result['metadata_str'] = self.metadata_str # 5 if self.device_ids: result['device_ids'] = Object18ECC5EE.expand_dict(self.device_ids) # 2 return result diff --git a/pubber/src/main/java/daq/pubber/MqttPublisher.java b/pubber/src/main/java/daq/pubber/MqttPublisher.java index 602f7a3671..9cab7caf7b 100644 --- a/pubber/src/main/java/daq/pubber/MqttPublisher.java +++ b/pubber/src/main/java/daq/pubber/MqttPublisher.java @@ -2,6 +2,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import static com.google.udmi.util.Common.SEC_TO_MS; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; import static com.google.udmi.util.GeneralUtils.ifTrueThen; import static com.google.udmi.util.GeneralUtils.isTrue; @@ -296,8 +297,8 @@ private void validateCloudIotOptions() { private MqttClient newProxyClient(String deviceId) { String gatewayId = getGatewayId(deviceId); - debug(format("Connecting device %s through gateway %s", deviceId, gatewayId)); - final MqttClient mqttClient = getConnectedClient(gatewayId); + info(format("Connecting device %s through gateway %s", deviceId, gatewayId)); + final MqttClient mqttClient = getConnectedClient(gatewayId, true); long timeToWait = mqttClient.getTimeToWait(); try { startupLatchWait(connectionLatch, "gateway startup exchange"); @@ -487,9 +488,11 @@ public void registerHandler(String deviceId, String mqttSuffix, String mqttTopic = getMessageTopic(deviceId, mqttSuffix); String handlerKey = getHandlerKey(mqttTopic); if (handler == null) { + info(format("Removing handler %s", handlerKey)); handlers.remove(handlerKey); handlersType.remove(handlerKey); } else if (handlers.put(handlerKey, (Consumer) handler) == null) { + info(format("Registered handler for %s as %s", handlerKey, messageType.getSimpleName())); handlersType.put(handlerKey, (Class) messageType); } else { throw new IllegalStateException("Overwriting existing handler " + handlerKey); @@ -516,7 +519,7 @@ private String getDeviceId(String topic) { public void connect(String targetId, boolean clean) { ifTrueThen(clean, () -> closeMqttClient(targetId)); - getConnectedClient(targetId); + getConnectedClient(targetId, true); } private void success(String message, String deviceId, String type, String phase) { @@ -550,12 +553,12 @@ private void sendMessage(String deviceId, String mqttTopic, private MqttClient getActiveClient(String targetId) { while (true) { checkAuthentication(targetId); - MqttClient connectedClient = getConnectedClient(targetId); - if (connectedClient.isConnected()) { + MqttClient connectedClient = getConnectedClient(targetId, false); + if (connectedClient != null && connectedClient.isConnected()) { return connectedClient; } - info("Client not active, deferring message..."); - safeSleep(DEFAULT_CONFIG_WAIT_SEC); + info(format("Client %s not active, deferring message...", targetId)); + safeSleep(DEFAULT_CONFIG_WAIT_SEC * SEC_TO_MS); } } @@ -586,10 +589,13 @@ private void checkAuthentication(String targetId) { } } - private MqttClient getConnectedClient(String deviceId) { + private MqttClient getConnectedClient(String deviceId, boolean proxyActiveOnly) { try { synchronized (mqttClients) { if (isProxyDevice(deviceId)) { + if (!proxyActiveOnly && !mqttClients.containsKey(deviceId)) { + return null; + } return mqttClients.computeIfAbsent(deviceId, this::newProxyClient); } return mqttClients.computeIfAbsent(deviceId, this::newDirectClient); @@ -704,7 +710,7 @@ private void messageArrivedCore(String topic, MqttMessage message) { Consumer handler = handlers.get(handlerKey); Class type = handlersType.get(handlerKey); if (handler == null) { - error("Missing handler", deviceId, messageType, "receive", + error("Missing handler " + handlerKey, deviceId, messageType, "receive", new RuntimeException("No registered handler for topic " + topic)); handlersType.put(handlerKey, Object.class); handlers.put(handlerKey, this::ignoringHandler); diff --git a/pubber/src/main/java/daq/pubber/ProxyDevice.java b/pubber/src/main/java/daq/pubber/ProxyDevice.java index 60195359f7..961383e772 100644 --- a/pubber/src/main/java/daq/pubber/ProxyDevice.java +++ b/pubber/src/main/java/daq/pubber/ProxyDevice.java @@ -45,12 +45,12 @@ private static PubberConfiguration makeProxyConfiguration(ManagerHost host, Stri protected void activate() { try { active.set(false); + info("Activating proxy device " + deviceId); MqttDevice mqttDevice = pubberHost.getMqttDevice(deviceId); mqttDevice.registerHandler(MqttDevice.CONFIG_TOPIC, this::configHandler, Config.class); mqttDevice.connect(deviceId); deviceManager.activate(); active.set(true); - info("Activated proxy device " + deviceId); } catch (Exception e) { error(format("Could not connect proxy device %s: %s", deviceId, friendlyStackTrace(e))); } diff --git a/schema/model_cloud.json b/schema/model_cloud.json index ec0790215a..b0d72cc90a 100644 --- a/schema/model_cloud.json +++ b/schema/model_cloud.json @@ -124,6 +124,10 @@ } } }, + "metadata_str": { + "type": "string", + "description": "Strigified version of the metadata object, used for internal backend processing" + }, "device_ids": { "description": "If operating on the entire registry, then this manifests as a map of devices not just one device.", "existingJavaType": "java.util.Map", diff --git a/udmis/etc/local_pod.json b/udmis/etc/local_pod.json index a27990aee0..1d6edace3c 100644 --- a/udmis/etc/local_pod.json +++ b/udmis/etc/local_pod.json @@ -25,6 +25,12 @@ "control": { "recv_id": "c/control/#" }, + "provision": { + "recv_id": "events/discovery", + "send_id": "events", + "side_id": "c/control", + "enabled": "yes" + }, "distributor": { "enabled": "false" } diff --git a/udmis/etc/prod_pod.json b/udmis/etc/prod_pod.json index 66faf5e987..080bd7e973 100644 --- a/udmis/etc/prod_pod.json +++ b/udmis/etc/prod_pod.json @@ -53,7 +53,7 @@ "recv_id": "${UDMI_PREFIX}udmi_target-discovery", "send_id": "${UDMI_PREFIX}udmi_target", "side_id": "${UDMI_PREFIX}udmi_control", - "enabled": "${BITBOX_DISCOVERY}" + "enabled": "true" }, "bitbox": { "recv_id": "${BITBOX_DISCOVERY}", diff --git a/udmis/src/main/java/com/google/bos/udmi/service/access/ImplicitIotAccessProvider.java b/udmis/src/main/java/com/google/bos/udmi/service/access/ImplicitIotAccessProvider.java index a3b27b15e2..81c10bc681 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/access/ImplicitIotAccessProvider.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/access/ImplicitIotAccessProvider.java @@ -3,14 +3,17 @@ import static com.google.bos.udmi.service.messaging.MessageDispatcher.rawString; import static com.google.common.base.Preconditions.checkState; import static com.google.udmi.util.Common.DEFAULT_REGION; +import static com.google.udmi.util.GeneralUtils.CSV_JOINER; import static com.google.udmi.util.GeneralUtils.booleanString; import static com.google.udmi.util.GeneralUtils.friendlyStackTrace; +import static com.google.udmi.util.GeneralUtils.ifNotNullGet; import static com.google.udmi.util.GeneralUtils.ifNotNullThen; import static com.google.udmi.util.GeneralUtils.ifNullThen; import static com.google.udmi.util.GeneralUtils.ifTrueThen; import static com.google.udmi.util.GeneralUtils.isNullOrNotEmpty; import static com.google.udmi.util.JsonUtil.asMap; import static com.google.udmi.util.JsonUtil.isoConvert; +import static com.google.udmi.util.JsonUtil.stringify; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.Optional.ofNullable; @@ -27,12 +30,14 @@ import com.google.bos.udmi.service.support.DataRef; import com.google.bos.udmi.service.support.IotDataProvider; import com.google.bos.udmi.service.support.MosquittoBroker; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.JsonUtil; import java.util.AbstractMap.SimpleEntry; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -60,7 +65,8 @@ public class ImplicitIotAccessProvider extends IotAccessBase { private static final String CONFIG_VER_KEY = "config_ver"; private static final String LAST_CONFIG_KEY = "last_config"; private static final String LAST_STATE_KEY = "last_state"; - private static final String DEVICES_COLLECTION = "devices"; + private static final String DEVICES_ACTIVE = "active"; + private static final String BOUND_TO_KEY = "bound_to"; private static final String BLOCKED_PROPERTY = "blocked"; private static final String CREATED_AT_PROPERTY = "created_at"; private static final String REGISTRIES_KEY = "registries"; @@ -71,12 +77,14 @@ public class ImplicitIotAccessProvider extends IotAccessBase { private static final String AUTH_PASSWORD_PROPERTY = "auth_pass"; private static final String LAST_CONFIG_ACKED = "last_config_ack"; private static final String CONFIG_SUFFIX = "/config"; + private static final String METADATA_STR_KEY = "metadata_str"; + private static final String RESOURCE_TYPE_PROPERTY = "resource_type"; private final boolean enabled; private final ConnectionBroker broker = new MosquittoBroker(this); private final Future connLogger; private IotDataProvider database; private ReflectProcessor reflect; - private Map configPublished = new ConcurrentHashMap<>(); + private final Map configPublished = new ConcurrentHashMap<>(); /** * Create an access provider with implicit internal resources. @@ -94,8 +102,10 @@ private static String hashedDeviceId(String registryId, String deviceId) { return String.valueOf(Math.abs(Objects.hash(registryId, deviceId))); } - private void bindDevice(String registryId, String deviceId, CloudModel model) { - // Binding is a nop, so silently do nothing! + private void bindDevicesToGateway(String registryId, String gatewayId, CloudModel cloudModel) { + Set deviceIds = cloudModel.device_ids.keySet(); + deviceIds.forEach( + deviceId -> registryDeviceRef(registryId, deviceId).put(BOUND_TO_KEY, gatewayId)); } private void blockDevice(String registryId, String deviceId, CloudModel cloudModel) { @@ -144,7 +154,7 @@ private void createDevice(String registryId, String deviceId, CloudModel cloudMo private void deleteDevice(String registryId, String deviceId, CloudModel cloudModel) { DataRef properties = registryDeviceRef(registryId, deviceId); properties.entries().keySet().forEach(properties::delete); - registryDevicesCollection(registryId).delete(deviceId); + registryDevicesRef(registryId).delete(deviceId); broker.authorize(clientId(registryId, deviceId), null); } @@ -173,9 +183,8 @@ private DataRef registryDeviceRef(String registryId, String deviceId) { return database.ref().registry(registryId).device(deviceId); } - private DataRef registryDevicesCollection(String registryId) { - return database.ref().registry(registryId).collection( - DEVICES_COLLECTION); + private DataRef registryDevicesRef(String registryId) { + return database.ref().registry(registryId).collection(DEVICES_ACTIVE); } private void sendConfigUpdate(String registryId, String deviceId, String config) { @@ -190,6 +199,8 @@ private void sendConfigUpdate(String registryId, String deviceId, String config) private Map toDeviceMap(CloudModel cloudModel, String createdAt) { Map properties = new HashMap<>(); ifNotNullThen(createdAt, x -> properties.put(CREATED_AT_PROPERTY, createdAt)); + properties.put(RESOURCE_TYPE_PROPERTY, + ofNullable(cloudModel.resource_type).orElse(DEVICE).toString()); properties.put(BLOCKED_PROPERTY, booleanString(cloudModel.blocked)); ifNotNullThen(cloudModel.num_id, id -> properties.put(NUM_ID_PROPERTY, id)); ifTrueThen(!cloudModel.credentials.isEmpty(), () -> { @@ -204,7 +215,7 @@ private Map toDeviceMap(CloudModel cloudModel, String createdAt) private String touchDeviceEntry(String registryId, String deviceId) { String timestamp = isoConvert(); - registryDevicesCollection(registryId).put(deviceId, timestamp); + registryDevicesRef(registryId).put(deviceId, timestamp); return timestamp; } @@ -240,7 +251,15 @@ public Entry fetchConfig(String registryId, String deviceId) { public CloudModel fetchDevice(String registryId, String deviceId) { touchDeviceEntry(registryId, deviceId); Map properties = registryDeviceRef(registryId, deviceId).entries(); - return JsonUtil.convertTo(CloudModel.class, properties); + if (properties == null) { + return null; + } + CloudModel cloudModel = requireNonNull(JsonUtil.convertTo(CloudModel.class, properties)); + cloudModel.metadata = ifNotNullGet(cloudModel.metadata_str, JsonUtil::toStringMapStr); + cloudModel.metadata_str = null; + + cloudModel.device_ids = listBoundDevices(registryId, deviceId); + return cloudModel; } @Override @@ -273,7 +292,7 @@ public boolean isEnabled() { @Override public CloudModel listDevices(String registryId, Consumer progress) { - Map entries = registryDevicesCollection(registryId).entries(); + Map entries = registryDevicesRef(registryId).entries(); ifNotNullThen(progress, p -> p.accept(entries.size())); CloudModel cloudModel = new CloudModel(); cloudModel.device_ids = entries.keySet().stream().collect( @@ -281,6 +300,19 @@ public CloudModel listDevices(String registryId, Consumer progress) { return cloudModel; } + private Map listBoundDevices(String registryId, String gatewayId) { + Set deviceIds = registryDevicesRef(registryId).entries().keySet(); + Map devices = deviceIds.stream().filter(deviceId -> { + String boundTo = registryDeviceRef(registryId, deviceId).get(BOUND_TO_KEY); + return gatewayId.equals(boundTo); + }).collect(Collectors.toMap(id -> id, id -> fetchDevice(registryId, id))); + List gateways = devices.values().stream() + .filter(model -> GATEWAY.equals(model.resource_type)).toList(); + checkState(gateways.isEmpty(), + format("Gateways found in gateway lookup of %s: %s", gatewayId, CSV_JOINER.join(gateways))); + return devices; + } + @Override public CloudModel modelDevice(String registryId, String deviceId, CloudModel cloudModel) { Operation operation = cloudModel.operation; @@ -294,7 +326,7 @@ public CloudModel modelDevice(String registryId, String deviceId, CloudModel clo case UPDATE -> updateDevice(registryId, deviceId, cloudModel); case MODIFY -> modifyDevice(registryId, deviceId, cloudModel); case DELETE -> deleteDevice(registryId, deviceId, cloudModel); - case BIND -> bindDevice(registryId, deviceId, cloudModel); + case BIND -> bindDevicesToGateway(registryId, deviceId, cloudModel); case BLOCK -> blockDevice(registryId, deviceId, cloudModel); default -> throw new RuntimeException("Unknown device operation " + operation); } @@ -315,8 +347,11 @@ public CloudModel modelRegistry(String registryId, String deviceId, CloudModel c } } - public CloudModel modifyDevice(String registryId, String deviceId, CloudModel cloudModel) { - throw new RuntimeException("modifyDevice not yet implemented"); + private void modifyDevice(String registryId, String deviceId, CloudModel cloudModel) { + CloudModel fetchedModel = fetchDevice(registryId, deviceId); + Map metadataMap = ofNullable(fetchedModel.metadata).orElseGet(HashMap::new); + metadataMap.putAll(cloudModel.metadata); + mungeDevice(registryId, deviceId, ImmutableMap.of(METADATA_STR_KEY, stringify(metadataMap))); } @Override diff --git a/udmis/src/main/java/com/google/bos/udmi/service/core/ProvisioningEngine.java b/udmis/src/main/java/com/google/bos/udmi/service/core/ProvisioningEngine.java index 2c82fe97c9..911dc977f7 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/core/ProvisioningEngine.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/core/ProvisioningEngine.java @@ -7,13 +7,12 @@ import static com.google.udmi.util.GeneralUtils.ifNullThen; import static com.google.udmi.util.GeneralUtils.ifTrueGet; import static com.google.udmi.util.GeneralUtils.ignoreValue; -import static com.google.udmi.util.JsonUtil.getDate; import static com.google.udmi.util.JsonUtil.isoConvert; import static com.google.udmi.util.JsonUtil.stringifyTerse; import static com.google.udmi.util.MetadataMapKeys.UDMI_DISCOVERED_FROM; import static com.google.udmi.util.MetadataMapKeys.UDMI_DISCOVERED_WITH; import static com.google.udmi.util.MetadataMapKeys.UDMI_GENERATION; -import static com.google.udmi.util.MetadataMapKeys.UDMI_PROVISION_GENERATION; +import static com.google.udmi.util.MetadataMapKeys.UDMI_PROVISION_ENABLE; import static com.google.udmi.util.MetadataMapKeys.UDMI_UPDATED; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -36,7 +35,8 @@ @ComponentName("provision") public class ProvisioningEngine extends ProcessorBase { - private static final String EXPECTED_DEVICE_FORMAT = "%s-%s"; + private static final String DISCOVERED_DEVICE_FORMAT = "discovered_%s-%s"; + private static final String GATEWAY_KEY_FORMAT = "%s-%s"; private final Map scanAgent = new ConcurrentHashMap<>(); @@ -67,14 +67,14 @@ private void createDeviceEntry(String registryId, String expectedId, String gate bindDeviceToGateway(registryId, expectedId, gatewayId); } - private CloudModel getCloudModel(String deviceRegistryId, String gatewayId) { - String gatewayKey = format(EXPECTED_DEVICE_FORMAT, deviceRegistryId, gatewayId); + private CloudModel getCachedModel(String deviceRegistryId, String gatewayId) { + String gatewayKey = format(GATEWAY_KEY_FORMAT, deviceRegistryId, gatewayId); return scanAgent.computeIfAbsent(gatewayKey, key -> new CloudModel()); } private synchronized Map refreshModelDevices(String deviceRegistryId, String gatewayId, Date generation) { - CloudModel cloudModel = getCloudModel(deviceRegistryId, gatewayId); + CloudModel cloudModel = getCachedModel(deviceRegistryId, gatewayId); if (!generation.equals(cloudModel.timestamp)) { cloudModel.timestamp = generation; cloudModel.device_ids = null; @@ -97,9 +97,7 @@ private synchronized Map refreshModelDevices(String deviceRe } private boolean shouldProvision(Date generation, CloudModel cloudModel) { - Date provisioningGeneration = getDate( - ifNotNullGet(cloudModel.metadata, m -> m.get(UDMI_PROVISION_GENERATION))); - return generation.equals(provisioningGeneration); + return TRUE_OPTION.equals(ifNotNullGet(cloudModel.metadata, m -> m.get(UDMI_PROVISION_ENABLE))); } /** @@ -123,7 +121,7 @@ public void discoveryEvent(DiscoveryEvents discoveryEvent) { } String family = requireNonNull(discoveryEvent.scan_family, "missing scan_family"); String addr = requireNonNull(discoveryEvent.scan_addr, "missing scan_addr"); - String expectedId = format(EXPECTED_DEVICE_FORMAT, family, addr); + String expectedId = format(DISCOVERED_DEVICE_FORMAT, family, addr); if (deviceIds.containsKey(expectedId)) { debug("Scan device %s/%s target %s already registered", registryId, gatewayId, expectedId); } else { diff --git a/udmis/src/test/java/com/google/bos/udmi/service/core/BitboxAdapterTest.java b/udmis/src/test/java/com/google/bos/udmi/service/core/BitboxAdapterTest.java index 64e9cab9cd..1cab600c77 100644 --- a/udmis/src/test/java/com/google/bos/udmi/service/core/BitboxAdapterTest.java +++ b/udmis/src/test/java/com/google/bos/udmi/service/core/BitboxAdapterTest.java @@ -33,7 +33,7 @@ public class BitboxAdapterTest extends ProcessorTestBase { protected void initializeTestInstance() { initializeTestInstance(BitboxAdapter.class); - ProvisioningEngineTest.initializeProvider(provider); + ProvisioningEngineTest.initializeProvider(provider, false); } private Envelope getLegacyEnvelope() { diff --git a/udmis/src/test/java/com/google/bos/udmi/service/core/ProvisioningEngineTest.java b/udmis/src/test/java/com/google/bos/udmi/service/core/ProvisioningEngineTest.java index 2af9004a54..b09bf3ae4e 100644 --- a/udmis/src/test/java/com/google/bos/udmi/service/core/ProvisioningEngineTest.java +++ b/udmis/src/test/java/com/google/bos/udmi/service/core/ProvisioningEngineTest.java @@ -1,6 +1,7 @@ package com.google.bos.udmi.service.core; -import static com.google.udmi.util.JsonUtil.isoConvert; +import static com.google.udmi.util.GeneralUtils.ifTrueThen; +import static com.google.udmi.util.MetadataMapKeys.UDMI_PROVISION_ENABLE; import static com.google.udmi.util.MetadataMapKeys.UDMI_PROVISION_GENERATION; import static java.lang.String.format; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -17,8 +18,8 @@ import com.google.bos.udmi.service.access.IotAccessBase; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.udmi.util.JsonUtil; import daq.pubber.ProtocolFamily; -import java.time.Duration; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -40,11 +41,13 @@ public class ProvisioningEngineTest extends ProcessorTestBase { private static final String SCAN_ADDR = "19273821"; private static final String SCAN_FAMILY = ProtocolFamily.VENDOR; private static final String TARGET_DEVICE = format("%s-%s", SCAN_FAMILY, SCAN_ADDR); + private static final String DISCOVERED_DEVICE = "discovered_" + TARGET_DEVICE; private static final Date SCAN_GENERATION = new Date(); - private static final Duration PROVISIONING_WINDOW = Duration.ofMinutes(5); private static Map getGatewayMetadata() { - return ImmutableMap.of(UDMI_PROVISION_GENERATION, isoConvert(SCAN_GENERATION)); + return ImmutableMap.of( + UDMI_PROVISION_ENABLE, "true", + UDMI_PROVISION_GENERATION, JsonUtil.isoConvert()); } @NotNull @@ -55,20 +58,27 @@ private static Envelope getScanEnvelope() { return envelope; } - static void initializeProvider(IotAccessBase provider) { + static void initializeProvider(IotAccessBase provider, boolean alreadyProvisioned) { CloudModel registryModel = new CloudModel(); registryModel.device_ids = new HashMap<>(); CloudModel deviceModel = new CloudModel(); - registryModel.device_ids.put(TARGET_DEVICE, deviceModel); - registryModel.device_ids.put(TEST_DEVICE, deviceModel); deviceModel.resource_type = Resource_type.DEVICE; + registryModel.device_ids.put(TEST_DEVICE, deviceModel); + + if (alreadyProvisioned) { + CloudModel provisionedModel = new CloudModel(); + provisionedModel.resource_type = Resource_type.DEVICE; + registryModel.device_ids.put(DISCOVERED_DEVICE, provisionedModel); + } CloudModel gatewayModel = new CloudModel(); registryModel.device_ids.put(TEST_GATEWAY, gatewayModel); gatewayModel.resource_type = Resource_type.GATEWAY; gatewayModel.device_ids = new HashMap<>(); gatewayModel.device_ids.put(TEST_DEVICE, new CloudModel()); + ifTrueThen(alreadyProvisioned, () -> + gatewayModel.device_ids.put(DISCOVERED_DEVICE, new CloudModel())); gatewayModel.metadata = getGatewayMetadata(); when(provider.getRegistries()).thenReturn(ImmutableSet.of(TEST_REGISTRY)); @@ -85,10 +95,10 @@ static void initializeProvider(IotAccessBase provider) { gatewayModel); } - protected void initializeTestInstance() { + protected void initializeTestInstance(boolean alreadyProvisioned) { initializeTestInstance(ProvisioningEngine.class); - initializeProvider(provider); + initializeProvider(provider, alreadyProvisioned); } private DiscoveryEvents getDiscoveryScanEvent(String targetDeviceId) { @@ -102,7 +112,7 @@ private DiscoveryEvents getDiscoveryScanEvent(String targetDeviceId) { @Test public void discoveryEventCreate() { - initializeTestInstance(); + initializeTestInstance(false); getReverseDispatcher() .withEnvelope(getScanEnvelope()) .publish(getDiscoveryScanEvent(TARGET_DEVICE)); @@ -117,21 +127,22 @@ public void discoveryEventCreate() { List devices = deviceCaptor.getAllValues(); List models = modelCaptor.getAllValues(); - assertEquals(TARGET_DEVICE, devices.get(0), "created device id"); + assertEquals(DISCOVERED_DEVICE, devices.get(0), "created device id"); assertEquals(Operation.CREATE, models.get(0).operation, "operation mismatch"); assertTrue(models.get(0).blocked, "device blocked"); assertEquals(TEST_GATEWAY, devices.get(1), "scanning gateway id"); assertEquals(Operation.BIND, models.get(1).operation, "operation mismatch"); - assertNotNull(models.get(1).device_ids.get(TARGET_DEVICE), "binding device entry"); + assertNotNull(models.get(1).device_ids.get(DISCOVERED_DEVICE), "binding device entry"); } @Test public void discoveryEventExisting() { - initializeTestInstance(); + initializeTestInstance(true); getReverseDispatcher() .withEnvelope(getScanEnvelope()) - .publish(getDiscoveryScanEvent(TEST_DEVICE)); + .publish(getDiscoveryScanEvent(TARGET_DEVICE)); + terminateAndWait(); verify(provider, times(1)).fetchDevice(eq(TEST_REGISTRY), eq(TEST_GATEWAY)); verify(provider, never()).modelDevice(eq(TEST_REGISTRY), any(), any()); diff --git a/validator/src/main/java/com/google/daq/mqtt/mapping/MappingAgent.java b/validator/src/main/java/com/google/daq/mqtt/mapping/MappingAgent.java index 708991a8f8..377e2e1d0e 100644 --- a/validator/src/main/java/com/google/daq/mqtt/mapping/MappingAgent.java +++ b/validator/src/main/java/com/google/daq/mqtt/mapping/MappingAgent.java @@ -8,6 +8,7 @@ import static com.google.udmi.util.JsonUtil.loadFileStrictRequired; import static com.google.udmi.util.JsonUtil.stringify; import static com.google.udmi.util.JsonUtil.writeFile; +import static com.google.udmi.util.MetadataMapKeys.UDMI_PROVISION_ENABLE; import static com.google.udmi.util.MetadataMapKeys.UDMI_PROVISION_GENERATION; import static com.google.udmi.util.SiteModel.METADATA_JSON; import static java.util.Objects.requireNonNull; @@ -87,6 +88,7 @@ void process(List argsList) { while (!argsList.isEmpty()) { String mappingCommand = removeNextArg(argsList, "mapping command"); switch (mappingCommand) { + case "provision" -> setupProvision(); case "discover" -> initiateDiscover(); case "reconcile" -> reconcileDiscovery(); default -> throw new RuntimeException("Unknown mapping command " + mappingCommand); @@ -94,6 +96,12 @@ void process(List argsList) { } } + private void setupProvision() { + CloudModel cloudModel = new CloudModel(); + cloudModel.metadata = ImmutableMap.of(UDMI_PROVISION_ENABLE, "true"); + cloudIotManager.modifyDevice(deviceId, cloudModel); + } + private void initiateDiscover() { String generation = isoConvert(new Date()); System.err.printf("Initiating discovery on %s/%s at %s%n", siteModel.getRegistryId(), deviceId, diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index ec5443677b..47835ec5d1 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -102,12 +102,16 @@ class LocalDevice { private static final String RSA_CERT_PEM = "rsa_cert.pem"; private static final String RSA_PRIVATE_PEM = "rsa_private.pem"; private static final String RSA_PRIVATE_PKCS8 = "rsa_private.pkcs8"; + private static final String RSA_PRIVATE_CRT = "rsa_private.crt"; + private static final String RSA_PRIVATE_CSR = "rsa_private.csr"; private static final String ES_PUBLIC_PEM = "ec_public.pem"; private static final String ES2_PUBLIC_PEM = "ec2_public.pem"; private static final String ES3_PUBLIC_PEM = "ec3_public.pem"; private static final String ES_CERT_PEM = "ec_cert.pem"; private static final String ES_PRIVATE_PEM = "ec_private.pem"; private static final String ES_PRIVATE_PKCS8 = "ec_private.pkcs8"; + private static final String EC_PRIVATE_CRT = "ec_private.crt"; + private static final String EC_PRIVATE_CSR = "ec_private.csr"; private static final String RSA_AUTH_TYPE = Auth_type.RS_256.toString(); private static final String RSA_CERT_TYPE = Auth_type.RS_256_X_509.toString(); private static final String ES_AUTH_TYPE = Auth_type.ES_256.toString(); @@ -147,8 +151,12 @@ class LocalDevice { ES_CERT_TYPE, ES_CERT_PEM); private static final Set OPTIONAL_FILES = ImmutableSet.of( + RSA_PRIVATE_CRT, + RSA_PRIVATE_CSR, RSA2_PUBLIC_PEM, RSA3_PUBLIC_PEM, + EC_PRIVATE_CRT, + EC_PRIVATE_CSR, ES2_PUBLIC_PEM, ES3_PUBLIC_PEM, SAMPLES_DIR,