From 71f4e301ec8590cf0ff9537e92d7fb9bcf2b1f1c Mon Sep 17 00:00:00 2001 From: marcel Date: Tue, 25 Jun 2024 16:08:44 +0200 Subject: [PATCH] Add subcomponent/subdevice processing Up to now the IFF-agent can only manage one single device with a certain id. This limits cases where the device is consisting of several subsystems. For such cases, all the subsystem data was mapped to the main device with the respective id. With these changes, a device can now consist of several subsystems and these IDs can be added to the device token. This PR contains everything needed to support subdevice/subcomponent processing: * IFF-Agent accepts deviceIds in the TCP/UDP messages * IFF-Agent utils offer additional options to add subcomponent IDs and send data for subcompoentns * Keycloak allows now the field "subdevice_ids" in the token to add subdevice IDs * The MQTT-Bridge permits subdevice IDs to stream data Related Epic: #514 Related User-stories: #555 Signed-off-by: marcel --- KafkaBridge/lib/authService/acl.js | 19 +- KafkaBridge/lib/authService/authenticate.js | 25 ++- KafkaBridge/lib/cache/index.js | 24 +++ .../mqttBridge/sparkplug_data_ingestion.js | 3 + KafkaBridge/test/lib_authServiceTest.js | 35 +++- KafkaBridge/test/lib_cacheTest.js | 36 ++++ .../META-INF/keycloak-scripts.json | 5 + .../iff-js-providers/subdeviceids-mapper.js | 77 ++++++++ NgsildAgent/README.md | 24 ++- NgsildAgent/lib/CloudProxy.js | 14 +- NgsildAgent/lib/ConnectionManager.js | 6 +- NgsildAgent/lib/SparkplugbConnector.js | 57 ++++-- NgsildAgent/lib/schemas/data.json | 3 + NgsildAgent/util/activate.sh | 5 +- NgsildAgent/util/init-device.sh | 61 ++++-- NgsildAgent/util/send_data.sh | 15 +- helm/charts/emqx/templates/emxq.yaml | 2 + .../keycloak/templates/keycloak-realm.yaml | 23 +++ test/bats/lib/config.bash | 3 +- test/bats/lib/mqtt.bash | 22 +-- .../test-device-agent.bats | 111 +++++++++-- .../test-device-authorization.bats | 176 +++++++++++++++++- 22 files changed, 644 insertions(+), 102 deletions(-) create mode 100644 Keycloak/iff-js-providers/subdeviceids-mapper.js diff --git a/KafkaBridge/lib/authService/acl.js b/KafkaBridge/lib/authService/acl.js index e6b0c03e..0c03ce97 100644 --- a/KafkaBridge/lib/authService/acl.js +++ b/KafkaBridge/lib/authService/acl.js @@ -38,6 +38,7 @@ class Acl { return; } const topic = req.query.topic; + const clientid = req.query.clientid; this.logger.debug('ACL request for username ' + username + ' and topic ' + topic); // allow all $SYS topics if (topic.startsWith('$SYS/')) { @@ -51,18 +52,26 @@ class Acl { const splitTopic = topic.split('/'); if (splitTopic[0] === 'spBv1.0') { const spBAccountId = splitTopic[1]; + const gateway = splitTopic[3]; + const command = splitTopic[2]; const spBdevId = splitTopic[4]; const spBAclKey = spBAccountId + '/' + spBdevId; - const allowed = await this.cache.getValue(spBAclKey, 'acl'); - if (allowed === undefined || !(allowed === 'true') || spBdevId !== username) { + let allowed = await this.cache.getValue(spBAclKey, 'acl'); + if (allowed === undefined && spBdevId === '' && command === 'NBIRTH') { // if it is a NBIRTH command check if gatewayid is permitted for this session + allowed = await this.cache.getValue(spBAccountId + '/' + gateway, 'acl'); + if (allowed === undefined) { + this.logger.warn('Gateway id not permitted for this token/session. Use a token which has device_id==gateway_id.'); + } + } + if (allowed === undefined || allowed !== clientid) { this.logger.info('Connection rejected for realm ' + spBAccountId + ' and device ' + spBdevId); - res.sendStatus(400); + return res.status(200).json({ result: 'deny' }); } else { - res.status(200).json({ result: 'allow' }); + return res.status(200).json({ result: 'allow' }); } } else { this.logger.warn('Topic sructure not valid.'); - res.sendStatus(400); + return res.status(200).json({ result: 'deny' }); } } } diff --git a/KafkaBridge/lib/authService/authenticate.js b/KafkaBridge/lib/authService/authenticate.js index 7b8e3ea9..68930bdf 100644 --- a/KafkaBridge/lib/authService/authenticate.js +++ b/KafkaBridge/lib/authService/authenticate.js @@ -53,11 +53,22 @@ class Authenticate { this.cache.init(); } + async addSubdeviceAcl (realm, clientid, decodedToken) { + if ('subdevice_ids' in decodedToken) { + const subdevices = decodedToken.subdevice_ids; + const parsedSubdevices = JSON.parse(subdevices); + for (const did of parsedSubdevices) { + await this.cache.setValue(realm + '/' + did, 'acl', clientid); + } + } + } + // expects "username" and "password" as url-query-parameters async authenticate (req, res) { this.logger.debug('Auth request ' + JSON.stringify(req.query)); const username = req.query.username; const token = req.query.password; + const clientid = req.query.clientid; if (username === this.config.mqtt.adminUsername) { if (token === this.config.mqtt.adminPassword) { // superuser @@ -67,7 +78,7 @@ class Authenticate { } else { // will also kick out tokens who use the superuser name as deviceId this.logger.warn('Wrong Superuser password.'); - res.sendStatus(400); + res.status(200).json({ result: 'deny' }); return; } } @@ -75,12 +86,12 @@ class Authenticate { this.logger.debug('token decoded: ' + JSON.stringify(decodedToken)); if (decodedToken === null) { this.logger.info('Could not decode token.'); - res.sendStatus(400); + res.status(200).json({ result: 'deny' }); return; } if (!validate(decodedToken, username)) { this.logger.warn('Validation of token failed. Username: ' + username); - res.sendStatus(400); + res.status(200).json({ result: 'deny' }); return; } // check whether accounts contains only one element and role is device @@ -89,15 +100,17 @@ class Authenticate { const realm = getRealm(decodedToken); if (did === null || did === undefined || realm === null || realm === undefined) { this.logger.warn('Validation failed: Device id or realm not valid.'); - res.sendStatus(400); + res.status(200).json({ result: 'deny' }); return; } if (did === this.config.mqtt.tainted || gateway === this.config.mqtt.tainted) { this.logger.warn('This token is tained! Rejecting.'); - res.sendStatus(400); + res.status(200).json({ result: 'deny' }); } // put realm/device into the list of accepted topics - await this.cache.setValue(realm + '/' + did, 'acl', 'true'); + await this.cache.deleteKeysWithValue('acl', clientid); + await this.addSubdeviceAcl(realm, clientid, decodedToken); + await this.cache.setValue(realm + '/' + did, 'acl', clientid); res.status(200).json({ result: 'allow', is_superuser: 'false' }); } diff --git a/KafkaBridge/lib/cache/index.js b/KafkaBridge/lib/cache/index.js index bef963c2..af52eea2 100644 --- a/KafkaBridge/lib/cache/index.js +++ b/KafkaBridge/lib/cache/index.js @@ -43,5 +43,29 @@ class Cache { const obj = await this.redisClient.hGetAll(key); return obj[valueKey]; } + + async deleteKeysWithValue (valueKey, clientid) { + let cursor = 0; + const keysToDelete = []; + + do { + const reply = await this.redisClient.scan(cursor); + cursor = parseInt(reply.cursor, 10); + const keys = reply.keys; + + for (const key of keys) { + const value = await this.redisClient.hGet(key, valueKey); + if (value === clientid) { + keysToDelete.push(key); + } + } + } while (cursor !== 0); + + for (const key of keysToDelete) { + await this.redisClient.del(key); + } + + this.logger.info(`Deleted keys with ${valueKey}=${clientid}: ${keysToDelete.join(', ')}`); + } } module.exports = Cache; diff --git a/KafkaBridge/mqttBridge/sparkplug_data_ingestion.js b/KafkaBridge/mqttBridge/sparkplug_data_ingestion.js index 1512701c..c474f732 100644 --- a/KafkaBridge/mqttBridge/sparkplug_data_ingestion.js +++ b/KafkaBridge/mqttBridge/sparkplug_data_ingestion.js @@ -292,6 +292,9 @@ module.exports = class SparkplugHandler { /* It will be checked if the ttl exist, if it exits the package need to be discarded */ const subTopic = topic.split('/'); + if (subTopic[2] !== 'DDATA') { + return; + } this.logger.debug('Data Submission Detected : ' + topic + ' Message: ' + JSON.stringify(message)); if (Object.values(MESSAGE_TYPE.WITHSEQ).includes(subTopic[2])) { const validationResult = this.validator.validate(message, dataSchema.SPARKPLUGB); diff --git a/KafkaBridge/test/lib_authServiceTest.js b/KafkaBridge/test/lib_authServiceTest.js index 31e9ca62..789f4cc5 100644 --- a/KafkaBridge/test/lib_authServiceTest.js +++ b/KafkaBridge/test/lib_authServiceTest.js @@ -240,8 +240,12 @@ describe(fileToTest, function () { } }; const res = { - sendStatus: function (status) { - assert.equal(status, 400, 'Received wrong status'); + status: function (status) { + assert.equal(status, 200, 'Received wrong status'); + return this; + }, + json: function (resultObj) { + resultObj.should.deep.equal({ result: 'deny' }); done(); } }; @@ -285,8 +289,12 @@ describe(fileToTest, function () { } }; const res = { - sendStatus: function (status) { - assert.equal(status, 400, 'Received wrong status'); + status: function (status) { + assert.equal(status, 200, 'Received wrong status'); + return this; + }, + json: function (resultObj) { + resultObj.should.deep.equal({ result: 'deny' }); done(); } }; @@ -386,7 +394,7 @@ describe(fileToTest, function () { getValue (subtopic, key) { assert.equal(aidSlashDid, subtopic, 'Wrong accountId/did subtopic'); assert.equal(key, 'acl', 'Wrong key value'); - return 'true'; + return 'clientid'; } }; ToTest.__set__('Cache', Cache); @@ -403,6 +411,7 @@ describe(fileToTest, function () { const req = { query: { username: 'deviceId', + clientid: 'clientid', topic: 'spBv1.0/accountId/DBIRTH/eonID/deviceId' } }; @@ -447,8 +456,12 @@ describe(fileToTest, function () { } }; const res = { - sendStatus: function (status) { - assert.equal(status, 400, 'Received wrong status'); + status: function (status) { + assert.equal(status, 200, 'Received wrong status'); + return this; + }, + json: function (resultObj) { + resultObj.should.deep.equal({ result: 'deny' }); done(); } }; @@ -480,8 +493,12 @@ describe(fileToTest, function () { } }; const res = { - sendStatus: function (status) { - assert.equal(status, 400, 'Received wrong status'); + status: function (status) { + assert.equal(status, 200, 'Received wrong status'); + return this; + }, + json: function (resultObj) { + resultObj.should.deep.equal({ result: 'deny' }); done(); } }; diff --git a/KafkaBridge/test/lib_cacheTest.js b/KafkaBridge/test/lib_cacheTest.js index aa56da0b..ef53916b 100644 --- a/KafkaBridge/test/lib_cacheTest.js +++ b/KafkaBridge/test/lib_cacheTest.js @@ -103,4 +103,40 @@ describe(fileToTest, function () { cache.getValue('key').then(result => result.should.equal('true')); done(); }); + + it('Shall test deleteKeysWithValue', function (done) { + const config = { + cache: { + port: 1234, + host: 'redishost' + } + }; + + const redis = { + createClient: function () { + return { + on: function (evType) { + evType.should.equal('error'); + }, + scan: async function (cursor) { + return { cursor: '0', keys: ['key1', 'key2', 'key3'] }; + }, + hGet: async function (key, valueKey) { + if (key === 'key1' && valueKey === 'field1') return 'clientid1'; + if (key === 'key2' && valueKey === 'field1') return 'clientid2'; + if (key === 'key3' && valueKey === 'field1') return 'clientid1'; + return null; + }, + del: async function (key) { + } + }; + } + }; + ToTest.__set__('redis', redis); + const cache = new ToTest(config); + cache.deleteKeysWithValue('field1', 'clientid1').then(() => { + // Add assertions for the deletion logic if needed + done(); + }); + }); }); diff --git a/Keycloak/iff-js-providers/META-INF/keycloak-scripts.json b/Keycloak/iff-js-providers/META-INF/keycloak-scripts.json index c222735c..0689611c 100644 --- a/Keycloak/iff-js-providers/META-INF/keycloak-scripts.json +++ b/Keycloak/iff-js-providers/META-INF/keycloak-scripts.json @@ -11,6 +11,11 @@ "name": "Device ID Mapper", "fileName": "deviceid-mapper.js", "description": "deviceId - only valid if access type is device" + }, + { + "name": "Device SUB IDs Mapper", + "fileName": "subdeviceids-mapper.js", + "description": "subdeviceIds - only valid if access type is device" } ], "saml-mappers": [] diff --git a/Keycloak/iff-js-providers/subdeviceids-mapper.js b/Keycloak/iff-js-providers/subdeviceids-mapper.js new file mode 100644 index 00000000..4d4422c3 --- /dev/null +++ b/Keycloak/iff-js-providers/subdeviceids-mapper.js @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Available variables: + * user - the current user + * realm - the current realm + * token - the current token + * userSession - the current userSession + * keycloakSession - the current keycloakSession + */ + +var onboarding_token_expiration = java.lang.System.getenv("OISP_FRONTEND_DEVICE_ACCOUNT_ENDPOINT"); +var subdeviceIdsH = keycloakSession.getContext().getRequestHeaders() + .getRequestHeader("X-SubDeviceIDs")[0]; +if (subdeviceIdsH !== null && subdeviceIdsH !== undefined) { + subdeviceIdsH = JSON.parse(subdeviceIdsH) +} +var inputRequest = keycloakSession.getContext().getHttpRequest(); +var params = inputRequest.getDecodedFormParameters(); +var origTokenParam = params.getFirst("orig_token"); +var grantType = params.getFirst("grant_type"); +var tokens = keycloakSession.tokens(); +var origToken = tokens.decode(origTokenParam, Java.type("org.keycloak.representations.AccessToken").class) + +if (typeof(onboarding_token_expiration) !== 'number') { + // if not otherwise configured onboardig token is valid for 5 minutes + onboarding_token_expiration = 300; +} +if (grantType === 'refresh_token' && origToken !== null) { + var session = userSession.getId(); + var otherClaims = origToken.getOtherClaims(); + var origTokenSubDeviceIds; + if (otherClaims !== null) { + + origTokenSubDeviceIds = otherClaims.get("sub_device_ids"); + } + var origTokenSession = origToken.getSessionId(); + + if (origTokenSubDeviceIds !== null && origTokenSubDeviceIds !== undefined) { + // Has origToken same session? + if (origTokenSession !== session) { + print("Warning: Rejecting subdeviceids claim due to session mismatch between refresh_token and orig_token") + exports = JSON.stringify([]); + } else { + exports = origTokenSubDeviceIds; + } + } else { + // If there is no origTokenDeviceId, there must be an X-DeviceId header AND origToken must be valid + if (!origToken.isExpired() && subdeviceIdsH !== null && subdeviceIdsH !== undefined) { + exports = subdeviceIdsH + } else { + print("Warning: Rejecting subdeviceid claim due to orig_token is expired or there is not valid X-SubDeviceIDs Header.") + exports = JSON.stringify([]); + } + } +} else if (grantType === 'password'){ + var currentTimeInSeconds = new Date().getTime() / 1000; + token.exp(currentTimeInSeconds + onboarding_token_expiration); + exports = null +} else if (origToken === null) { + print("Warning: Rejecting token due to invalid orig_token.") + exports = JSON.stringify([]) +} diff --git a/NgsildAgent/README.md b/NgsildAgent/README.md index 0ea33c3b..25756121 100755 --- a/NgsildAgent/README.md +++ b/NgsildAgent/README.md @@ -8,16 +8,19 @@ The utils directory contains bash scripts to setup and activate a device. ### init-device.sh This script is setting up the default device-file and metadata. ```bash -Usage: init-device.sh [-k keycloakurl] [-r realmId] +Usage: init-device.sh[-k keycloakurl] [-r realmId] [-d additionalDeviceIds] Defaults: keycloakurl=http://keycloak.local/auth/realms realmid=iff + ``` Example: ```bash ./init-device.sh urn:iff:deviceid:1 gatewayid +./init-device.sh -d urn:iff:subdeviceid:1 -d urn:iff:subdeviceid:2 urn:iff:deviceid:1 gatewayid ``` -Note that `deviceid` must be compliant wiht URN format. +First example creates a device with a single urn and a gateway id. Second example creates a device with subcomponents `urn:iff:subdeviceid:1` and `urn:iff:subdeviceid:2`. +Note that `deviceid` and `additionalDeviceIds` must be compliant wiht URN format. ### get-onboarding-token.sh This script assumes a setup device-file, creates an onboarding token and stores it in the data directory. ```bash @@ -45,16 +48,17 @@ Example: ``` ### send-data.sh -This script sends data to a device. It uses as default the UDP API (see below) to communicate to the Agent. +This script sends data to a device. It uses as default the UDP API (see below) to communicate to the Agent. If no id is given, it is assumed to use the default `deviceId`. If data is sent to subcomponents, the `-i` switch is used together with the URN of the respective subdevice id. ```bash -Usage: send_data.sh [-a] [-t] [-y ] [-d datasetId] [ ]+ +Usage: send_data.sh [-a] [-t] [-y ] [-d datasetId] [-i subdeviceid] [ ]+ -a: send array of values -t: use tcp connection to agent (default: udp) -d: give ngsild datasetId (must be iri) +-i: id of subdevice -y: attribute types are {Literal, Iri, Relationship, Json} ``` -### Use tools alltogether to activate a device +### Use tools alltogether to activate a device and send data On a test system with a local kubernetes installed the following flow creates a default test device ```bash @@ -65,6 +69,16 @@ password=$(kubectl -n iff get secret/credential-iff-realm-user-iff -o jsonpath=' ./send_data.sh "https://example.com/state" "ON" ``` +Send data in combination of subdevices looks as follows: + +```bash +./init-device.sh -d urn:iff:subdeviceid:1 -d urn:iff:subdeviceid:2 urn:iff:deviceid:1 gatewayid +./get-onboarding-token.sh -p ${password} realm_user +./activate.sh -f +./send_data.sh "https://example.com/state" "ON" # sends data to root/main device urn:iff:deviceid:1 +./send_data.sh -i urn:iff:subdeviceid:1 "https://example.com/state" "OFF" # sends data to subdevice/subcomponent urn:iff:subdeviceid:1 +``` + ### iff-agent This is a "agent" program intended to run as a service. You can send a very simple NGSI-LD component message, such as ``` diff --git a/NgsildAgent/lib/CloudProxy.js b/NgsildAgent/lib/CloudProxy.js index f28a3a24..bc2cf001 100644 --- a/NgsildAgent/lib/CloudProxy.js +++ b/NgsildAgent/lib/CloudProxy.js @@ -123,7 +123,8 @@ class CloudProxy { edgeNodeId: deviceConf.gateway_id, clientId: deviceConf.device_name, deviceId: deviceConf.device_id, - componentMetric: this.spbMetricList + componentMetric: this.spbMetricList, + subdeviceIds: deviceConf.subdevice_ids, }; this.spBProxy = new SparkplugbConnector(conf, logger); this.logger.info('SparkplugB MQTT proxy found! Configuring Sparkplug and MQTT for data sending.'); @@ -153,7 +154,11 @@ class CloudProxy { } } try { - await this.spBProxy.nodeBirth(this.devProf); + if (this.deviceId === this.gatewayId) { + await this.spBProxy.nodeBirth(this.devProf); + } else { + this.logger.info('No Nodebirth sent because gatewayid != deviceid'); + } } catch (err) { if (err instanceof ConnectionError && err.errno === 1) { this.logger.error('SparkplugB MQTT NBIRTH Metric not sent. Trying to refresh token.'); @@ -204,6 +209,9 @@ class CloudProxy { const compMetric = {}; compMetric.value = metric.v; compMetric.name = metric.n; + if ('i' in metric) { + compMetric.deviceId = metric.i; + } compMetric.dataType = 'string'; compMetric.timestamp = metric.on || new Date().getTime(); compMetric.properties = metric.properties; @@ -230,7 +238,7 @@ class CloudProxy { me.logger.debug('SparkplugB MQTT device profile: ' + me.devProf); await me.spBProxy.publishData(me.devProf, componentMetrics); - me.logger.info('SparkplugB MQTT DDATA Metric sent successfully'); + me.logger.info('SparkplugB MQTT DDATA Metric sent'); } }; diff --git a/NgsildAgent/lib/ConnectionManager.js b/NgsildAgent/lib/ConnectionManager.js index 0a776001..f79d3ef5 100644 --- a/NgsildAgent/lib/ConnectionManager.js +++ b/NgsildAgent/lib/ConnectionManager.js @@ -97,7 +97,11 @@ class ConnectionManager { }; connected () { - return this.client.connected; + if (this.client !== undefined) { + return this.client.connected; + } else { + return false; + } }; authorized () { diff --git a/NgsildAgent/lib/SparkplugbConnector.js b/NgsildAgent/lib/SparkplugbConnector.js index 55554833..50e7339c 100644 --- a/NgsildAgent/lib/SparkplugbConnector.js +++ b/NgsildAgent/lib/SparkplugbConnector.js @@ -121,13 +121,21 @@ class SparkplugbConnector { * Payload for device birth is in device profile componentMetric */ async deviceBirth (devProf) { - const topic = common.buildPath(this.topics.metric_topic, [this.spbConf.version, devProf.groupId, 'DBIRTH', devProf.edgeNodeId, devProf.deviceId]); - const payload = { - timestamp: new Date().getTime(), - metrics: devProf.componentMetric, - seq: incSeqNum() + let subdeviceIds = []; + if ('subdeviceIds' in devProf) { + subdeviceIds = devProf.subdeviceIds }; - return await this.client.publish(topic, payload, this.pubArgs); + const alldeviceIds = subdeviceIds; + alldeviceIds.push(devProf.deviceId); + for (const deviceId of alldeviceIds) { + const topic = common.buildPath(this.topics.metric_topic, [this.spbConf.version, devProf.groupId, 'DBIRTH', devProf.edgeNodeId, deviceId]); + const payload = { + timestamp: new Date().getTime(), + metrics: devProf.componentMetric, + seq: incSeqNum() + }; + await this.client.publish(topic, payload, this.pubArgs); + } }; /* For publishing sparkplugB standard device DATA message @@ -135,14 +143,35 @@ class SparkplugbConnector { * its component ids * @payloadMetric: Contains submitted data value in spB metric format to be sent to server */ - publishData = async function (devProf, payloadMetric) { - const topic = common.buildPath(this.topics.metric_topic, [this.spbConf.version, devProf.groupId, 'DDATA', devProf.edgeNodeId, devProf.deviceId]); - const payload = { - timestamp: new Date().getTime(), - metrics: payloadMetric, - seq: incSeqNum() - }; - await this.client.publish(topic, payload, this.pubArgs); + publishData = async function (devProf, payloadMetrics) { + let deviceId = devProf.deviceId; + const topics = this.topics; + const spbConf = this.spbConf; + const client = this.client; + const pubArgs = this.pubArgs; + const deviceIdMetrics = {}; + for (const payloadMetric of payloadMetrics) { + if ('deviceId' in payloadMetric && devProf.subdeviceIds.includes(payloadMetric.deviceId)) { + deviceId = payloadMetric.deviceId; + } else if ('deviceId' in payloadMetric && !(payloadMetric.deviceId in devProf.subdeviceIds) && (payloadMetric.deviceId !== devProf.deviceId)) { + console.warn('Unknown deviceid: ' + payloadMetric.deviceId); + return; + } + delete payloadMetric.deviceId; + if (!(deviceId in deviceIdMetrics)) { + deviceIdMetrics[deviceId] = []; + } + deviceIdMetrics[deviceId].push(payloadMetric); + } + Object.keys(deviceIdMetrics).forEach(async function (did) { + const topic = common.buildPath(topics.metric_topic, [spbConf.version, devProf.groupId, 'DDATA', devProf.edgeNodeId, did]); + const payload = { + timestamp: new Date().getTime(), + metrics: deviceIdMetrics[did], + seq: incSeqNum() + }; + await client.publish(topic, payload, pubArgs); + }); }; disconnect = function () { diff --git a/NgsildAgent/lib/schemas/data.json b/NgsildAgent/lib/schemas/data.json index 6cef3af4..8bf0a64b 100644 --- a/NgsildAgent/lib/schemas/data.json +++ b/NgsildAgent/lib/schemas/data.json @@ -17,6 +17,9 @@ }, "d": { "type": "string" + }, + "i": { + "type": "string" } }, "required": [ diff --git a/NgsildAgent/util/activate.sh b/NgsildAgent/util/activate.sh index ffbd43ab..163797fa 100755 --- a/NgsildAgent/util/activate.sh +++ b/NgsildAgent/util/activate.sh @@ -98,7 +98,8 @@ keycloakurl=$(jq -r '.keycloak_url' "$DEVICE_FILE") realmid=$(jq -r '.realm_id' "$DEVICE_FILE") gatewayid=$(jq -r '.gateway_id' "$DEVICE_FILE") deviceid=$(jq -r '.device_id' "$DEVICE_FILE") - +deviceids=$(jq -r '.subdevice_ids' "$DEVICE_FILE" | tr -d '\n') +deviceids='"'${deviceids//\"/\\\"}'"' # Check if the file exists if [ -z "$keycloakurl" ] || [ -z "$gatewayid" ] || [ -z "$deviceid" ] || [ -z "$realmid" ]; then echo "device json file doesnot contain required item, please do initialize device." @@ -111,7 +112,7 @@ echo "API endpoint is : $DEVICE_TOKEN_ENDPOINT" # Make the curl request with access token as a header and store the response in the temporary file device_token=$(curl -X POST "$DEVICE_TOKEN_ENDPOINT" -d "client_id=device" \ -d "grant_type=refresh_token" -d "refresh_token=${refresh_token}" -d "orig_token=${orig_token}" -d "audience=device" \ --H "X-GatewayID: $gatewayid" -H "X-DeviceID: $deviceid" 2>/dev/null | jq '.') +-H "X-GatewayID: $gatewayid" -H "X-DeviceID: $deviceid" -H "X-SubDeviceIDs: $deviceids" 2>/dev/null | jq '.') if [ "$(echo "$device_token" | jq 'has("error")')" = "true" ]; then echo "Error: Onboarding token coule not be retrieved." diff --git a/NgsildAgent/util/init-device.sh b/NgsildAgent/util/init-device.sh index 683d0201..33a77b96 100755 --- a/NgsildAgent/util/init-device.sh +++ b/NgsildAgent/util/init-device.sh @@ -17,10 +17,21 @@ set -e # shellcheck disable=SC1091 . common.sh + +function checkurn(){ + local deviceid="$1" + urnPattern='^urn:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}:[a-zA-Z0-9()+,\-\.:=@;$_!*%/?#]+$' + if echo "$deviceid" | grep -E -q "$urnPattern"; then + echo "$deviceid is URN compliant." + else + echo "$deviceid must be an URN. Please fix the parameter $deviceid. Exiting." + exit 1 + fi +} keycloakurl="http://keycloak.local/auth/realms" realmid="iff" -usage="Usage: $(basename "$0") [-k keycloakurl] [-r realmId]\nDefaults: \nkeycloakurl=${keycloakurl}\nrealmid=${realmid}\n" -while getopts 'k:r:h' opt; do +usage="Usage: $(basename "$0")[-k keycloakurl] [-r realmId] [-d additionalDeviceIds] \nDefaults: \nkeycloakurl=${keycloakurl}\nrealmid=${realmid}\n" +while getopts 'k:r:d:h' opt; do # shellcheck disable=SC2221,SC2222 case "$opt" in k) @@ -33,8 +44,12 @@ while getopts 'k:r:h' opt; do echo "Realm url is set to '${arg}'" realmid="${OPTARG}" ;; + d) + additionalDeviceIds+=("$OPTARG") + echo "Added additional deviceId ${OPTARG}" + ;; ?|h) - echo "$usage" + printf "$usage" exit 1 ;; esac @@ -50,15 +65,19 @@ else exit 1 fi - # shellcheck disable=2016 -urnPattern='^urn:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}:[a-zA-Z0-9()+,\-\.:=@;$_!*%/?#]+$' -if echo "$deviceid" | grep -E -q "$urnPattern"; then - echo "$deviceid is URN compliant." -else - echo "$deviceid must be an URN. Please fix the deviceId. Exiting." - exit 1 +checkurn $deviceid +if [ ! -z "${additionalDeviceIds}" ]; then + #deviceid='["'${deviceid}'"' + for i in "${additionalDeviceIds[@]}"; do + checkurn $i + #echo proecessing $i + #deviceid=${deviceid}', "'$i'"' + done + #deviceid=${deviceid}']' +#else +# deviceid='["'${deviceid}'"]' fi - + # shellcheck disable=2016 echo Processing with deviceid="${deviceid}" gatewayid="${gatewayid}" keycloakurl="${keycloakurl}" realmid="${realmid}" if [ ! -d ../data ]; then @@ -71,15 +90,27 @@ if ! dpkg -l | grep -q "jq"; then exit 1 fi -# Define the JSON file path +# To preserve backward compatibility, there are now two fields, device_id and device_ids +commaSeparatedIds= +for i in "${additionalDeviceIds[@]}"; do + if [ -n "$commaSeparatedIds" ]; then + commaSeparatedIds+="," + fi + commaSeparatedIds+=$i +done + +# Define the JSON file path json_data=$(jq -n \ - --arg deviceId "$deviceid" \ + --arg deviceIds "$commaSeparatedIds" \ + --arg deviceid "$deviceid" \ --arg gatewayId "$gatewayid" \ --arg realmId "$realmid" \ --arg keycloakUrl "$keycloakurl" \ - '{ - "device_id": $deviceId, + ' + $deviceIds | split(",") as $ids | { + "device_id": $deviceid, + "subdevice_ids": $ids, "gateway_id": $gatewayId, "realm_id": $realmId, "keycloak_url": $keycloakUrl diff --git a/NgsildAgent/util/send_data.sh b/NgsildAgent/util/send_data.sh index 37415f37..824f655b 100755 --- a/NgsildAgent/util/send_data.sh +++ b/NgsildAgent/util/send_data.sh @@ -18,12 +18,13 @@ set +e # shellcheck disable=SC1091 . ./common.sh -usage="Usage: $(basename "$0") [-a] [-t] [-y ] [-d datasetId] [ ]+ \n\ +usage="Usage: $(basename "$0") [-a] [-t] [-y ] [-d datasetId] [-i subdeviceid] [ ]+ \n\ -a: send array of values\n\ -t: use tcp connection to agent (default: udp)\n\ -d: give ngsild datasetId (must be iri)\n\ +-i: id of subdevice -y: attribute types are {Literal, Iri, Relationship, Json}\n" -while getopts 'athy:d:' opt; do +while getopts 'athy:d:i:' opt; do # shellcheck disable=SC2221,SC2222 case "$opt" in a) @@ -32,6 +33,10 @@ while getopts 'athy:d:' opt; do t) tcp=true ;; + i) + arg="$OPTARG" + deviceId=$arg + ;; y) arg="$OPTARG" attribute_type=$arg @@ -79,6 +84,9 @@ if [ "${num_args}" -eq 2 ] && [ -z "$array" ]; then if [ -n "$datasetId" ]; then payload=${payload}', "d":"'$datasetId'"' fi + if [ -n "$deviceId" ]; then + payload=${payload}', "i":"'$deviceId'"' + fi payload=${payload}'}' echo $payload elif [ "$((num_args%2))" -eq 0 ] && [ -n "$array" ]; then @@ -88,6 +96,9 @@ elif [ "$((num_args%2))" -eq 0 ] && [ -n "$array" ]; then if [ -n "$datasetId" ]; then payload=${payload}', "d":"'$datasetId'"' fi + if [ -n "$deviceId" ]; then + payload=${payload}', "i":"'$deviceId'"' + fi payload=${payload}'}' shift 2 if [ $# -gt 0 ]; then diff --git a/helm/charts/emqx/templates/emxq.yaml b/helm/charts/emqx/templates/emxq.yaml index dee354b8..219def9b 100644 --- a/helm/charts/emqx/templates/emxq.yaml +++ b/helm/charts/emqx/templates/emxq.yaml @@ -43,6 +43,7 @@ spec: body { username = "${username}" password = "${password}" + clientid = "${clientid}" } headers { "X-Request-Source" = "EMQX" @@ -59,6 +60,7 @@ spec: body { username = "${username}" topic = "${topic}" + clientid = "${clientid}" } headers { "X-Request-Source" = "EMQX" diff --git a/helm/charts/keycloak/templates/keycloak-realm.yaml b/helm/charts/keycloak/templates/keycloak-realm.yaml index 359019a7..19645df2 100644 --- a/helm/charts/keycloak/templates/keycloak-realm.yaml +++ b/helm/charts/keycloak/templates/keycloak-realm.yaml @@ -98,6 +98,28 @@ spec: claim.name: "device_id" multivalued: "false" userinfo.token.claim: "true" + - id: 674b4aac-397d-46bb-84ac-3594408a5e6c + name: subdevice_ids + description: '' + protocol: openid-connect + attributes: + include.in.token.scope: 'true' + display.on.consent.screen: 'true' + gui.order: '' + consent.screen.text: '' + protocolMappers: + - id: ce7158c1-88c2-42ec-9b44-bf44f092fb86 + name: subdeviceids + protocol: openid-connect + protocolMapper: script-subdeviceids-mapper.js + consentRequired: false + config: + multivalued: 'false' + userinfo.token.claim: 'true' + id.token.claim: 'true' + access.token.claim: 'true' + claim.name: subdevice_ids + jsonType.label: String - id: 36d973cc-4c8c-4dad-b6b7-a56eb9c75e57 name: if-company protocol: openid-connect @@ -529,6 +551,7 @@ spec: - offline_access - type - gateway + - subdevice_ids - id: 31c8cc5a-9df2-4606-927a-4aeda07c1e56 clientId: {{ .Values.keycloak.alerta.client }} publicClient: False diff --git a/test/bats/lib/config.bash b/test/bats/lib/config.bash index 6bb55751..a32b09b0 100644 --- a/test/bats/lib/config.bash +++ b/test/bats/lib/config.bash @@ -24,4 +24,5 @@ export USER=realm_user export REALM_ID=iff export KEYCLOAK_URL="http://keycloak.local/auth/realms" export MQTT_URL=emqx-listeners:1883 -export KAFKA_BOOTSTRAP=my-cluster-kafka-bootstrap:9092 \ No newline at end of file +export KAFKA_BOOTSTRAP=my-cluster-kafka-bootstrap:9092 +export EMQX_LABEL="apps.emqx.io/instance=emqx" \ No newline at end of file diff --git a/test/bats/lib/mqtt.bash b/test/bats/lib/mqtt.bash index 8a3caf68..12dd3c1a 100644 --- a/test/bats/lib/mqtt.bash +++ b/test/bats/lib/mqtt.bash @@ -15,28 +15,14 @@ # -MQTT_SERVICE=mqtt-test-service -MQTT_EMQX_SELECTOR="apps.emqx.io/instance: emqx" +MQTT_SERVICE=mqtt-service MQTT_SERVICE_PORT=1883 mqtt_setup_service() { - # shellcheck disable=SC2153 - cat << EOF | kubectl -n "${NAMESPACE}" apply -f - -apiVersion: v1 -kind: Service -metadata: - name: ${MQTT_SERVICE} -spec: - selector: - ${MQTT_EMQX_SELECTOR} - ports: - - protocol: TCP - port: ${MQTT_SERVICE_PORT} - targetPort: ${MQTT_SERVICE_PORT} -EOF -run try "at most 30 times every 5s to find 1 service named '${MQTT_SERVICE}'" + pod=$(kubectl -n ${NAMESPACE} get pods -l ${EMQX_LABEL} -o jsonpath='{.items[0].metadata.name}') + exec stdbuf -oL kubectl -n ${NAMESPACE} port-forward $pod 1883:1883 & } mqtt_delete_service() { - kubectl -n "${NAMESPACE}" delete svc "${MQTT_SERVICE}" + killall kubectl } \ No newline at end of file diff --git a/test/bats/test-device-connection/test-device-agent.bats b/test/bats/test-device-connection/test-device-agent.bats index 27eb0abc..5031814d 100644 --- a/test/bats/test-device-connection/test-device-agent.bats +++ b/test/bats/test-device-connection/test-device-agent.bats @@ -33,10 +33,17 @@ TEST_DIR="$(dirname "$BATS_TEST_FILENAME")" CLIENT_ID=scorpio GATEWAY_ID="testgateway" DEVICE_ID="urn:iff:testdevice:1" +SUBDEVICE_ID1="urn:iff:testsubdevice:1" +SUBDEVICE_ID2="urn:iff:testsubdevice:2" +SUBDEVICE_IDS='[ + urn:iff:testsubdevice:1, + urn:iff:testsubdevice:2 +]' DEVICE_FILE="device.json" ONBOARDING_TOKEN="onboard-token.json" NGSILD_AGENT_DIR=${TEST_DIR}/../../../NgsildAgent DEVICE_ID2="testdevice2" +SUBDEVICE_ID3="testsubdevice3" SECRET_FILENAME=/tmp/SECRET AGENT_CONFIG1=/tmp/AGENT_CONFIG1 AGENT_CONFIG2=/tmp/AGENT_CONFIG2 @@ -49,7 +56,6 @@ PGREST_URL="http://pgrest.local/entityhistory" PGREST_RESULT=/tmp/PGREST_RESULT - cat << EOF > ${AGENT_CONFIG1} { "data_directory": "./data", @@ -70,7 +76,7 @@ cat << EOF > ${AGENT_CONFIG1} }, "connector": { "mqtt": { - "host": "${MQTT_SERVICE}", + "host": "localhost", "port": 1883, "websockets": false, "qos": 1, @@ -85,7 +91,6 @@ cat << EOF > ${AGENT_CONFIG1} } EOF - cat << EOF > ${AGENT_CONFIG2} { "data_directory": "./data", @@ -106,7 +111,7 @@ cat << EOF > ${AGENT_CONFIG2} }, "connector": { "mqtt": { - "host": "${MQTT_SERVICE}", + "host": "localhost", "port": 1883, "websockets": false, "qos": 1, @@ -129,14 +134,31 @@ check_device_file_contains() { [ "$deviceid" = "$1" ] && [ "$gatewayid" = "$2" ] && [ "$realmid" = "$3" ] && [ "$keycloakurl" = "$4" ] } +check_device_file_contains_with_subcomponents() { + deviceid=$(jq '.device_id' "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}" | tr -d '"') + gatewayid=$(jq '.gateway_id' "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}" | tr -d '"') + realmid=$(jq '.realm_id' "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}" | tr -d '"') + keycloakurl=$(jq '.keycloak_url' "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}" | tr -d '"') + subdevice_ids=$(jq '.subdevice_ids' "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}" | tr -d '"') + echo "# subdevice_ids $subdevice_ids" + echo [ "$subdevice_ids" = "$5" ] + [ "$deviceid" = "$1" ] && [ "$gatewayid" = "$2" ] && [ "$realmid" = "$3" ] && [ "$keycloakurl" = "$4" ] && [ "$subdevice_ids" = "$5" ] +} + init_agent_and_device_file() { (rm -f "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}") - ( { cd "${NGSILD_AGENT_DIR}" && [ -d node_modules ] && echo "iff-agent already insalled"; } || { npm install && echo "iff-agent successfully installed."; } ) + ( { cd "${NGSILD_AGENT_DIR}" && [ -d node_modules ] && echo "iff-agent already installed"; } || { npm install && echo "iff-agent successfully installed."; } ) (cd "${NGSILD_AGENT_DIR}"/util && bash ./init-device.sh "${DEVICE_ID}" "${GATEWAY_ID}") } +init_agent_and_device_file_with_subcomponents() { + (rm -f "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}") + ( { cd "${NGSILD_AGENT_DIR}" && [ -d node_modules ] && echo "iff-agent already installed"; } || { npm install && echo "iff-agent successfully installed."; } ) + (cd "${NGSILD_AGENT_DIR}"/util && bash ./init-device.sh -d "$SUBDEVICE_ID1" -d "$SUBDEVICE_ID2" "${DEVICE_ID}" "${GATEWAY_ID}") +} + delete_tmp() { rm -f "${PGREST_RESULT}" } @@ -178,7 +200,8 @@ get_tsdb_samples() { # compare entity with reference # $1: file to compare with compare_pgrest_result1() { - cat << EOF | jq | diff "$1" - >&3 + number=$1 + cat << EOF | jq | diff "$2" - >&3 [ { "attributeId": "http://example.com/property1", @@ -187,13 +210,34 @@ compare_pgrest_result1() { "entityId": "urn:iff:testdevice:1", "index": 0, "nodeType": "@value", - "value": "0", + "value": "${number}", + "valueType": null + } +] +EOF +} + +# compare entity with reference +# $1: file to compare with +compare_pgrest_subdevice_result1() { + number=$1 + cat << EOF | jq | diff "$2" - >&3 +[ + { + "attributeId": "http://example.com/property1", + "attributeType": "https://uri.etsi.org/ngsi-ld/Property", + "datasetId": "@none", + "entityId": "urn:iff:testsubdevice:2", + "index": 0, + "nodeType": "@value", + "value": "${number}", "valueType": null } ] EOF } + # compare entity with reference # $1: file to compare with compare_pgrest_result2() { @@ -606,6 +650,13 @@ setup() { [ "${status}" -eq "0" ] } +@test "test init_device.sh with subcomponents" { + $SKIP + init_agent_and_device_file_with_subcomponents + run check_device_file_contains_with_subcomponents "${DEVICE_ID}" "${GATEWAY_ID}" "${REALM_ID}" "${KEYCLOAK_URL}" "${SUBDEVICE_IDS}" + [ "${status}" -eq "0" ] +} + @test "test init_device.sh with deviceid no URN" { $SKIP (rm -f "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}") @@ -614,6 +665,14 @@ setup() { [ "${status}" -eq "0" ] } +@test "test init_device.sh with subdeviceid no URN" { + $SKIP + (rm -f "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}") + (cd "${NGSILD_AGENT_DIR}"/util && bash ./init-device.sh -d "${SUBDEVICE_ID3}" "${DEVICE_ID}" "${GATEWAY_ID}" || echo "failed as expected") + run [ ! -f "${NGSILD_AGENT_DIR}"/data/"${DEVICE_FILE}" ] + [ "${status}" -eq "0" ] +} + @test "test get-onboarding-token.sh" { $SKIP init_agent_and_device_file @@ -657,12 +716,36 @@ setup() { cp "${AGENT_CONFIG1}" "${NGSILD_AGENT_DIR}"/config/config.json (cd "${NGSILD_AGENT_DIR}" && exec stdbuf -oL node ./iff-agent.js) & sleep 2 - (cd "${NGSILD_AGENT_DIR}"/util && bash ./send_data.sh "${PROPERTY1}" 0 ) + randomnr=$RANDOM + (cd "${NGSILD_AGENT_DIR}"/util && bash ./send_data.sh "${PROPERTY1}" ${randomnr} ) sleep 2 pkill -f iff-agent mqtt_delete_service get_tsdb_samples "${DEVICE_ID}" 1 "${token}" > ${PGREST_RESULT} - run compare_pgrest_result1 ${PGREST_RESULT} + run compare_pgrest_result1 ${randomnr} ${PGREST_RESULT} + [ "${status}" -eq "0" ] +} + +@test "test agent starting up and sending subcomponent data" { + $SKIP + init_agent_and_device_file_with_subcomponents + delete_tmp + mqtt_setup_service + password=$(get_password) + token=$(get_token "$password") + (cd "${NGSILD_AGENT_DIR}"/util && bash ./get-onboarding-token.sh -p "$password" "${USER}") + (cd "${NGSILD_AGENT_DIR}"/util && bash ./activate.sh -f) + cp "${AGENT_CONFIG1}" "${NGSILD_AGENT_DIR}"/config/config.json + (cd "${NGSILD_AGENT_DIR}" && exec stdbuf -oL node ./iff-agent.js) & + sleep 2 + randomnr=$RANDOM + (cd "${NGSILD_AGENT_DIR}"/util && bash ./send_data.sh -i ${SUBDEVICE_ID2} "${PROPERTY1}" ${randomnr} ) + sleep 2 + pkill -f iff-agent + mqtt_delete_service + rm -f ${PGREST_RESULT} + get_tsdb_samples "${SUBDEVICE_ID2}" 1 "${token}" > ${PGREST_RESULT} + run compare_pgrest_subdevice_result1 "${randomnr}" "${PGREST_RESULT}" [ "${status}" -eq "0" ] } @@ -729,8 +812,8 @@ setup() { echo -n '{"n": "'$PROPERTY1'", "v": "8"}' >/dev/tcp/127.0.0.1/7070 echo '{"n": "'$PROPERTY2'", "v": "9"}' >/dev/udp/127.0.0.1/41234 sleep 1 - pkill -f iff-agent mqtt_delete_service + pkill -f iff-agent get_tsdb_samples "${DEVICE_ID}" 6 "${token}" > ${PGREST_RESULT} run compare_pgrest_result4 ${PGREST_RESULT} [ "${status}" -eq "0" ] @@ -750,14 +833,14 @@ setup() { sleep 2 (cd "${NGSILD_AGENT_DIR}"/util && bash ./send_data.sh -at "${PROPERTY1}" 10 "${PROPERTY2}" 11 ) sleep 1 - mqtt_delete_service - sleep 10 + mqtt_delete_service + sleep 5 mqtt_setup_service - sleep 2 + sleep 3 (cd "${NGSILD_AGENT_DIR}"/util && bash ./send_data.sh -t "${PROPERTY1}" 12 ) sleep 1 - pkill -f iff-agent mqtt_delete_service + pkill -f iff-agent get_tsdb_samples "${DEVICE_ID}" 3 "${token}" > ${PGREST_RESULT} run compare_pgrest_result5 ${PGREST_RESULT} [ "${status}" -eq "0" ] diff --git a/test/bats/test-device-connection/test-device-authorization.bats b/test/bats/test-device-connection/test-device-authorization.bats index 67f77b22..b7b92683 100644 --- a/test/bats/test-device-connection/test-device-authorization.bats +++ b/test/bats/test-device-connection/test-device-authorization.bats @@ -4,6 +4,12 @@ # SUDO="sudo -E" # fi +load "../lib/utils" +load "../lib/detik" +load "../lib/config" +load "../lib/db" +load "../lib/mqtt" + DEBUG=${DEBUG:-false} SKIP= NAMESPACE=iff @@ -15,11 +21,17 @@ GATEWAY_ID="testgateway" GATEWAY_ID2="testgateway2" DEVICE_ID="testdevice" DEVICE_ID2="testdevice2" -DEVICE_TOKEN_SCOPE="device_id gateway mqtt-broker offline_access" +DEVICE_TOKEN_SCOPE="device_id gateway mqtt-broker offline_access subdevice_ids" DEVICE_TOKEN_AUDIENCE_FROM_DIRECT='mqtt-broker' -MQTT_URL=emqx-listeners:1883 +SUBDEVICE_IDS='"[\"testsubdevice1\"]"' +# shellcheck disable=SC2089 +SUBDEVICE_IDS2='"[\"testsubdevice1\",\"testsubdevice2\",\"testsubdevice3\"]"' MQTT_TOPIC_NAME="spBv1.0/${NAMESPACE}/DDATA/${GATEWAY_ID}/${DEVICE_ID}" +MQTT_SUBDEVICE_TOPIC_NAME="spBv1.0/${NAMESPACE}/DDATA/${GATEWAY_ID}/testsubdevice1" +MQTT_SUBDEVICE_TOPIC_NAME2="spBv1.0/${NAMESPACE}/DDATA/${GATEWAY_ID}/testsubdevice2" MQTT_MESSAGE='{"timestamp":1655974018778,"metrics":[{ "name":"Property/https://industry-fusion.com/types/v0.9/state","timestamp":1655974018777,"dataType":"string","value":"https://industry-fusion.com/types/v0.9/state_OFF"}],"seq":1}' +MQTT_MESSAGE2='{"timestamp":1655974018778,"metrics":[{ "name":"Property/https://industry-fusion.com/types/v0.9/state","timestamp":1655974018777,"dataType":"string","value":"https://industry-fusion.com/types/v0.9/state_ON"}],"seq":1}' +MQTT_MESSAGE3='{"timestamp":1655974018778,"metrics":[{ "name":"Property/https://industry-fusion.com/types/v0.9/state","timestamp":1655974018777,"dataType":"string","value":"no"}],"seq":1}' KAFKA_BOOTSTRAP=my-cluster-kafka-bootstrap:9092 KAFKACAT_ATTRIBUTES=/tmp/KAFKACAT_ATTRIBUTES KAFKACAT_ATTRIBUTES_TOPIC=iff.ngsild.attributes @@ -64,6 +76,30 @@ get_refreshed_device_token() { | jq ".access_token" | tr -d '"' } +get_refreshed_device_token_with_subcomponents() { + curl -X POST "${KEYCLOAK_URL}/${NAMESPACE}/protocol/openid-connect/token" \ + -d "client_id=${DEVICE_CLIENT_ID}" \ + -d "grant_type=refresh_token" \ + -d "refresh_token=$1" \ + -d "orig_token=$2" \ + -H "X-DeviceID: ${DEVICE_ID}" \ + -H "X-GatewayID: ${GATEWAY_ID}" \ + -H "X-SubDeviceIDs: ${SUBDEVICE_IDS}" \ + | jq ".access_token" | tr -d '"' +} + +get_refreshed_device_token_with_subcomponents2() { + curl -X POST "${KEYCLOAK_URL}/${NAMESPACE}/protocol/openid-connect/token" \ + -d "client_id=${DEVICE_CLIENT_ID}" \ + -d "grant_type=refresh_token" \ + -d "refresh_token=$1" \ + -d "orig_token=$2" \ + -H "X-DeviceID: ${DEVICE_ID}" \ + -H "X-GatewayID: ${GATEWAY_ID}" \ + -H "X-SubDeviceIDs: ${SUBDEVICE_IDS2}" \ + | jq ".access_token" | tr -d '"' +} + get_refreshed_device_token_with_wrong_ids() { curl -X POST "${KEYCLOAK_URL}/${NAMESPACE}/protocol/openid-connect/token" \ -d "client_id=${DEVICE_CLIENT_ID}" \ @@ -128,6 +164,27 @@ check_refreshed_device_token() { check_vanilla_device_token_audience "${jwt}" || return 1 } +check_refreshed_device_token_with_subcomponents() { + jwt=$(echo "$1" | jq -R 'split(".") | .[1] | @base64d | fromjson') + check_json_field "${jwt}" "azp" "device" || return 1 + check_json_field "${jwt}" "device_id" "${DEVICE_ID}" || return 1 + check_json_field "${jwt}" "gateway" "${GATEWAY_ID}" || return 1 + check_json_field "${jwt}" "subdevice_ids" "$(echo "${SUBDEVICE_IDS}"| tr -d '\"')" || return 1 + check_device_token_scope "${jwt}" || return 1 + check_vanilla_device_token_audience "${jwt}" || return 1 +} + +check_refreshed_device_token_with_subcomponents2() { + jwt=$(echo "$1" | jq -R 'split(".") | .[1] | @base64d | fromjson') + check_json_field "${jwt}" "azp" "device" || return 1 + check_json_field "${jwt}" "device_id" "${DEVICE_ID}" || return 1 + check_json_field "${jwt}" "gateway" "${GATEWAY_ID}" || return 1 + check_json_field "${jwt}" "subdevice_ids" "$(echo "${SUBDEVICE_IDS2}"| tr -d '\"')" || return 1 + check_device_token_scope "${jwt}" || return 1 + check_vanilla_device_token_audience "${jwt}" || return 1 +} + + check_refreshed_device_token_fail() { jwt=$(echo "$1" | jq -R 'split(".") | .[1] | @base64d | fromjson') check_json_field "${jwt}" "azp" "device" || return 1 @@ -191,6 +248,19 @@ compare_create_attributes() { EOF } +compare_create_attributes2() { + cat << EOF | diff "$1" - >&3 +{"id":"testsubdevice1\\\\https://industry-fusion.com/types/v0.9/state",\ +"entityId":"testsubdevice1",\ +"nodeType":"@value",\ +"name":"https://industry-fusion.com/types/v0.9/state",\ +"type":"https://uri.etsi.org/ngsi-ld/Property",\ +"https://uri.etsi.org/ngsi-ld/hasValue":"https://industry-fusion.com/types/v0.9/state_ON",\ +"index":0,"datasetId":"@none"} +EOF +} + + compare_mqtt_sub(){ cat << EOF | diff "$1" - >&3 {"timestamp":1655974018778,"metrics":[{ "name":"Property/https://industry-fusion.com/types/v0.9/state",\ @@ -215,6 +285,9 @@ setup() { fi } +teardown() { + killall kafkacat mosquitto_sub || true +} @test "verify user can request onboarding token" { $SKIP @@ -225,7 +298,6 @@ setup() { [ "${status}" -eq "0" ] } - @test "verify device token can be refreshed" { $SKIP password=$(get_password) @@ -242,6 +314,35 @@ setup() { [ "${status}" -eq "0" ] } +@test "verify device token can be refreshed with a subcomponent" { + $SKIP + password=$(get_password) + token=$(get_vanilla_refresh_and_access_token) + refresh_token=$(echo "$token" | jq ".refresh_token" | tr -d '"') + access_token=$(echo "$token" | jq ".access_token" | tr -d '"') + echo "# refresh_token=$refresh_token" + echo "# access_token=$access_token" + device_token=$(get_refreshed_device_token_with_subcomponents "${refresh_token}" "${access_token}") + echo "# device_token=$device_token" + run check_refreshed_device_token_with_subcomponents "${device_token}" + [ "${status}" -eq "0" ] +} + + +@test "verify device token can be refreshed with several subcomponents" { + $SKIP + password=$(get_password) + token=$(get_vanilla_refresh_and_access_token) + refresh_token=$(echo "$token" | jq ".refresh_token" | tr -d '"') + access_token=$(echo "$token" | jq ".access_token" | tr -d '"') + echo "# refresh_token=$refresh_token" + echo "# access_token=$access_token" + device_token=$(get_refreshed_device_token_with_subcomponents2 "${refresh_token}" "${access_token}") + echo "# device_token=$device_token" + run check_refreshed_device_token_with_subcomponents2 "${device_token}" + [ "${status}" -eq "0" ] +} + @test "verify device token becomes tainted if refreshed without headers" { $SKIP password=$(get_password) @@ -300,7 +401,10 @@ setup() { echo "# access_token=$access_token" device_token=$(get_refreshed_device_token "${refresh_token}" "${access_token}") echo "# device_token: $device_token" - mosquitto_pub -L "mqtt://${DEVICE_ID}:${device_token}@${MQTT_URL}/${MQTT_TOPIC_NAME}" -m "${MQTT_MESSAGE}" + mqtt_setup_service + sleep 2 + mosquitto_pub -L "mqtt://${DEVICE_ID}:${device_token}@localhost/${MQTT_TOPIC_NAME}" -m "${MQTT_MESSAGE}" + mqtt_delete_service echo "# Sent mqtt sparkplugB message, sleep 2s to let bridge react" sleep 2 echo "# now killing kafkacat and evaluate result" @@ -311,6 +415,58 @@ setup() { [ "$status" -eq 0 ] } + +@test "verify device token can send data to subcomponent and is forwarded to Kafka" { + $SKIP + (exec stdbuf -oL kafkacat -C -t ${KAFKACAT_ATTRIBUTES_TOPIC} -b ${KAFKA_BOOTSTRAP} -o end >${KAFKACAT_ATTRIBUTES}) & + password=$(get_password) + token=$(get_vanilla_refresh_and_access_token) + refresh_token=$(echo "$token" | jq ".refresh_token" | tr -d '"') + access_token=$(echo "$token" | jq ".access_token" | tr -d '"') + echo "# refresh_token=$refresh_token" + echo "# access_token=$access_token" + device_token=$(get_refreshed_device_token_with_subcomponents "${refresh_token}" "${access_token}") + echo "# device_token: $device_token" + mqtt_setup_service + sleep 2 + mosquitto_pub -L "mqtt://${DEVICE_ID}:${device_token}@localhost/${MQTT_SUBDEVICE_TOPIC_NAME}" -m "${MQTT_MESSAGE2}" + mqtt_delete_service + echo "# Sent mqtt sparkplugB message, sleep 2s to let bridge react" + sleep 2 + echo "# now killing kafkacat and evaluate result" + killall kafkacat + LC_ALL="en_US.UTF-8" sort -o ${KAFKACAT_ATTRIBUTES} ${KAFKACAT_ATTRIBUTES} + echo "# Compare ATTRIBUTES" + run compare_create_attributes2 ${KAFKACAT_ATTRIBUTES} + [ "$status" -eq 0 ] +} + +@test "verify device token can not send data to unknown subcomponent" { + $SKIP + (exec stdbuf -oL kafkacat -C -t ${KAFKACAT_ATTRIBUTES_TOPIC} -b ${KAFKA_BOOTSTRAP} -o end >${KAFKACAT_ATTRIBUTES}) & + password=$(get_password) + token=$(get_vanilla_refresh_and_access_token) + refresh_token=$(echo "$token" | jq ".refresh_token" | tr -d '"') + access_token=$(echo "$token" | jq ".access_token" | tr -d '"') + echo "# refresh_token=$refresh_token" + echo "# access_token=$access_token" + device_token=$(get_refreshed_device_token_with_subcomponents "${refresh_token}" "${access_token}") + echo "# device_token: $device_token" + mqtt_setup_service + sleep 2 + mosquitto_pub -L "mqtt://${DEVICE_ID}:${device_token}@localhost/${MQTT_SUBDEVICE_TOPIC_NAME}" -m "${MQTT_MESSAGE2}" + mosquitto_pub -L "mqtt://${DEVICE_ID}:${device_token}@localhost/${MQTT_SUBDEVICE_TOPIC_NAME2}" -m "${MQTT_MESSAGE3}" + mqtt_delete_service + echo "# Sent mqtt sparkplugB message, sleep 2s to let bridge react" + sleep 2 + echo "# now killing kafkacat and evaluate result" + killall kafkacat + LC_ALL="en_US.UTF-8" sort -o ${KAFKACAT_ATTRIBUTES} ${KAFKACAT_ATTRIBUTES} + echo "# Compare ATTRIBUTES" + run compare_create_attributes2 ${KAFKACAT_ATTRIBUTES} + [ "$status" -eq 0 ] +} + @test "verify tainted device is rejected" { $SKIP password=$(get_password) @@ -318,7 +474,10 @@ setup() { refresh_token=$(echo "$token" | jq ".refresh_token" | tr -d '"') access_token=$(echo "$token" | jq ".access_token" | tr -d '"') device_token=$(get_refreshed_vanilla_token "${refresh_token}" "${access_token}") - mosquitto_pub -L "mqtt://${DEVICE_ID}:${device_token}@${MQTT_URL}/${MQTT_TOPIC_NAME}" -m "${MQTT_MESSAGE}" 2>${MQTT_RESULT} || true + mqtt_setup_service + sleep 2 + mosquitto_pub -L "mqtt://${DEVICE_ID}:${device_token}@localhost/${MQTT_TOPIC_NAME}" -m "${MQTT_MESSAGE}" 2>${MQTT_RESULT} || true + mqtt_delete_service cat ${MQTT_RESULT} | grep "not authorised" } @@ -326,13 +485,16 @@ setup() { $SKIP password=$(get_adminPassword | tr -d '"') username=$(get_adminUsername | tr -d '"') - (exec stdbuf -oL mosquitto_sub -L "mqtt://${username}:${password}@${MQTT_URL}/${MQTT_TOPIC_NAME}" >${MQTT_SUB}) & + mqtt_setup_service + sleep 2 + (exec stdbuf -oL mosquitto_sub -L "mqtt://${username}:${password}@localhost/${MQTT_TOPIC_NAME}" >${MQTT_SUB}) & sleep 2 - mosquitto_pub -L "mqtt://${username}:${password}@${MQTT_URL}/${MQTT_TOPIC_NAME}" -m "${MQTT_MESSAGE}" + mosquitto_pub -L "mqtt://${username}:${password}@localhost/${MQTT_TOPIC_NAME}" -m "${MQTT_MESSAGE}" echo "# Sent mqtt sparkplugB message, sleep 2s to let bridge react" sleep 2 echo "# now killing kafkacat and evaluate result" killall mosquitto_sub + mqtt_delete_service echo "# Compare ATTRIBUTES" run compare_mqtt_sub ${MQTT_SUB} [ "$status" -eq 0 ]