From d8f21397dbe8b3ca4441332bf2170780d4d39f99 Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 5 Nov 2023 17:09:47 +0100 Subject: [PATCH 01/32] Added examples for simple NGSI-LD objects --- .../examples/simple_plasmacutter2_data.json | 5 +++++ .../examples/simple_plasmacutter2_schema.json | 17 +++++++++++++++++ .../examples/simple_plasmacutter_data.json | 5 +++++ .../examples/simple_plasmacutter_schema.json | 17 +++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 semantic-model/datamodel/examples/simple_plasmacutter2_data.json create mode 100644 semantic-model/datamodel/examples/simple_plasmacutter2_schema.json create mode 100644 semantic-model/datamodel/examples/simple_plasmacutter_data.json create mode 100644 semantic-model/datamodel/examples/simple_plasmacutter_schema.json diff --git a/semantic-model/datamodel/examples/simple_plasmacutter2_data.json b/semantic-model/datamodel/examples/simple_plasmacutter2_data.json new file mode 100644 index 00000000..434de876 --- /dev/null +++ b/semantic-model/datamodel/examples/simple_plasmacutter2_data.json @@ -0,0 +1,5 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "plasmacutter" +} diff --git a/semantic-model/datamodel/examples/simple_plasmacutter2_schema.json b/semantic-model/datamodel/examples/simple_plasmacutter2_schema.json new file mode 100644 index 00000000..35c93de9 --- /dev/null +++ b/semantic-model/datamodel/examples/simple_plasmacutter2_schema.json @@ -0,0 +1,17 @@ +[{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base/v0.1/Plasmacutter", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "plasmacutter" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"] +}] \ No newline at end of file diff --git a/semantic-model/datamodel/examples/simple_plasmacutter_data.json b/semantic-model/datamodel/examples/simple_plasmacutter_data.json new file mode 100644 index 00000000..10da1870 --- /dev/null +++ b/semantic-model/datamodel/examples/simple_plasmacutter_data.json @@ -0,0 +1,5 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017" +} diff --git a/semantic-model/datamodel/examples/simple_plasmacutter_schema.json b/semantic-model/datamodel/examples/simple_plasmacutter_schema.json new file mode 100644 index 00000000..72149bdd --- /dev/null +++ b/semantic-model/datamodel/examples/simple_plasmacutter_schema.json @@ -0,0 +1,17 @@ +[{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/eclass%230173-1%2301-AKJ975%23017", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "eclass:0173-1#01-AKJ975#017" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"] +}] \ No newline at end of file From 2cd8d11a08d5a6fe2334dcd5aa245c1e60836f11 Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 5 Nov 2023 17:16:57 +0100 Subject: [PATCH 02/32] Added validate.js tool --- semantic-model/datamodel/README.md | 291 ++++++++++++++++++++ semantic-model/datamodel/tools/package.json | 15 + semantic-model/datamodel/tools/validate.js | 76 +++++ 3 files changed, 382 insertions(+) create mode 100644 semantic-model/datamodel/README.md create mode 100644 semantic-model/datamodel/tools/package.json create mode 100644 semantic-model/datamodel/tools/validate.js diff --git a/semantic-model/datamodel/README.md b/semantic-model/datamodel/README.md new file mode 100644 index 00000000..19fe4ce4 --- /dev/null +++ b/semantic-model/datamodel/README.md @@ -0,0 +1,291 @@ +# Digital Twin Datamodel + +The following motivates, descibes and defines version 0.1 of the Datamodel. It is in alpha stage and subject to changes. + +## JSON-LD (Linked Data) + +JSON-LD, which stands for "JavaScript Object Notation for Linked Data," is a powerful data serialization format that extends the capabilities of traditional JSON. Its strength lies in its ability to represent structured data in a way that is both human-readable and machine-understandable. JSON-LD is particularly well-suited for the web and semantic data integration for several key reasons: + + * **Semantic Structure:** JSON-LD allows you to add context to your data, defining the meaning of each piece of information. This semantic structure enables machines to understand the data and its relationships, fostering interoperability and knowledge sharing. + + * **Linked Data:** JSON-LD is designed to facilitate the linking of data across the web. It enables you to reference and interconnect data from various sources and domains, forming a comprehensive and coherent information ecosystem. + + * **SEO and Searchability:** Search engines like Google understand and favor JSON-LD for structuring data. Implementing JSON-LD can improve your website's visibility in search results by providing search engines with valuable information about your content. + + * **Interoperability:** JSON-LD supports the integration of data from diverse sources, making it an ideal choice for data exchange, data sharing, and data synchronization between applications, platforms, and services. + + * **Easy to Read:** JSON-LD retains the simplicity and human-readability of traditional JSON, making it accessible for both developers and non-technical users. Its natural syntax encourages widespread adoption. + + * **Standards-Based:** JSON-LD is based on W3C standards and recommendations, ensuring a well-defined and widely accepted approach to structuring and sharing linked data on the web. + +In summary, JSON-LD is a versatile and powerful tool for structuring data with semantic meaning, linking data across the web, improving search engine visibility, fostering interoperability, and promoting data exchange. It plays a crucial role in the modern web ecosystem and is a valuable asset for businesses and organizations looking to harness the full potential of their data. + +## JSON-LD Forms + +Since JSON-LD represents graph data, it can become very explicit and details. However, In many cases aspects of a graph can also be simplified and described implicitly. + +The following shows a so called *compacted* JSON-LD expression. It contains a *context* and a minimized key *name*: + +``` +{ + "@context": { + "name": "http://schema.org/name" + }, + "@id": "https://iri/max.mustermann", + "name": "Max Mustermann" +} +``` + +This is an implicit representation of a *expanded* form +``` +[{ + "@id": "https://iri/max.mustermann", + "http://schema.org/name": [{"@value": "Max Mustermann"}] +}] +``` + +Note that in the Expanded form, the *context* is missing, but everything is now provided with *namespaces* and *@value* which indicates that "Max Mustermann" is a *literal*, i.e. string, number or boolean. +The expanded form can easiliy be transformed into a *Semantic Web* graph representation: + +```meermaid +A(https://iri/max.mustermann) -- http://schema.org/name --> B("Max Mustermann") +``` + +which can also be serialized as turtle graph: + +``` +@prefix schema: . + schema:name "Max Mustermann" . +``` + +## NGSI-LD (Next Generation Service Interface for Linked Data) + +[NGSI-LD](https://www.etsi.org/deliver/etsi_gs/CIM/001_099/009/01.07.01_60/gs_CIM009v010701p.pdf) is an open standard developed by the European Telecommunications Standards Institute (ETSI) as part of the NGSI (Next Generation Service Interface) framework. It extends the capabilities of JSON-LD to enable a powerful, standardized approach to managing and exchanging context information for the Internet of Things (IoT) and smart city applications. + +Key features and concepts of NGSI-LD: + +* **Linked Data Model:** NGSI-LD is based on the principles of Linked Data, making it a part of the Semantic Web ecosystem. It allows the representation of real-world entities and their attributes as linked data resources. + + * **Entity-Attribute-Value (EAV):** NGSI-LD follows an Entity-Attribute-Value (EAV) model where entities (e.g., IoT devices or physical objects) have attributes (e.g., temperature, location) with associated values (e.g., 25°C, GPS coordinates). + + * **Context Information:** NGSI-LD is designed for sharing context information about entities. This context information can include real-time data, historical data, metadata, and relationships between entities. + +* **Interoperability:** One of the main goals of NGSI-LD is to enable interoperability between different IoT platforms, systems, and services. It provides a common data representation format and query language for IoT context information. + +* **Standardized APIs:** NGSI-LD specifies a set of standardized APIs for querying, updating, and subscribing to context information. This helps developers create applications that can work with a variety of data sources and platforms. + +* **Scalability:** NGSI-LD is designed to handle vast amounts of context information generated by IoT devices, sensors, and other sources, making it suitable for smart city and industrial IoT applications. + +* **Semantic Descriptions:** Similar to JSON-LD, NGSI-LD uses semantic descriptions (context) to define the meaning of data attributes. This enables data to be easily understood and used by both humans and machines. + +* **Real-Time Updates:** NGSI-LD supports real-time updates and notifications, making it ideal for applications that require immediate access to changing context information. + +NGSI-LD is a significant advancement in the field of IoT, as it provides a standardized approach for managing and exchanging context data, enabling more efficient and interoperable IoT solutions. It leverages the power of Linked Data to create a dynamic and interconnected IoT ecosystem. NGSI-LD is already used heavily in smart city applications + + +## NGSI-LD forms + +Since NGSI-LD is extending JSON-LD, it inherits the capability of creating different forms like *expanded* or *compacted*. In addition, it provides a simplification called *concise* form and a more explicit form, called *normalized* form. + +NGSI-LD reuqires from every entity to have at least *id* and *type*. All other data is either a *property* or a *relationship*: +``` +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/" + } + ], + "id": "urn:x:1", + "type": "cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:filter:1" + }, + "machine_state": { + "type": "Property", + "value": "Testing" + } +} + +``` + +The *type* field is here redundant, that is why NGSI-LD defines a *concise* form which is reducing redudancy: + +``` +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/" + } + ], + "id": "urn:x:1", + "type": "cutter", + "hasFilter": { + "object": "urn:filter:1" + }, + "machine_state": "Testing" +} +``` + +Since NGSI-LD is JSON-LD compliant, it can be *compacted* and *extended*. Note that both forms, *normalized* and *concise* NGSI-LD, are already *compacted* JSON-LD forms. The expanded *normalized* form of the above example looks like + +``` +[ + { + "https://industry-fusion.org/base/v0.1/hasFilter": [ + { + "https://uri.etsi.org/ngsi-ld/hasObject": [ + { + "@id": "urn:filter:1" + } + ], + "@type": [ + "https://uri.etsi.org/ngsi-ld/Relationship" + ] + } + ], + "@id": "urn:x:1", + "https://industry-fusion.org/base/v0.1/machine_state": [ + { + "@type": [ + "https://uri.etsi.org/ngsi-ld/Property" + ], + "https://uri.etsi.org/ngsi-ld/hasValue": [ + { + "@value": "Testing" + } + ] + } + ], + "@type": [ + "https://industry-fusion.org/base/v0.1/cutter" + ] + } +] +``` + +## Validation with JSON-Schema and SHACL + +Validation of JSON objects is typically done with [JSON-Schema](https://json-schema.org/). A plain JSON object structure can therefore be validated. However, as described above, JSON-LD represent a graph and has different forms. For instance, the following two expressions are equivalent in JSON-LD but cannot be schemed with JSON-Schema: + +Expression 1 + +``` +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/" + } + ], + "id": "urn:x:1", + "type": "cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:filter:1" + }, + "machine_state": { + "type": "Property", + "value": "Testing" + } +}, +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/" + } + ], + "id": "urn:y:1", + "type": "filter", + "machine_state": { + "type": "Property", + "value": "Testing" + } +} +] +``` + +Expression 2 + +``` +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/" + } + ], + "id": "urn:x:1", + "type": "cutter", + "hasFilter": { + "type": "Relationship", + "object": { + "id": "urn:y:1", + "type": "filter", + "machine_state": { + "type": "Property", + "value": "Testing" + } + } + }, + "machine_state": { + "type": "Property", + "value": "Testing" + } +} +``` + +In addition, JSON-schema is not able to properly process *Namespaces*. + +Therefore, a proper validation must consider the Graph structure of JSON-LD. A standard, which allows to define Constraints within a Graph is [SHACL](https://www.w3.org/TR/shacl/). + + +## JSON-LD Validation with JSON-Schema + +As shown in the last section, it is impossible to use JSON-Schema to validate JSON-LD objects properly. However, as a compromise, many non-linked data related attributes can be validated if one is applying a propoer normalization. Therefore, we use the JSON-Schema in the following to validate a *concise* NGSI-LD form with a pre-defined *context*. + +In the following, we describe the validation schema. +We use the default *contex* https://industryfusion.github.io/contexts/v0.1/context.jsonld. An example for a *concise* form with this *conext* can be seen in the following. It contains an *ECLASS* type, one *ECLASS* property and attributes (properties and relationships) `machine_state` and `hasFilter` from the default vocabulary. The object has an ID expressed as URN: + +``` +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "machine_state": "Testing", + "hasFilter": { + "object": "urn:filter:1" + }, + "eclass:0173-1#02-AAH880#003": "10", + "id": "urn:x:1", + "type": "eclass:0173-1#01-AKJ975#017" +} +``` + +In order to validate it with a JSON-Schema, first the base object must be described: + +``` + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/eclass#0173-1#01-AKJ975#017", + "title": "Plasmacutter", + "description": "Plasmacutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"], + "allOf": [] + } +``` + +## Translating JSON-Schema to SHACL + +## Tools \ No newline at end of file diff --git a/semantic-model/datamodel/tools/package.json b/semantic-model/datamodel/tools/package.json new file mode 100644 index 00000000..4496e8f1 --- /dev/null +++ b/semantic-model/datamodel/tools/package.json @@ -0,0 +1,15 @@ +{ + "name": "pdt-validation", + "version": "0.1.0", + "description": "This package contains tools for validating NGSI-LD data for the Process Data Twin (PDT)", + "main": "validate.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Marcel Wagner", + "license": "Apache-2.0", + "dependencies": { + "ajv": "^8.12.0", + "yargs": "^17.7.2" + } +} diff --git a/semantic-model/datamodel/tools/validate.js b/semantic-model/datamodel/tools/validate.js new file mode 100644 index 00000000..62fb8822 --- /dev/null +++ b/semantic-model/datamodel/tools/validate.js @@ -0,0 +1,76 @@ +/** +* Copyright (c) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +'use strict' + +const fs = require('fs') +const url = require('url') +const Ajv = require('ajv/dist/2020') +const yargs = require('yargs') +const path = require('path') + +const argv = yargs + .option('schema', { + alias: 's', + description: 'Schema File', + demandOption: true, + type: 'string' + }) + .option('datafile', { + alias: 'd', + description: 'File to validate', + demandOption: true, + type: 'string' + }) + .option('schemaid', { + alias: 'i', + description: 'Schema-id to validate', + demandOption: true, + type: 'string' + }) + .help() + .alias('help', 'h') + .argv + +const ajv = new Ajv({strict: false, strictTypes: false, strictSchema: false, strictTuples: false}) + +const schema = fs.readFileSync(argv.s, 'utf8') +const parsedSchema = JSON.parse(schema) + +ajv.addSchema(parsedSchema) + +// This special processing is needed to allow ECLASS integration with +// URL. ECLASS uses IRDI which makes a lot of use of '#' which is +// incompatible with $id definition of JSON-Schema. Workaround +// is to use URL-encoding + +const id = url.parse(argv.i) +var idFragment = ''; +var idPath = ''; +if (id.hash !== null) { + idFragment = encodeURIComponent(id.hash); +} +if (id.path !== null) { + idPath = id.path; +} +const idUrl = id.protocol + '//' + id.host + idPath + idFragment; + +const data = JSON.parse(fs.readFileSync(argv.d, 'utf8')) +if (ajv.validate(idUrl, data)) { + console.log('The Datafile is compliant with Schema') +} else { + console.log('Not Compliant:') + console.log(ajv.errors) +}; From fde84af7f1bdc25a5f72ff46baf2b72ec98e15ea Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 5 Nov 2023 18:18:39 +0100 Subject: [PATCH 03/32] added properties and relationships --- semantic-model/datamodel/README.md | 135 +++++++++++++++++- .../datamodel/examples/plasmacutter_data.json | 9 ++ .../examples/plasmacutter_schema.json | 79 ++++++++++ 3 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 semantic-model/datamodel/examples/plasmacutter_data.json create mode 100644 semantic-model/datamodel/examples/plasmacutter_schema.json diff --git a/semantic-model/datamodel/README.md b/semantic-model/datamodel/README.md index 19fe4ce4..73582c45 100644 --- a/semantic-model/datamodel/README.md +++ b/semantic-model/datamodel/README.md @@ -268,7 +268,7 @@ In order to validate it with a JSON-Schema, first the base object must be descri ``` { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://industry-fusion.org/eclass#0173-1#01-AKJ975#017", + "$id": "", "title": "Plasmacutter", "description": "Plasmacutter template for IFF", "type": "object", @@ -285,6 +285,139 @@ In order to validate it with a JSON-Schema, first the base object must be descri "allOf": [] } ``` +This specifies the mandatory `id` and `type` field of every NGSI-LD object. `id` must be an *URN* and `type` must contain a *compacted* form. +The "$schema" field must be "https://json-schema.org/draft/2020-12/schema", the $id of the schema must be a valid URL with the additional constraint that all '#' fields must be URL-Encoded. An example for a schema and related data can be seen in the following: + +``` +# JSON-Schema: +[{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/eclass%230173-1%2301-AKJ975%23017", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "eclass:0173-1#01-AKJ975#017" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"] +}] + +# JSON-LD Object +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017" +} +``` + +The *properties* and *relationships* can be grouped and aggregated by the `allOf` array. In the following a *string* property is validated by a schema: + +``` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/cutter/properties", + "title": "Cutter properties", + "description": "Properties for class cutter", + "type": "object", + "properties": { + "machine_state": { + "type": "string", + "title": "Machine Status", + "description": "Current status of the machine (Online_Idle, Run, Online_Error, Online_Maintenance, Setup, Testing)", + "enum": [ + "Online_Idle", + "Run", + "Online_Error", + "Online_Maintenance", + "Setup", + "Testing" + ] + } + } +} +``` +The following validates a *relationship*: + + +``` + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/cutter/relationships", + "title": "IFF template for cutter relationship", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "hasFilter": { + "relationship": "eclass:0173-1#01-ACK991#016", + "$ref": "https://industry-fusion.org/base-objects/v0.1/link" + } + } + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/link", + "title": "IFF template for cutter relationship", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "object": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}:[a-zA-Z0-9()+,\\-.:=@;$_!*']*[a-zA-Z0-9()+,\\-.:=@;$_!*']$" + } + }, + "required": ["object"] + }, + +``` +Overall, the resulting full JSON-Schema looks like follows: + +``` +# JSON-Schema: +[{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/eclass%230173-1%2301-AKJ975%23017", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "eclass:0173-1#01-AKJ975#017" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "allOf": [ + { + "$ref": "https://industry-fusion.org/base-objects/v0.1/cutter/properties" + }, + { + "$ref": "https://industry-fusion.org/base-objects/v0.1/cutter/relationships" + } + ], + "required": ["type", "id"] +}] + +# JSON-LD Object +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": { + "object": "urn:iff:filter:1" + }, + "machine_state": "Testing" +} + +``` +## Integrating ECLASS Properties ## Translating JSON-Schema to SHACL diff --git a/semantic-model/datamodel/examples/plasmacutter_data.json b/semantic-model/datamodel/examples/plasmacutter_data.json new file mode 100644 index 00000000..799eb2dd --- /dev/null +++ b/semantic-model/datamodel/examples/plasmacutter_data.json @@ -0,0 +1,9 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": { + "object": "urn:iff:filter:1" + }, + "machine_state": "Testing" +} diff --git a/semantic-model/datamodel/examples/plasmacutter_schema.json b/semantic-model/datamodel/examples/plasmacutter_schema.json new file mode 100644 index 00000000..ebc29cba --- /dev/null +++ b/semantic-model/datamodel/examples/plasmacutter_schema.json @@ -0,0 +1,79 @@ +[ + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/eclass%230173-1%2301-AKJ975%23017", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "eclass:0173-1#01-AKJ975#017" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"], + "allOf": [ + { + "$ref": "https://industry-fusion.org/base-objects/v0.1/cutter/properties" + }, + { + "$ref": "https://industry-fusion.org/base-objects/v0.1/cutter/relationships" + } + ] + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/cutter/relationships", + "title": "IFF template for cutter relationship", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "hasFilter": { + "relationship": "eclass:0173-1#01-ACK991#016", + "$ref": "https://industry-fusion.org/base-objects/v0.1/link" + } + } + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/link", + "title": "IFF template for cutter relationship", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "object": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}:[a-zA-Z0-9()+,\\-.:=@;$_!*']*[a-zA-Z0-9()+,\\-.:=@;$_!*']$" + } + }, + "required": ["object"] + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/cutter/properties", + "title": "Cutter properties", + "description": "Properties for class cutter", + "type": "object", + "properties": { + "machine_state": { + "type": "string", + "title": "Machine Status", + "description": "Current status of the machine (Online_Idle, Run, Online_Error, Online_Maintenance, Setup, Testing)", + "enum": [ + "Online_Idle", + "Run", + "Online_Error", + "Online_Maintenance", + "Setup", + "Testing" + ] + } + }, + "required": [ + "machine_state" + ] + } +] \ No newline at end of file From 1919f0c338bd785770528fc8618e1690ad15274b Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 5 Nov 2023 21:25:33 +0100 Subject: [PATCH 04/32] Added keyword management for schema --- semantic-model/datamodel/README.md | 54 ++++++++++++++++++- .../examples/simple_plasmacutter2_data.json | 2 +- semantic-model/datamodel/tools/validate.js | 13 ++++- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/semantic-model/datamodel/README.md b/semantic-model/datamodel/README.md index 73582c45..2a67c37c 100644 --- a/semantic-model/datamodel/README.md +++ b/semantic-model/datamodel/README.md @@ -417,8 +417,60 @@ Overall, the resulting full JSON-Schema looks like follows: } ``` + +## Forbidden JSON-Schema keywords: +The following JSON-Schema Keywords from the standard are forbidded: + +* anyOf, oneOf +* if, then, else +* prefixItems, items +* valid, error, annotations +* additionalProperties +* propertyNames +* $vocabulary, $defs +* multipleOf +* uniqueItems +* maxContains, minContains +* maxProperties, minProperties +* dependentRequired + +## Added JSON-Schmea Keywords: +The following JSON-Schema Keywords are added to the standard: + +* **relationship:** Contains the *compacted* type for a NGSI-LD relationship. + ## Integrating ECLASS Properties +`ECLASS` provides additional data for every `IRDI` which can be added/mapped to `JSON-Schema`: + +- `Preferred Name` is mapped to `title` field +- `Definition` is mapped to `description` field +- The unit symbol of `Unit` is mapped to `unit` field +- `Type` of the field is mapped to `datatype` and xsd-type as described [here](https://eclass.eu/support/technical-specification/data-model/datatype-to-xsd-mapping) +- The `JSON` `type` field of every `ECLASS` property is `string`. + +For [example](./schema-ngsild-eclass/schema.json), the `ECLASS` property `` is described in the `JSON-Schema` as: + +``` + "eclass:0173-1#02-AAH880#003": { + "type": "string", + "datatype": "double", + "title": "min. cutting current", + "description": "specification of the minimum cutting current", + "unit": "A" + } +``` ## Translating JSON-Schema to SHACL -## Tools \ No newline at end of file +## Tools +This section describes the tools which are used for validation, data conversion and SHACL creation. The tools can be found in the `./tools` + +### Validation + +The validation tool is `validate.js`. It is a nodejs tool working with node v1.16 + +`node tools/validate.js -s examples/plasmacutter_schema.json -d examples/plasmacutter_data.json -i https://industry-fusion.org/eclass#0173-1#01-AKJ975#017` + +### Convert JSON-Schema to SHACL + +### Convert NGSI-LD forms \ No newline at end of file diff --git a/semantic-model/datamodel/examples/simple_plasmacutter2_data.json b/semantic-model/datamodel/examples/simple_plasmacutter2_data.json index 434de876..40072f30 100644 --- a/semantic-model/datamodel/examples/simple_plasmacutter2_data.json +++ b/semantic-model/datamodel/examples/simple_plasmacutter2_data.json @@ -1,5 +1,5 @@ { "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", "id": "urn:iff:abc123", - "type": "plasmacutter" + "type": "Plasmacutter" } diff --git a/semantic-model/datamodel/tools/validate.js b/semantic-model/datamodel/tools/validate.js index 62fb8822..915924ff 100644 --- a/semantic-model/datamodel/tools/validate.js +++ b/semantic-model/datamodel/tools/validate.js @@ -21,6 +21,12 @@ const Ajv = require('ajv/dist/2020') const yargs = require('yargs') const path = require('path') +const removedKeywords = ["anyOf", "oneOf", "if", "then", "else", "prefixItems", "items", + "valid", "error", "annotation", "additionalProperties", "propertyNames", + "$vocabulary", "$defs", "multipleOf", "uniqueItems", "maxContains", + "minContains", "maxProperties", "minPropeties", "dependentRequired"] +const addedKeywords = ["relationship"] + const argv = yargs .option('schema', { alias: 's', @@ -44,7 +50,7 @@ const argv = yargs .alias('help', 'h') .argv -const ajv = new Ajv({strict: false, strictTypes: false, strictSchema: false, strictTuples: false}) +const ajv = new Ajv({strict: true, strictTypes: true, strictSchema: true, strictTuples: false}) const schema = fs.readFileSync(argv.s, 'utf8') const parsedSchema = JSON.parse(schema) @@ -68,6 +74,11 @@ if (id.path !== null) { const idUrl = id.protocol + '//' + id.host + idPath + idFragment; const data = JSON.parse(fs.readFileSync(argv.d, 'utf8')) + +// Remove all non supported and add all proprietary keywords. +removedKeywords.forEach(kw => ajv.removeKeyword(kw)); +addedKeywords.forEach(kw => ajv.addKeyword({keyword: kw})); + if (ajv.validate(idUrl, data)) { console.log('The Datafile is compliant with Schema') } else { From f96ef91eea2ad9e313ba1369f0acb27d5edc9efa Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 5 Nov 2023 22:49:21 +0100 Subject: [PATCH 05/32] added eslint --- semantic-model/datamodel/tools/.eslintrc.js | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 semantic-model/datamodel/tools/.eslintrc.js diff --git a/semantic-model/datamodel/tools/.eslintrc.js b/semantic-model/datamodel/tools/.eslintrc.js new file mode 100644 index 00000000..0e3df72f --- /dev/null +++ b/semantic-model/datamodel/tools/.eslintrc.js @@ -0,0 +1,26 @@ +module.exports = { + env: { + browser: true, + commonjs: true, + es2021: true + }, + extends: 'standard', + overrides: [ + { + env: { + node: true + }, + files: [ + '.eslintrc.{js,cjs}' + ], + parserOptions: { + sourceType: 'script' + } + } + ], + parserOptions: { + ecmaVersion: 'latest' + }, + rules: { + } +} From 9bfce7d66501be1209a27df8c0c02967763ca68d Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 5 Nov 2023 22:59:45 +0100 Subject: [PATCH 06/32] linting --- semantic-model/datamodel/tools/package.json | 10 +++++- semantic-model/datamodel/tools/validate.js | 37 ++++++++++++--------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/semantic-model/datamodel/tools/package.json b/semantic-model/datamodel/tools/package.json index 4496e8f1..bba2e917 100644 --- a/semantic-model/datamodel/tools/package.json +++ b/semantic-model/datamodel/tools/package.json @@ -4,12 +4,20 @@ "description": "This package contains tools for validating NGSI-LD data for the Process Data Twin (PDT)", "main": "validate.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "lint": "./node_modules/.bin/eslint *.js" }, "author": "Marcel Wagner", "license": "Apache-2.0", "dependencies": { "ajv": "^8.12.0", "yargs": "^17.7.2" + }, + "devDependencies": { + "eslint": "^8.53.0", + "eslint-config-standard": "^17.1.0", + "eslint-plugin-import": "^2.29.0", + "eslint-plugin-n": "^16.2.0", + "eslint-plugin-promise": "^6.1.1" } } diff --git a/semantic-model/datamodel/tools/validate.js b/semantic-model/datamodel/tools/validate.js index 915924ff..374d2b44 100644 --- a/semantic-model/datamodel/tools/validate.js +++ b/semantic-model/datamodel/tools/validate.js @@ -19,13 +19,13 @@ const fs = require('fs') const url = require('url') const Ajv = require('ajv/dist/2020') const yargs = require('yargs') -const path = require('path') -const removedKeywords = ["anyOf", "oneOf", "if", "then", "else", "prefixItems", "items", - "valid", "error", "annotation", "additionalProperties", "propertyNames", - "$vocabulary", "$defs", "multipleOf", "uniqueItems", "maxContains", - "minContains", "maxProperties", "minPropeties", "dependentRequired"] -const addedKeywords = ["relationship"] +const removedKeywords = [ + 'anyOf', 'oneOf', 'if', 'then', 'else', 'prefixItems', 'items', + 'valid', 'error', 'annotation', 'additionalProperties', 'propertyNames', + '$vocabulary', '$defs', 'multipleOf', 'uniqueItems', 'maxContains', + 'minContains', 'maxProperties', 'minPropeties', 'dependentRequired'] +const addedKeywords = ['relationship'] const argv = yargs .option('schema', { @@ -50,10 +50,15 @@ const argv = yargs .alias('help', 'h') .argv -const ajv = new Ajv({strict: true, strictTypes: true, strictSchema: true, strictTuples: false}) +const ajv = new Ajv({ + strict: true, + strictTypes: true, + strictSchema: true, + strictTuples: false +}) const schema = fs.readFileSync(argv.s, 'utf8') -const parsedSchema = JSON.parse(schema) +const parsedSchema = JSON.parse(schema) ajv.addSchema(parsedSchema) @@ -62,22 +67,22 @@ ajv.addSchema(parsedSchema) // incompatible with $id definition of JSON-Schema. Workaround // is to use URL-encoding -const id = url.parse(argv.i) -var idFragment = ''; -var idPath = ''; +const id = url.URL(argv.i) +let idFragment = '' +let idPath = '' if (id.hash !== null) { - idFragment = encodeURIComponent(id.hash); + idFragment = encodeURIComponent(id.hash) } if (id.path !== null) { - idPath = id.path; + idPath = id.path } -const idUrl = id.protocol + '//' + id.host + idPath + idFragment; +const idUrl = id.protocol + '//' + id.host + idPath + idFragment const data = JSON.parse(fs.readFileSync(argv.d, 'utf8')) // Remove all non supported and add all proprietary keywords. -removedKeywords.forEach(kw => ajv.removeKeyword(kw)); -addedKeywords.forEach(kw => ajv.addKeyword({keyword: kw})); +removedKeywords.forEach(kw => ajv.removeKeyword(kw)) +addedKeywords.forEach(kw => ajv.addKeyword({ keyword: kw })) if (ajv.validate(idUrl, data)) { console.log('The Datafile is compliant with Schema') From bd4502da2391e51c91be68c1686bc97e9b21082b Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 6 Nov 2023 00:04:27 +0100 Subject: [PATCH 07/32] added tests for validation --- .../tools/tests/validationt1_data1.json | 5 +++++ .../tools/tests/validationt1_data1.json_id | 1 + .../tools/tests/validationt1_data1.json_result | 1 + .../tools/tests/validationt1_data2.json | 5 +++++ .../tools/tests/validationt1_data2.json_id | 1 + .../tools/tests/validationt1_data2.json_result | 12 ++++++++++++ .../tools/tests/validationt1_data3.json | 5 +++++ .../tools/tests/validationt1_data3.json_id | 1 + .../tools/tests/validationt1_data3.json_result | 10 ++++++++++ .../tools/tests/validationt1_schema.json | 17 +++++++++++++++++ 10 files changed, 58 insertions(+) create mode 100644 semantic-model/datamodel/tools/tests/validationt1_data1.json create mode 100644 semantic-model/datamodel/tools/tests/validationt1_data1.json_id create mode 100644 semantic-model/datamodel/tools/tests/validationt1_data1.json_result create mode 100644 semantic-model/datamodel/tools/tests/validationt1_data2.json create mode 100644 semantic-model/datamodel/tools/tests/validationt1_data2.json_id create mode 100644 semantic-model/datamodel/tools/tests/validationt1_data2.json_result create mode 100644 semantic-model/datamodel/tools/tests/validationt1_data3.json create mode 100644 semantic-model/datamodel/tools/tests/validationt1_data3.json_id create mode 100644 semantic-model/datamodel/tools/tests/validationt1_data3.json_result create mode 100644 semantic-model/datamodel/tools/tests/validationt1_schema.json diff --git a/semantic-model/datamodel/tools/tests/validationt1_data1.json b/semantic-model/datamodel/tools/tests/validationt1_data1.json new file mode 100644 index 00000000..10da1870 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_data1.json @@ -0,0 +1,5 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017" +} diff --git a/semantic-model/datamodel/tools/tests/validationt1_data1.json_id b/semantic-model/datamodel/tools/tests/validationt1_data1.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_data1.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validationt1_data1.json_result b/semantic-model/datamodel/tools/tests/validationt1_data1.json_result new file mode 100644 index 00000000..ee71d9c0 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_data1.json_result @@ -0,0 +1 @@ +The Datafile is compliant with Schema diff --git a/semantic-model/datamodel/tools/tests/validationt1_data2.json b/semantic-model/datamodel/tools/tests/validationt1_data2.json new file mode 100644 index 00000000..a7233622 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_data2.json @@ -0,0 +1,5 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "myid", + "type": "eclass:0173-1#01-AKJ975#017" +} diff --git a/semantic-model/datamodel/tools/tests/validationt1_data2.json_id b/semantic-model/datamodel/tools/tests/validationt1_data2.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_data2.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validationt1_data2.json_result b/semantic-model/datamodel/tools/tests/validationt1_data2.json_result new file mode 100644 index 00000000..f9eaceba --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_data2.json_result @@ -0,0 +1,12 @@ +Not Compliant: +[ + { + instancePath: '/id', + schemaPath: '#/properties/id/pattern', + keyword: 'pattern', + params: { + pattern: "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + }, + message: `must match pattern "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$"` + } +] diff --git a/semantic-model/datamodel/tools/tests/validationt1_data3.json b/semantic-model/datamodel/tools/tests/validationt1_data3.json new file mode 100644 index 00000000..ce61a57c --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_data3.json @@ -0,0 +1,5 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "mytype" +} diff --git a/semantic-model/datamodel/tools/tests/validationt1_data3.json_id b/semantic-model/datamodel/tools/tests/validationt1_data3.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_data3.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validationt1_data3.json_result b/semantic-model/datamodel/tools/tests/validationt1_data3.json_result new file mode 100644 index 00000000..4372bdb9 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_data3.json_result @@ -0,0 +1,10 @@ +Not Compliant: +[ + { + instancePath: '/type', + schemaPath: '#/properties/type/const', + keyword: 'const', + params: { allowedValue: 'eclass:0173-1#01-AKJ975#017' }, + message: 'must be equal to constant' + } +] diff --git a/semantic-model/datamodel/tools/tests/validationt1_schema.json b/semantic-model/datamodel/tools/tests/validationt1_schema.json new file mode 100644 index 00000000..72149bdd --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validationt1_schema.json @@ -0,0 +1,17 @@ +[{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/eclass%230173-1%2301-AKJ975%23017", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "eclass:0173-1#01-AKJ975#017" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"] +}] \ No newline at end of file From cb35f6841bf0e79a760475e50514adfe3af2f711 Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 6 Nov 2023 00:07:14 +0100 Subject: [PATCH 08/32] fix filename typos --- ...alidationt1_data1.json_result => validation_data1.json_result} | 0 ...alidationt1_data2.json_result => validation_data2.json_result} | 0 ...alidationt1_data3.json_result => validation_data3.json_result} | 0 .../tests/{validationt1_schema.json => validation_schema.json} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename semantic-model/datamodel/tools/tests/{validationt1_data1.json_result => validation_data1.json_result} (100%) rename semantic-model/datamodel/tools/tests/{validationt1_data2.json_result => validation_data2.json_result} (100%) rename semantic-model/datamodel/tools/tests/{validationt1_data3.json_result => validation_data3.json_result} (100%) rename semantic-model/datamodel/tools/tests/{validationt1_schema.json => validation_schema.json} (100%) diff --git a/semantic-model/datamodel/tools/tests/validationt1_data1.json_result b/semantic-model/datamodel/tools/tests/validation_data1.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validationt1_data1.json_result rename to semantic-model/datamodel/tools/tests/validation_data1.json_result diff --git a/semantic-model/datamodel/tools/tests/validationt1_data2.json_result b/semantic-model/datamodel/tools/tests/validation_data2.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validationt1_data2.json_result rename to semantic-model/datamodel/tools/tests/validation_data2.json_result diff --git a/semantic-model/datamodel/tools/tests/validationt1_data3.json_result b/semantic-model/datamodel/tools/tests/validation_data3.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validationt1_data3.json_result rename to semantic-model/datamodel/tools/tests/validation_data3.json_result diff --git a/semantic-model/datamodel/tools/tests/validationt1_schema.json b/semantic-model/datamodel/tools/tests/validation_schema.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validationt1_schema.json rename to semantic-model/datamodel/tools/tests/validation_schema.json From f56e5571090cd13bd2b19f58c7f2953a586e78ce Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 6 Nov 2023 00:08:09 +0100 Subject: [PATCH 09/32] filename ... --- semantic-model/datamodel/tools/tests/validation_data1.json | 5 +++++ .../datamodel/tools/tests/validation_data1.json_id | 1 + semantic-model/datamodel/tools/tests/validation_data2.json | 5 +++++ .../datamodel/tools/tests/validation_data2.json_id | 1 + semantic-model/datamodel/tools/tests/validation_data3.json | 5 +++++ .../datamodel/tools/tests/validation_data3.json_id | 1 + 6 files changed, 18 insertions(+) create mode 100644 semantic-model/datamodel/tools/tests/validation_data1.json create mode 100644 semantic-model/datamodel/tools/tests/validation_data1.json_id create mode 100644 semantic-model/datamodel/tools/tests/validation_data2.json create mode 100644 semantic-model/datamodel/tools/tests/validation_data2.json_id create mode 100644 semantic-model/datamodel/tools/tests/validation_data3.json create mode 100644 semantic-model/datamodel/tools/tests/validation_data3.json_id diff --git a/semantic-model/datamodel/tools/tests/validation_data1.json b/semantic-model/datamodel/tools/tests/validation_data1.json new file mode 100644 index 00000000..10da1870 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation_data1.json @@ -0,0 +1,5 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017" +} diff --git a/semantic-model/datamodel/tools/tests/validation_data1.json_id b/semantic-model/datamodel/tools/tests/validation_data1.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation_data1.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validation_data2.json b/semantic-model/datamodel/tools/tests/validation_data2.json new file mode 100644 index 00000000..a7233622 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation_data2.json @@ -0,0 +1,5 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "myid", + "type": "eclass:0173-1#01-AKJ975#017" +} diff --git a/semantic-model/datamodel/tools/tests/validation_data2.json_id b/semantic-model/datamodel/tools/tests/validation_data2.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation_data2.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validation_data3.json b/semantic-model/datamodel/tools/tests/validation_data3.json new file mode 100644 index 00000000..ce61a57c --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation_data3.json @@ -0,0 +1,5 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "mytype" +} diff --git a/semantic-model/datamodel/tools/tests/validation_data3.json_id b/semantic-model/datamodel/tools/tests/validation_data3.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation_data3.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 From 8480f179a216ba5d4735a952915b56de68c80efb Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 6 Nov 2023 00:13:04 +0100 Subject: [PATCH 10/32] this time? --- .../datamodel/tools/tests/validationt1_data1.json | 5 ----- .../datamodel/tools/tests/validationt1_data1.json_id | 1 - .../datamodel/tools/tests/validationt1_data2.json | 5 ----- .../datamodel/tools/tests/validationt1_data2.json_id | 1 - .../datamodel/tools/tests/validationt1_data3.json | 5 ----- .../datamodel/tools/tests/validationt1_data3.json_id | 1 - semantic-model/datamodel/tools/validate.js | 6 +++--- 7 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 semantic-model/datamodel/tools/tests/validationt1_data1.json delete mode 100644 semantic-model/datamodel/tools/tests/validationt1_data1.json_id delete mode 100644 semantic-model/datamodel/tools/tests/validationt1_data2.json delete mode 100644 semantic-model/datamodel/tools/tests/validationt1_data2.json_id delete mode 100644 semantic-model/datamodel/tools/tests/validationt1_data3.json delete mode 100644 semantic-model/datamodel/tools/tests/validationt1_data3.json_id diff --git a/semantic-model/datamodel/tools/tests/validationt1_data1.json b/semantic-model/datamodel/tools/tests/validationt1_data1.json deleted file mode 100644 index 10da1870..00000000 --- a/semantic-model/datamodel/tools/tests/validationt1_data1.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", - "id": "urn:iff:abc123", - "type": "eclass:0173-1#01-AKJ975#017" -} diff --git a/semantic-model/datamodel/tools/tests/validationt1_data1.json_id b/semantic-model/datamodel/tools/tests/validationt1_data1.json_id deleted file mode 100644 index a1064ee6..00000000 --- a/semantic-model/datamodel/tools/tests/validationt1_data1.json_id +++ /dev/null @@ -1 +0,0 @@ -https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validationt1_data2.json b/semantic-model/datamodel/tools/tests/validationt1_data2.json deleted file mode 100644 index a7233622..00000000 --- a/semantic-model/datamodel/tools/tests/validationt1_data2.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", - "id": "myid", - "type": "eclass:0173-1#01-AKJ975#017" -} diff --git a/semantic-model/datamodel/tools/tests/validationt1_data2.json_id b/semantic-model/datamodel/tools/tests/validationt1_data2.json_id deleted file mode 100644 index a1064ee6..00000000 --- a/semantic-model/datamodel/tools/tests/validationt1_data2.json_id +++ /dev/null @@ -1 +0,0 @@ -https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validationt1_data3.json b/semantic-model/datamodel/tools/tests/validationt1_data3.json deleted file mode 100644 index ce61a57c..00000000 --- a/semantic-model/datamodel/tools/tests/validationt1_data3.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", - "id": "urn:iff:abc123", - "type": "mytype" -} diff --git a/semantic-model/datamodel/tools/tests/validationt1_data3.json_id b/semantic-model/datamodel/tools/tests/validationt1_data3.json_id deleted file mode 100644 index a1064ee6..00000000 --- a/semantic-model/datamodel/tools/tests/validationt1_data3.json_id +++ /dev/null @@ -1 +0,0 @@ -https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/validate.js b/semantic-model/datamodel/tools/validate.js index 374d2b44..14c2d4ba 100644 --- a/semantic-model/datamodel/tools/validate.js +++ b/semantic-model/datamodel/tools/validate.js @@ -67,14 +67,14 @@ ajv.addSchema(parsedSchema) // incompatible with $id definition of JSON-Schema. Workaround // is to use URL-encoding -const id = url.URL(argv.i) +const id = new url.URL(argv.i) let idFragment = '' let idPath = '' if (id.hash !== null) { idFragment = encodeURIComponent(id.hash) } -if (id.path !== null) { - idPath = id.path +if (id.pathname !== null) { + idPath = id.pathname } const idUrl = id.protocol + '//' + id.host + idPath + idFragment From 37b9aeb2c3b27159a16c999119c7f8c3c0d7d81e Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 6 Nov 2023 00:22:05 +0100 Subject: [PATCH 11/32] added additional validation --- .../tools/tests/validation2_data1.json | 5 +++++ .../tools/tests/validation2_data1.json_id | 1 + .../tools/tests/validation2_data1.json_result | 1 + .../tools/tests/validation2_schema.json | 17 +++++++++++++++++ 4 files changed, 24 insertions(+) create mode 100644 semantic-model/datamodel/tools/tests/validation2_data1.json create mode 100644 semantic-model/datamodel/tools/tests/validation2_data1.json_id create mode 100644 semantic-model/datamodel/tools/tests/validation2_data1.json_result create mode 100644 semantic-model/datamodel/tools/tests/validation2_schema.json diff --git a/semantic-model/datamodel/tools/tests/validation2_data1.json b/semantic-model/datamodel/tools/tests/validation2_data1.json new file mode 100644 index 00000000..40072f30 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation2_data1.json @@ -0,0 +1,5 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "Plasmacutter" +} diff --git a/semantic-model/datamodel/tools/tests/validation2_data1.json_id b/semantic-model/datamodel/tools/tests/validation2_data1.json_id new file mode 100644 index 00000000..1c63c300 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation2_data1.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/base/v0.1/Plasmacutter diff --git a/semantic-model/datamodel/tools/tests/validation2_data1.json_result b/semantic-model/datamodel/tools/tests/validation2_data1.json_result new file mode 100644 index 00000000..ee71d9c0 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation2_data1.json_result @@ -0,0 +1 @@ +The Datafile is compliant with Schema diff --git a/semantic-model/datamodel/tools/tests/validation2_schema.json b/semantic-model/datamodel/tools/tests/validation2_schema.json new file mode 100644 index 00000000..f429d56b --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation2_schema.json @@ -0,0 +1,17 @@ +[{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base/v0.1/Plasmacutter", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "Plasmacutter" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"] +}] \ No newline at end of file From d75b747a23324f6ba0b8f8c2837591b5936c442a Mon Sep 17 00:00:00 2001 From: marcel Date: Tue, 7 Nov 2023 00:40:21 +0100 Subject: [PATCH 12/32] added more tests for validation --- semantic-model/datamodel/README.md | 21 ++++- semantic-model/datamodel/tools/package.json | 2 +- .../tools/tests/validation3_data1.json | 9 +++ .../tools/tests/validation3_data1.json_id | 1 + .../tools/tests/validation3_data1.json_result | 1 + .../tools/tests/validation3_data2.json | 9 +++ .../tools/tests/validation3_data2.json_id | 1 + .../tools/tests/validation3_data2.json_result | 12 +++ .../tools/tests/validation3_data3.json | 9 +++ .../tools/tests/validation3_data3.json_id | 1 + .../tools/tests/validation3_data3.json_result | 1 + .../tools/tests/validation3_data4.json | 7 ++ .../tools/tests/validation3_data4.json_id | 1 + .../tools/tests/validation3_data4.json_result | 10 +++ .../tools/tests/validation3_schema.json | 79 +++++++++++++++++++ 15 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 semantic-model/datamodel/tools/tests/validation3_data1.json create mode 100644 semantic-model/datamodel/tools/tests/validation3_data1.json_id create mode 100644 semantic-model/datamodel/tools/tests/validation3_data1.json_result create mode 100644 semantic-model/datamodel/tools/tests/validation3_data2.json create mode 100644 semantic-model/datamodel/tools/tests/validation3_data2.json_id create mode 100644 semantic-model/datamodel/tools/tests/validation3_data2.json_result create mode 100644 semantic-model/datamodel/tools/tests/validation3_data3.json create mode 100644 semantic-model/datamodel/tools/tests/validation3_data3.json_id create mode 100644 semantic-model/datamodel/tools/tests/validation3_data3.json_result create mode 100644 semantic-model/datamodel/tools/tests/validation3_data4.json create mode 100644 semantic-model/datamodel/tools/tests/validation3_data4.json_id create mode 100644 semantic-model/datamodel/tools/tests/validation3_data4.json_result create mode 100644 semantic-model/datamodel/tools/tests/validation3_schema.json diff --git a/semantic-model/datamodel/README.md b/semantic-model/datamodel/README.md index 2a67c37c..14f32065 100644 --- a/semantic-model/datamodel/README.md +++ b/semantic-model/datamodel/README.md @@ -467,7 +467,26 @@ This section describes the tools which are used for validation, data conversion ### Validation -The validation tool is `validate.js`. It is a nodejs tool working with node v1.16 +The validation tool is `validate.js`. + +#### Install + +``` +npm install +``` + +#### Usage + +``` +Optionen: + --version Version anzeigen [boolean] + -s, --schema Schema File [string] [erforderlich] + -d, --datafile File to validate [string] [erforderlich] + -i, --schemaid Schema-id to validate [string] [erforderlich] + -h, --help Hilfe anzeigen [boolean] +``` + +#### Examples `node tools/validate.js -s examples/plasmacutter_schema.json -d examples/plasmacutter_data.json -i https://industry-fusion.org/eclass#0173-1#01-AKJ975#017` diff --git a/semantic-model/datamodel/tools/package.json b/semantic-model/datamodel/tools/package.json index bba2e917..72638297 100644 --- a/semantic-model/datamodel/tools/package.json +++ b/semantic-model/datamodel/tools/package.json @@ -4,7 +4,7 @@ "description": "This package contains tools for validating NGSI-LD data for the Process Data Twin (PDT)", "main": "validate.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", + "test": "cd tests && bash ./test_validation.sh && exit 0", "lint": "./node_modules/.bin/eslint *.js" }, "author": "Marcel Wagner", diff --git a/semantic-model/datamodel/tools/tests/validation3_data1.json b/semantic-model/datamodel/tools/tests/validation3_data1.json new file mode 100644 index 00000000..799eb2dd --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data1.json @@ -0,0 +1,9 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": { + "object": "urn:iff:filter:1" + }, + "machine_state": "Testing" +} diff --git a/semantic-model/datamodel/tools/tests/validation3_data1.json_id b/semantic-model/datamodel/tools/tests/validation3_data1.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data1.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validation3_data1.json_result b/semantic-model/datamodel/tools/tests/validation3_data1.json_result new file mode 100644 index 00000000..ee71d9c0 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data1.json_result @@ -0,0 +1 @@ +The Datafile is compliant with Schema diff --git a/semantic-model/datamodel/tools/tests/validation3_data2.json b/semantic-model/datamodel/tools/tests/validation3_data2.json new file mode 100644 index 00000000..bf22e725 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data2.json @@ -0,0 +1,9 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": { + "object": "wrongurn" + }, + "machine_state": "Testing" +} diff --git a/semantic-model/datamodel/tools/tests/validation3_data2.json_id b/semantic-model/datamodel/tools/tests/validation3_data2.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data2.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validation3_data2.json_result b/semantic-model/datamodel/tools/tests/validation3_data2.json_result new file mode 100644 index 00000000..e9cef687 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data2.json_result @@ -0,0 +1,12 @@ +Not Compliant: +[ + { + instancePath: '/hasFilter/object', + schemaPath: 'https://industry-fusion.org/base-objects/v0.1/link/properties/object/pattern', + keyword: 'pattern', + params: { + pattern: "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}:[a-zA-Z0-9()+,\\-.:=@;$_!*']*[a-zA-Z0-9()+,\\-.:=@;$_!*']$" + }, + message: `must match pattern "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}:[a-zA-Z0-9()+,\\-.:=@;$_!*']*[a-zA-Z0-9()+,\\-.:=@;$_!*']$"` + } +] diff --git a/semantic-model/datamodel/tools/tests/validation3_data3.json b/semantic-model/datamodel/tools/tests/validation3_data3.json new file mode 100644 index 00000000..799eb2dd --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data3.json @@ -0,0 +1,9 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": { + "object": "urn:iff:filter:1" + }, + "machine_state": "Testing" +} diff --git a/semantic-model/datamodel/tools/tests/validation3_data3.json_id b/semantic-model/datamodel/tools/tests/validation3_data3.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data3.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validation3_data3.json_result b/semantic-model/datamodel/tools/tests/validation3_data3.json_result new file mode 100644 index 00000000..ee71d9c0 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data3.json_result @@ -0,0 +1 @@ +The Datafile is compliant with Schema diff --git a/semantic-model/datamodel/tools/tests/validation3_data4.json b/semantic-model/datamodel/tools/tests/validation3_data4.json new file mode 100644 index 00000000..a0a280d7 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data4.json @@ -0,0 +1,7 @@ +{ + "@context": "https://industryfusion.github.io/contexts/v0.1/context.jsonld", + "id": "urn:iff:abc123", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": "urn:iff:filter:1", + "machine_state": "Testing" +} diff --git a/semantic-model/datamodel/tools/tests/validation3_data4.json_id b/semantic-model/datamodel/tools/tests/validation3_data4.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data4.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/validation3_data4.json_result b/semantic-model/datamodel/tools/tests/validation3_data4.json_result new file mode 100644 index 00000000..3092ce90 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_data4.json_result @@ -0,0 +1,10 @@ +Not Compliant: +[ + { + instancePath: '/hasFilter', + schemaPath: 'https://industry-fusion.org/base-objects/v0.1/link/type', + keyword: 'type', + params: { type: 'object' }, + message: 'must be object' + } +] diff --git a/semantic-model/datamodel/tools/tests/validation3_schema.json b/semantic-model/datamodel/tools/tests/validation3_schema.json new file mode 100644 index 00000000..ebc29cba --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation3_schema.json @@ -0,0 +1,79 @@ +[ + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/eclass%230173-1%2301-AKJ975%23017", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "eclass:0173-1#01-AKJ975#017" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"], + "allOf": [ + { + "$ref": "https://industry-fusion.org/base-objects/v0.1/cutter/properties" + }, + { + "$ref": "https://industry-fusion.org/base-objects/v0.1/cutter/relationships" + } + ] + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/cutter/relationships", + "title": "IFF template for cutter relationship", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "hasFilter": { + "relationship": "eclass:0173-1#01-ACK991#016", + "$ref": "https://industry-fusion.org/base-objects/v0.1/link" + } + } + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/link", + "title": "IFF template for cutter relationship", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "object": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}:[a-zA-Z0-9()+,\\-.:=@;$_!*']*[a-zA-Z0-9()+,\\-.:=@;$_!*']$" + } + }, + "required": ["object"] + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/cutter/properties", + "title": "Cutter properties", + "description": "Properties for class cutter", + "type": "object", + "properties": { + "machine_state": { + "type": "string", + "title": "Machine Status", + "description": "Current status of the machine (Online_Idle, Run, Online_Error, Online_Maintenance, Setup, Testing)", + "enum": [ + "Online_Idle", + "Run", + "Online_Error", + "Online_Maintenance", + "Setup", + "Testing" + ] + } + }, + "required": [ + "machine_state" + ] + } +] \ No newline at end of file From 4598084c355779213b2352987baee6ddfb6f4056 Mon Sep 17 00:00:00 2001 From: marcel Date: Tue, 7 Nov 2023 00:47:00 +0100 Subject: [PATCH 13/32] fixed test --- .../datamodel/tools/tests/validation3_data3.json | 2 +- .../tools/tests/validation3_data3.json_result | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/semantic-model/datamodel/tools/tests/validation3_data3.json b/semantic-model/datamodel/tools/tests/validation3_data3.json index 799eb2dd..292b16ee 100644 --- a/semantic-model/datamodel/tools/tests/validation3_data3.json +++ b/semantic-model/datamodel/tools/tests/validation3_data3.json @@ -5,5 +5,5 @@ "hasFilter": { "object": "urn:iff:filter:1" }, - "machine_state": "Testing" + "machine_state": "wrongState" } diff --git a/semantic-model/datamodel/tools/tests/validation3_data3.json_result b/semantic-model/datamodel/tools/tests/validation3_data3.json_result index ee71d9c0..c4011856 100644 --- a/semantic-model/datamodel/tools/tests/validation3_data3.json_result +++ b/semantic-model/datamodel/tools/tests/validation3_data3.json_result @@ -1 +1,10 @@ -The Datafile is compliant with Schema +Not Compliant: +[ + { + instancePath: '/machine_state', + schemaPath: 'https://industry-fusion.org/base-objects/v0.1/cutter/properties/properties/machine_state/enum', + keyword: 'enum', + params: { allowedValues: [Array] }, + message: 'must be equal to one of the allowed values' + } +] From 7425f445d722d9edd3b16633a79328a7667d34af Mon Sep 17 00:00:00 2001 From: marcel Date: Wed, 8 Nov 2023 00:32:12 +0100 Subject: [PATCH 14/32] Added jsonschema2shacl tool --- .../datamodel/tools/jsonschema2shacl.js | 311 ++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 semantic-model/datamodel/tools/jsonschema2shacl.js diff --git a/semantic-model/datamodel/tools/jsonschema2shacl.js b/semantic-model/datamodel/tools/jsonschema2shacl.js new file mode 100644 index 00000000..666a0671 --- /dev/null +++ b/semantic-model/datamodel/tools/jsonschema2shacl.js @@ -0,0 +1,311 @@ +/** +* Copyright (c) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +'use strict' + +const $RefParser = require('json-schema-ref-parser') +const $rdf = require('rdflib') +const fs = require('fs') +const yargs = require('yargs') +// const N3 = require('n3') +const url = require('url') +// const { DataFactory } = N3 +// const { namedNode, literal, blankNode, defaultGraph, quad } = DataFactory +// const { URL } = require('url') +const ContextParser = require('jsonld-context-parser').ContextParser +const ContextUtil = require('jsonld-context-parser').Util +const myParser = new ContextParser() + +const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') +const SHACL = $rdf.Namespace('http://www.w3.org/ns/shacl#') +const IFFK = $rdf.Namespace('https://industry-fusion.org/knowledge/v0.1/') +const argv = yargs + .command('$0', 'Converting an IFF Schema file for NGSI-LD objects into a SHACL constraint.') + .option('schema', { + alias: 's', + description: 'Schema File containing array of Schemas', + demandOption: true, + type: 'string' + }) + .option('schemaid', { + alias: 'i', + description: 'Schma-id of object to generate SHACL for', + demandOption: true, + type: 'string' + }) + .option('context', { + alias: 'c', + description: 'JSON-LD-Context', + demandOption: true, + type: 'string' + }) + .help() + .alias('help', 'h') + .argv + +// Read in an array of JSON-Schemas +const jsonSchemaText = fs.readFileSync(argv.s, 'utf8') +const jsonSchema = JSON.parse(jsonSchemaText) +let globalContext +let globalPrefixHash + +class NodeShape { + constructor (targetClass) { + this.targetClass = targetClass + this.properties = [] + } + + addPropertyShape (propertyShape) { + this.properties.push(propertyShape) + } + + get properties () { + return this._properties + } + + set properties (prop) { + this._properties = prop + } +} + +class PropertyShape { + constructor (mincount, maxcount, nodeKind, path, isProperty) { + this.mincount = mincount + this.maxcount = maxcount + this.nodeKind = nodeKind + this.path = path + this.constraints = [] + this.isProperty = isProperty + } + + addConstraint (constraint) { + this.constraints.push(constraint) + } + + set propertyNode (node) { + this._propertyNode = node + } + + get propertyNode () { + return this._propertyNode + } +} + +class Constraint { + constructor (type, params) { + this.type = type + this.params = params + } +} + +function scanNodeShape (typeschema) { + const id = typeschema.$id + + const nodeShape = new NodeShape(id) + scanProperties(nodeShape, typeschema) + return nodeShape +} + +function scanProperties (nodeShape, typeschema) { + let required = [] + if ('required' in typeschema) { + required = typeschema.required + } + if ('properties' in typeschema) { + Object.keys(typeschema.properties).forEach( + (property) => { + if (property === 'type' || property === 'id') { + return + } + let nodeKind = SHACL('Literal') + let klass = null + let isProperty = true + if ('relationship' in typeschema.properties[property]) { + nodeKind = SHACL('IRI') + klass = typeschema.properties[property].relationship + klass = globalContext.expandTerm(klass, true) + isProperty = false + } + let mincount = 0 + const maxcount = 1 + if (required.includes(property)) { + mincount = 1 + } + let path = property + if (!ContextUtil.isValidIri(path)) { + path = globalContext.expandTerm(path, true) + } + const propertyShape = new PropertyShape(mincount, maxcount, nodeKind, $rdf.sym(path), isProperty) + nodeShape.addPropertyShape(propertyShape) + if (klass !== null) { + propertyShape.addConstraint(new Constraint(SHACL('class'), $rdf.sym(klass))) + } + scanConstraints(propertyShape, typeschema.properties[property]) + }) + } + if ('allOf' in typeschema) { + typeschema.allOf.forEach((elem) => { + scanProperties(nodeShape, elem) + }) + } +} + +function scanConstraints (propertyShape, typeschema) { + if ('enum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('in'), typeschema.enum)) + } + if ('datatype' in typeschema) { + // datatype constraints are not used actively. It is not testing the value but only checks if the formal + // datatype "tag" conforms + // propertyShape.addConstraint(new Constraint(SHACL('datatype'), typeschema.datatype)) + } + if ('maxiumum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('maxInclusive'), typeschema.maximum)) + } + if ('miniumum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('minInclusive'), typeschema.minimum)) + } + if ('exclusiveMiniumum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('minExclusive'), typeschema.exclusiveMinimum)) + } + if ('exclusiveMaxiumum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('maxExclusive'), typeschema.exclusiveMaximum)) + } + if ('maxLength' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('maxLength'), typeschema.maxLength)) + } + if ('minLength' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('minLength'), typeschema.minLength)) + } +} + +function dumpShacl (nodeShape, store) { + dumpNodeShape(nodeShape, store) +} + +function dumpNodeShape (nodeShape, store) { + const nodeName = decodeURIComponent(globalContext.expandTerm(nodeShape.targetClass)) + const parsedUrl = new url.URL(nodeName) + const fragment = parsedUrl.hash.substring(1) + const shapeName = fragment + 'Shape' + store.add(IFFK(shapeName), RDF('type'), SHACL('NodeShape')) + store.add(IFFK(shapeName), SHACL('targetClass'), $rdf.sym(nodeName)) + nodeShape.properties.forEach((property) => { + const propNode = $rdf.blankNode() + property.propertyNode = propNode + store.add(IFFK(shapeName), SHACL('property'), propNode) + dumpPropertyShape(property, store) + }) +} + +function dumpPropertyShape (propertyShape, store) { + const propNode = propertyShape.propertyNode + store.add(propNode, SHACL('minCount'), propertyShape.mincount) + store.add(propNode, SHACL('maxCount'), propertyShape.maxcount) + store.add(propNode, SHACL('nodeKind'), SHACL('BlankNode')) + store.add(propNode, SHACL('path'), propertyShape.path) + const attributeNode = $rdf.blankNode() + store.add(propNode, SHACL('property'), attributeNode) + const ngsildPrefix = globalPrefixHash['ngsi-ld'] + const NGSILD = $rdf.Namespace(ngsildPrefix) + if (propertyShape.isProperty) { + store.add(attributeNode, SHACL('path'), NGSILD('hasValue')) + } else { + store.add(attributeNode, SHACL('path'), NGSILD('hasObject')) + } + store.add(attributeNode, SHACL('minCount'), 1) + store.add(attributeNode, SHACL('maxCount'), 1) + store.add(attributeNode, SHACL('nodeKind'), propertyShape.nodeKind) + const constraints = propertyShape.constraints + constraints.forEach((constraint) => { + store.add(attributeNode, constraint.type, constraint.params) + }) +} + +function encodeHash (id) { + const url = new URL(id) + const hash = encodeURIComponent(url.hash) + return `${url.protocol}//${url.hostname}${url.pathname}${hash}` +} + +function shaclize (schemas, id) { + id = encodeHash(id) + const store = new $rdf.IndexedFormula() + const typeschema = schemas.find((schema) => schema.$id === id) + const nodeShape = scanNodeShape(typeschema) + dumpShacl(nodeShape, store) + const serializer = new $rdf.Serializer(store) + serializer.setFlags('u') + serializer.setNamespaces(globalPrefixHash) + const turtle = serializer.statementsToN3(store.statementsMatching(undefined, undefined, undefined, undefined)) + console.log(turtle) +} + +async function loadContext (uriOrContext) { + const parseUrl = new url.URL(uriOrContext) + if (parseUrl.protocol === 'file:') { + uriOrContext = JSON.parse(fs.readFileSync(parseUrl.pathname, 'utf-8')) + } + const context = await myParser.parse(uriOrContext) + globalContext = context + const prefixHash = {} + Object.keys(context.getContextRaw()).filter((key) => key !== '@vocab').forEach((key) => { + const value = context.getContextRaw()[key] + if (typeof value === 'string') { + if (ContextUtil.isPrefixIriEndingWithGenDelim(value)) { + prefixHash[key] = value + } + } else if (typeof value === 'object') { + if (ContextUtil.isPrefixIriEndingWithGenDelim(value['@id'])) { + prefixHash[key] = value['@id'] + } + } + }) + globalPrefixHash = prefixHash +} + +(async (jsconSchema) => { + const myResolver = { + order: 1, + + canRead: function (file) { + return true + }, + + read: function (file, callback, $refs) { + return jsonSchema.find((schema) => schema.$id === file.url) + } + } + const options = { + resolve: { + file: false, + http: false, + test: myResolver + } + } + try { + const schema = await $RefParser.dereference(jsonSchema, options) + return schema + } catch (err) { + console.error(err) + } +})(jsonSchema) + .then(async (schema) => { + await loadContext(argv.c) + return schema + }) + .then(schema => { + shaclize(schema, argv.i) + }) From 1f76bf0267f1b84ad7b3b2ceeeaf9db45437650d Mon Sep 17 00:00:00 2001 From: marcel Date: Wed, 8 Nov 2023 16:03:26 +0100 Subject: [PATCH 15/32] Added tests for jsonld conversion --- .../datamodel/tools/tests/jsonld2jsonld/c0 | 38 +++++++++++ .../tools/tests/jsonld2jsonld/payload1.jsonld | 13 ++++ .../tests/jsonld2jsonld/payload1.jsonld_n_c0 | 53 +++++++++++++++ .../tests/jsonld2jsonld/payload1.jsonld_r_c0 | 66 +++++++++++++++++++ .../tests/jsonld2jsonld/payload1.jsonld_x_c0 | 37 +++++++++++ .../tools/tests/jsonld2jsonld/test.sh | 33 ++++++++++ .../datamodel/tools/tests/validation/test.sh | 28 ++++++++ .../{ => validation}/validation2_data1.json | 0 .../validation2_data1.json_id | 0 .../validation2_data1.json_result | 0 .../{ => validation}/validation2_schema.json | 0 .../{ => validation}/validation3_data1.json | 0 .../validation3_data1.json_id | 0 .../validation3_data1.json_result | 0 .../{ => validation}/validation3_data2.json | 0 .../validation3_data2.json_id | 0 .../validation3_data2.json_result | 0 .../{ => validation}/validation3_data3.json | 0 .../validation3_data3.json_id | 0 .../validation3_data3.json_result | 0 .../{ => validation}/validation3_data4.json | 0 .../validation3_data4.json_id | 0 .../validation3_data4.json_result | 0 .../{ => validation}/validation3_schema.json | 0 .../{ => validation}/validation_data1.json | 0 .../{ => validation}/validation_data1.json_id | 0 .../validation_data1.json_result | 0 .../{ => validation}/validation_data2.json | 0 .../{ => validation}/validation_data2.json_id | 0 .../validation_data2.json_result | 0 .../{ => validation}/validation_data3.json | 0 .../{ => validation}/validation_data3.json_id | 0 .../validation_data3.json_result | 0 .../{ => validation}/validation_schema.json | 0 34 files changed, 268 insertions(+) create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/c0 create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_n_c0 create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_r_c0 create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_x_c0 create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh create mode 100644 semantic-model/datamodel/tools/tests/validation/test.sh rename semantic-model/datamodel/tools/tests/{ => validation}/validation2_data1.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation2_data1.json_id (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation2_data1.json_result (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation2_schema.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data1.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data1.json_id (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data1.json_result (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data2.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data2.json_id (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data2.json_result (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data3.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data3.json_id (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data3.json_result (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data4.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data4.json_id (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_data4.json_result (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation3_schema.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_data1.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_data1.json_id (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_data1.json_result (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_data2.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_data2.json_id (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_data2.json_result (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_data3.json (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_data3.json_id (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_data3.json_result (100%) rename semantic-model/datamodel/tools/tests/{ => validation}/validation_schema.json (100%) diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/c0 new file mode 100644 index 00000000..6d7e122e --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/c0 @@ -0,0 +1,38 @@ +[ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld new file mode 100644 index 00000000..94610b65 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld @@ -0,0 +1,13 @@ +{ + "machine_state": "Testing", + "machine_state_from_smartbox": "Offline_Idle", + "hasFilter": { + "object": "urn:filter:1" + }, + "testiri": { + "@id": "iffk:testiri" + }, + "eclass:0173-1#02-AAH880#003": "10", + "id": "urn:x:1", + "type": "eclass:0173-1#01-AKJ975#017" +} diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_n_c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_n_c0 new file mode 100644 index 00000000..b6d02f84 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_n_c0 @@ -0,0 +1,53 @@ +[ + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } + ], + "id": "urn:x:1", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": { + "object": "urn:filter:1" + }, + "machine_state": "Testing", + "machine_state_from_smartbox": "Offline_Idle", + "testiri": { + "id": "iffk:testiri" + }, + "eclass:0173-1#02-AAH880#003": "10" + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_r_c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_r_c0 new file mode 100644 index 00000000..da8c4394 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_r_c0 @@ -0,0 +1,66 @@ +[ + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } + ], + "id": "urn:x:1", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": { + "type": "Relationship", + "object": "urn:filter:1" + }, + "machine_state": { + "type": "Property", + "value": "Testing" + }, + "machine_state_from_smartbox": { + "type": "Property", + "value": "Offline_Idle" + }, + "testiri": { + "type": "Property", + "value": { + "id": "iffk:testiri" + } + }, + "eclass:0173-1#02-AAH880#003": { + "type": "Property", + "value": "10" + } + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_x_c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_x_c0 new file mode 100644 index 00000000..72713926 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload1.jsonld_x_c0 @@ -0,0 +1,37 @@ +[ + { + "https://industry-fusion.org/eclass#0173-1#02-AAH880#003": [ + { + "@value": "10" + } + ], + "https://industry-fusion.org/base/v0.1/hasFilter": [ + { + "https://uri.etsi.org/ngsi-ld/hasObject": [ + { + "@id": "urn:filter:1" + } + ] + } + ], + "@id": "urn:x:1", + "https://industry-fusion.org/base/v0.1/machine_state": [ + { + "@value": "Testing" + } + ], + "https://industry-fusion.org/base/v0.1/machine_state_from_smartbox": [ + { + "@value": "Offline_Idle" + } + ], + "https://industry-fusion.org/base/v0.1/testiri": [ + { + "@id": "https://industry-fusion.org/knowledge/v0.1/testiri" + } + ], + "@type": [ + "https://industry-fusion.org/eclass#0173-1#01-AKJ975#017" + ] + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh b/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh new file mode 100644 index 00000000..54810e1c --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#for testdir in $(ls payload*.jsonld_?_c*); do +while IFS='_' read -ra ADDR; do + payload=${ADDR[0]} + switch=${ADDR[1]} + context=${ADDR[2]} + comparewith=${payload}_${switch}_${context} + echo comparewith $comparewith + command="node ../../jsonldConverter.js $payload -$switch -c file://$PWD/$context" + echo Executing: $command + $command | diff ${comparewith} - + #$command + #for i in "${ADDR[@]}"; do + # process "$i" + # echo $i + #done +done <<< $(ls payload*.jsonld_?_c*) diff --git a/semantic-model/datamodel/tools/tests/validation/test.sh b/semantic-model/datamodel/tools/tests/validation/test.sh new file mode 100644 index 00000000..50fcd9b7 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/validation/test.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +while IFS='_' read -ra ADDR; do + testname=${ADDR[0]} + schemaname=${testname}_schema.json + dataname=${testname}_${ADDR[1]} + resultname=${dataname}_result + idname=${dataname}_id + command="node ../validate.js -s ${schemaname} -d ${dataname} -i $(cat ${idname})" + echo -n "Executing test $command ... " + $command | diff ${resultname} - + echo OK +done <<< $(ls validation*_data*.json) diff --git a/semantic-model/datamodel/tools/tests/validation2_data1.json b/semantic-model/datamodel/tools/tests/validation/validation2_data1.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation2_data1.json rename to semantic-model/datamodel/tools/tests/validation/validation2_data1.json diff --git a/semantic-model/datamodel/tools/tests/validation2_data1.json_id b/semantic-model/datamodel/tools/tests/validation/validation2_data1.json_id similarity index 100% rename from semantic-model/datamodel/tools/tests/validation2_data1.json_id rename to semantic-model/datamodel/tools/tests/validation/validation2_data1.json_id diff --git a/semantic-model/datamodel/tools/tests/validation2_data1.json_result b/semantic-model/datamodel/tools/tests/validation/validation2_data1.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validation2_data1.json_result rename to semantic-model/datamodel/tools/tests/validation/validation2_data1.json_result diff --git a/semantic-model/datamodel/tools/tests/validation2_schema.json b/semantic-model/datamodel/tools/tests/validation/validation2_schema.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation2_schema.json rename to semantic-model/datamodel/tools/tests/validation/validation2_schema.json diff --git a/semantic-model/datamodel/tools/tests/validation3_data1.json b/semantic-model/datamodel/tools/tests/validation/validation3_data1.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data1.json rename to semantic-model/datamodel/tools/tests/validation/validation3_data1.json diff --git a/semantic-model/datamodel/tools/tests/validation3_data1.json_id b/semantic-model/datamodel/tools/tests/validation/validation3_data1.json_id similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data1.json_id rename to semantic-model/datamodel/tools/tests/validation/validation3_data1.json_id diff --git a/semantic-model/datamodel/tools/tests/validation3_data1.json_result b/semantic-model/datamodel/tools/tests/validation/validation3_data1.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data1.json_result rename to semantic-model/datamodel/tools/tests/validation/validation3_data1.json_result diff --git a/semantic-model/datamodel/tools/tests/validation3_data2.json b/semantic-model/datamodel/tools/tests/validation/validation3_data2.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data2.json rename to semantic-model/datamodel/tools/tests/validation/validation3_data2.json diff --git a/semantic-model/datamodel/tools/tests/validation3_data2.json_id b/semantic-model/datamodel/tools/tests/validation/validation3_data2.json_id similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data2.json_id rename to semantic-model/datamodel/tools/tests/validation/validation3_data2.json_id diff --git a/semantic-model/datamodel/tools/tests/validation3_data2.json_result b/semantic-model/datamodel/tools/tests/validation/validation3_data2.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data2.json_result rename to semantic-model/datamodel/tools/tests/validation/validation3_data2.json_result diff --git a/semantic-model/datamodel/tools/tests/validation3_data3.json b/semantic-model/datamodel/tools/tests/validation/validation3_data3.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data3.json rename to semantic-model/datamodel/tools/tests/validation/validation3_data3.json diff --git a/semantic-model/datamodel/tools/tests/validation3_data3.json_id b/semantic-model/datamodel/tools/tests/validation/validation3_data3.json_id similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data3.json_id rename to semantic-model/datamodel/tools/tests/validation/validation3_data3.json_id diff --git a/semantic-model/datamodel/tools/tests/validation3_data3.json_result b/semantic-model/datamodel/tools/tests/validation/validation3_data3.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data3.json_result rename to semantic-model/datamodel/tools/tests/validation/validation3_data3.json_result diff --git a/semantic-model/datamodel/tools/tests/validation3_data4.json b/semantic-model/datamodel/tools/tests/validation/validation3_data4.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data4.json rename to semantic-model/datamodel/tools/tests/validation/validation3_data4.json diff --git a/semantic-model/datamodel/tools/tests/validation3_data4.json_id b/semantic-model/datamodel/tools/tests/validation/validation3_data4.json_id similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data4.json_id rename to semantic-model/datamodel/tools/tests/validation/validation3_data4.json_id diff --git a/semantic-model/datamodel/tools/tests/validation3_data4.json_result b/semantic-model/datamodel/tools/tests/validation/validation3_data4.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_data4.json_result rename to semantic-model/datamodel/tools/tests/validation/validation3_data4.json_result diff --git a/semantic-model/datamodel/tools/tests/validation3_schema.json b/semantic-model/datamodel/tools/tests/validation/validation3_schema.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation3_schema.json rename to semantic-model/datamodel/tools/tests/validation/validation3_schema.json diff --git a/semantic-model/datamodel/tools/tests/validation_data1.json b/semantic-model/datamodel/tools/tests/validation/validation_data1.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_data1.json rename to semantic-model/datamodel/tools/tests/validation/validation_data1.json diff --git a/semantic-model/datamodel/tools/tests/validation_data1.json_id b/semantic-model/datamodel/tools/tests/validation/validation_data1.json_id similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_data1.json_id rename to semantic-model/datamodel/tools/tests/validation/validation_data1.json_id diff --git a/semantic-model/datamodel/tools/tests/validation_data1.json_result b/semantic-model/datamodel/tools/tests/validation/validation_data1.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_data1.json_result rename to semantic-model/datamodel/tools/tests/validation/validation_data1.json_result diff --git a/semantic-model/datamodel/tools/tests/validation_data2.json b/semantic-model/datamodel/tools/tests/validation/validation_data2.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_data2.json rename to semantic-model/datamodel/tools/tests/validation/validation_data2.json diff --git a/semantic-model/datamodel/tools/tests/validation_data2.json_id b/semantic-model/datamodel/tools/tests/validation/validation_data2.json_id similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_data2.json_id rename to semantic-model/datamodel/tools/tests/validation/validation_data2.json_id diff --git a/semantic-model/datamodel/tools/tests/validation_data2.json_result b/semantic-model/datamodel/tools/tests/validation/validation_data2.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_data2.json_result rename to semantic-model/datamodel/tools/tests/validation/validation_data2.json_result diff --git a/semantic-model/datamodel/tools/tests/validation_data3.json b/semantic-model/datamodel/tools/tests/validation/validation_data3.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_data3.json rename to semantic-model/datamodel/tools/tests/validation/validation_data3.json diff --git a/semantic-model/datamodel/tools/tests/validation_data3.json_id b/semantic-model/datamodel/tools/tests/validation/validation_data3.json_id similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_data3.json_id rename to semantic-model/datamodel/tools/tests/validation/validation_data3.json_id diff --git a/semantic-model/datamodel/tools/tests/validation_data3.json_result b/semantic-model/datamodel/tools/tests/validation/validation_data3.json_result similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_data3.json_result rename to semantic-model/datamodel/tools/tests/validation/validation_data3.json_result diff --git a/semantic-model/datamodel/tools/tests/validation_schema.json b/semantic-model/datamodel/tools/tests/validation/validation_schema.json similarity index 100% rename from semantic-model/datamodel/tools/tests/validation_schema.json rename to semantic-model/datamodel/tools/tests/validation/validation_schema.json From 38c76a544d0ee469e7f65ab6d7a26b313f1847e8 Mon Sep 17 00:00:00 2001 From: marcel Date: Wed, 8 Nov 2023 16:31:30 +0100 Subject: [PATCH 16/32] more tests --- .../tools/tests/jsonld2jsonld/payload2.jsonld | 37 ++++++ .../tests/jsonld2jsonld/payload2.jsonld_n_c0 | 53 ++++++++ .../tests/jsonld2jsonld/payload2.jsonld_r_c0 | 66 ++++++++++ .../tests/jsonld2jsonld/payload2.jsonld_x_c0 | 37 ++++++ .../tools/tests/jsonld2jsonld/payload3.jsonld | 24 ++++ .../tests/jsonld2jsonld/payload3.jsonld_n_c0 | 97 +++++++++++++++ .../tests/jsonld2jsonld/payload3.jsonld_r_c0 | 116 ++++++++++++++++++ .../tests/jsonld2jsonld/payload3.jsonld_x_c0 | 63 ++++++++++ 8 files changed, 493 insertions(+) create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_n_c0 create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_r_c0 create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_x_c0 create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_n_c0 create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_r_c0 create mode 100644 semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_x_c0 diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld new file mode 100644 index 00000000..b3840783 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld @@ -0,0 +1,37 @@ +[ + { + "https://industry-fusion.org/eclass#0173-1#02-AAH880#003": [ + { + "@value": "10" + } + ], + "https://industry-fusion.org/base/v0.1/hasFilter": [ + { + "https://uri.etsi.org/ngsi-ld/hasObject": [ + { + "@id": "urn:filter:1" + } + ] + } + ], + "@id": "urn:x:1", + "https://industry-fusion.org/base/v0.1/machine_state": [ + { + "@value": "Testing" + } + ], + "https://industry-fusion.org/base/v0.1/machine_state_from_smartbox": [ + { + "@value": "Offline_Idle" + } + ], + "https://industry-fusion.org/base/v0.1/testiri": [ + { + "@id": "https://industry-fusion.org/knowledge/v0.1/testiri" + } + ], + "@type": [ + "https://industry-fusion.org/knowledge/v0.1/0173-1#01-AKJ975#017" + ] + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_n_c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_n_c0 new file mode 100644 index 00000000..6b6f244c --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_n_c0 @@ -0,0 +1,53 @@ +[ + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } + ], + "id": "urn:x:1", + "type": "iffk:0173-1#01-AKJ975#017", + "hasFilter": { + "object": "urn:filter:1" + }, + "machine_state": "Testing", + "machine_state_from_smartbox": "Offline_Idle", + "testiri": { + "id": "iffk:testiri" + }, + "eclass:0173-1#02-AAH880#003": "10" + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_r_c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_r_c0 new file mode 100644 index 00000000..bd0638c9 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_r_c0 @@ -0,0 +1,66 @@ +[ + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } + ], + "id": "urn:x:1", + "type": "iffk:0173-1#01-AKJ975#017", + "hasFilter": { + "type": "Relationship", + "object": "urn:filter:1" + }, + "machine_state": { + "type": "Property", + "value": "Testing" + }, + "machine_state_from_smartbox": { + "type": "Property", + "value": "Offline_Idle" + }, + "testiri": { + "type": "Property", + "value": { + "id": "iffk:testiri" + } + }, + "eclass:0173-1#02-AAH880#003": { + "type": "Property", + "value": "10" + } + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_x_c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_x_c0 new file mode 100644 index 00000000..478de1b9 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload2.jsonld_x_c0 @@ -0,0 +1,37 @@ +[ + { + "@id": "urn:x:1", + "@type": [ + "https://industry-fusion.org/knowledge/v0.1/0173-1#01-AKJ975#017" + ], + "https://industry-fusion.org/base/v0.1/hasFilter": [ + { + "https://uri.etsi.org/ngsi-ld/hasObject": [ + { + "@id": "urn:filter:1" + } + ] + } + ], + "https://industry-fusion.org/base/v0.1/machine_state": [ + { + "@value": "Testing" + } + ], + "https://industry-fusion.org/base/v0.1/machine_state_from_smartbox": [ + { + "@value": "Offline_Idle" + } + ], + "https://industry-fusion.org/base/v0.1/testiri": [ + { + "@id": "https://industry-fusion.org/knowledge/v0.1/testiri" + } + ], + "https://industry-fusion.org/eclass#0173-1#02-AAH880#003": [ + { + "@value": "10" + } + ] + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld new file mode 100644 index 00000000..36d9a67c --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld @@ -0,0 +1,24 @@ +[{ + "machine_state": "Testing", + "machine_state_from_smartbox": "Offline_Idle", + "hasFilter": { + "type": "Relationship", + "object": "urn:filter:1" + }, + "testiri": { + "@id": "iffk:testiri" + }, + "eclass:0173-1#02-AAH880#003": { + "value": "10", + "type": "Property" + }, + "id": "urn:x:1", + "type": "eclass:0173-1#01-AKJ975#017" +}, +{ + "machine_state": "Testing", + "machine_state_from_smartbox": "Offline_Idle", + "id": "urn:filter:1", + "type": "eclass:0173-1#01-ACK991#016" +} +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_n_c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_n_c0 new file mode 100644 index 00000000..a9993a77 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_n_c0 @@ -0,0 +1,97 @@ +[ + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } + ], + "id": "urn:x:1", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": { + "object": "urn:filter:1" + }, + "machine_state": "Testing", + "machine_state_from_smartbox": "Offline_Idle", + "testiri": { + "id": "iffk:testiri" + }, + "eclass:0173-1#02-AAH880#003": "10" + }, + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } + ], + "id": "urn:filter:1", + "type": "eclass:0173-1#01-ACK991#016", + "machine_state": "Testing", + "machine_state_from_smartbox": "Offline_Idle" + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_r_c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_r_c0 new file mode 100644 index 00000000..c3cd9469 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_r_c0 @@ -0,0 +1,116 @@ +[ + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } + ], + "id": "urn:x:1", + "type": "eclass:0173-1#01-AKJ975#017", + "hasFilter": { + "type": "Relationship", + "object": "urn:filter:1" + }, + "machine_state": { + "type": "Property", + "value": "Testing" + }, + "machine_state_from_smartbox": { + "type": "Property", + "value": "Offline_Idle" + }, + "testiri": { + "type": "Property", + "value": { + "id": "iffk:testiri" + } + }, + "eclass:0173-1#02-AAH880#003": { + "type": "Property", + "value": "10" + } + }, + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } + ], + "id": "urn:filter:1", + "type": "eclass:0173-1#01-ACK991#016", + "machine_state": { + "type": "Property", + "value": "Testing" + }, + "machine_state_from_smartbox": { + "type": "Property", + "value": "Offline_Idle" + } + } +] diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_x_c0 b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_x_c0 new file mode 100644 index 00000000..80d4b3f8 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/payload3.jsonld_x_c0 @@ -0,0 +1,63 @@ +[ + { + "https://industry-fusion.org/eclass#0173-1#02-AAH880#003": [ + { + "@type": [ + "https://uri.etsi.org/ngsi-ld/Property" + ], + "https://uri.etsi.org/ngsi-ld/hasValue": [ + { + "@value": "10" + } + ] + } + ], + "https://industry-fusion.org/base/v0.1/hasFilter": [ + { + "https://uri.etsi.org/ngsi-ld/hasObject": [ + { + "@id": "urn:filter:1" + } + ], + "@type": [ + "https://uri.etsi.org/ngsi-ld/Relationship" + ] + } + ], + "@id": "urn:x:1", + "https://industry-fusion.org/base/v0.1/machine_state": [ + { + "@value": "Testing" + } + ], + "https://industry-fusion.org/base/v0.1/machine_state_from_smartbox": [ + { + "@value": "Offline_Idle" + } + ], + "https://industry-fusion.org/base/v0.1/testiri": [ + { + "@id": "https://industry-fusion.org/knowledge/v0.1/testiri" + } + ], + "@type": [ + "https://industry-fusion.org/eclass#0173-1#01-AKJ975#017" + ] + }, + { + "@id": "urn:filter:1", + "https://industry-fusion.org/base/v0.1/machine_state": [ + { + "@value": "Testing" + } + ], + "https://industry-fusion.org/base/v0.1/machine_state_from_smartbox": [ + { + "@value": "Offline_Idle" + } + ], + "@type": [ + "https://industry-fusion.org/eclass#0173-1#01-ACK991#016" + ] + } +] From ac624deaa93c88e2a0c4fabb7114ec32f6638e8e Mon Sep 17 00:00:00 2001 From: marcel Date: Wed, 8 Nov 2023 20:33:19 +0100 Subject: [PATCH 17/32] added Makefile --- semantic-model/datamodel/tools/Makefile | 27 ++ .../datamodel/tools/jsonldConverter.js | 270 ++++++++++++++++++ semantic-model/datamodel/tools/package.json | 5 +- .../datamodel/tools/tests/schema2shacl/c0 | 38 +++ .../tools/tests/schema2shacl/schema1_c0.json | 17 ++ .../tests/schema2shacl/schema1_c0.json_id | 1 + .../tests/schema2shacl/schema1_c0.json_result | 7 + .../tools/tests/schema2shacl/schema2_c0.json | 17 ++ .../tests/schema2shacl/schema2_c0.json_id | 1 + .../tests/schema2shacl/schema2_c0.json_result | 6 + .../tools/tests/schema2shacl/schema3_c0.json | 79 +++++ .../tests/schema2shacl/schema3_c0.json_id | 1 + .../tests/schema2shacl/schema3_c0.json_result | 40 +++ .../tools/tests/schema2shacl/test.sh | 28 ++ .../datamodel/tools/tests/validation/test.sh | 2 +- 15 files changed, 537 insertions(+), 2 deletions(-) create mode 100644 semantic-model/datamodel/tools/Makefile create mode 100644 semantic-model/datamodel/tools/jsonldConverter.js create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/c0 create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_id create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_result create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json_id create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json_result create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json_id create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json_result create mode 100644 semantic-model/datamodel/tools/tests/schema2shacl/test.sh diff --git a/semantic-model/datamodel/tools/Makefile b/semantic-model/datamodel/tools/Makefile new file mode 100644 index 00000000..f828b05d --- /dev/null +++ b/semantic-model/datamodel/tools/Makefile @@ -0,0 +1,27 @@ +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +setup: + pip3 install pyshacl + npm install + +lint: + npm run lint + shellcheck tests/*/test.sh + +test: + npm run test \ No newline at end of file diff --git a/semantic-model/datamodel/tools/jsonldConverter.js b/semantic-model/datamodel/tools/jsonldConverter.js new file mode 100644 index 00000000..770df5b4 --- /dev/null +++ b/semantic-model/datamodel/tools/jsonldConverter.js @@ -0,0 +1,270 @@ +/** +* Copyright (c) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +'use strict' + +// const $RefParser = require('json-schema-ref-parser'); +// const $rdf = require('rdflib') +const fs = require('fs') +const yargs = require('yargs') +// const path = require('path') +// const N3 = require('n3') +const url = require('url') +// const { DataFactory } = N3 +// const { namedNode, literal, blankNode, defaultGraph, quad } = DataFactory; +// const { URL } = require('url'); // Import the URL module +// const ContextParser = require('jsonld-context-parser').ContextParser +// const ContextUtil = require('jsonld-context-parser').Util; +// const myParser = new ContextParser() +const jsonld = require('jsonld') +// const exp = require('constants'); + +// const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#'); +// const SHACL = $rdf.Namespace('http://www.w3.org/ns/shacl#'); +// const IFFK = $rdf.Namespace('https://industry-fusion.org/knowledge/v0.1/'); +let jsonFileName +const argv = yargs + .option('concise', { + alias: 'n', + description: 'Create concise/compacted from', + demandOption: false, + type: 'boolean' + }) + .option('expand', { + alias: 'x', + description: 'Create expanded from', + demandOption: false, + type: 'boolean' + }) + .option('normalize', { + alias: 'r', + description: 'Create normalized from', + demandOption: false, + type: 'boolean' + }) + .option('context', { + alias: 'c', + description: 'JSON-LD-Context', + demandOption: false, + type: 'string' + }) + .command({ + command: '$0 ', + describe: 'Convert a JSON-LD file into different normal forms.', + handler: (argv) => { + const { filename } = argv + jsonFileName = filename + } + }) + .help() + .alias('help', 'h') + .argv + +const jsonText = fs.readFileSync(jsonFileName, 'utf8') +const jsonObj = JSON.parse(jsonText) +let jsonArr + +if (!(argv.x === undefined) && !(argv.n === undefined)) { + console.error('Expand and Concise are mutally exclusive. Bye!') + process.exit(1) +} +if (!(argv.r === undefined) && !(argv.n === undefined)) { + console.error('Normalized and Concise are mutally exclusive. Bye!') + process.exit(1) +} +if (!(argv.x === undefined) && !(argv.r === undefined)) { + console.error('Normalized and Expanded are mutally exclusive. Bye!') + process.exit(1) +} +if (argv.x === undefined && argv.r === undefined && argv.n === undefined) { + console.error('No processing switch selected. Bye!') + process.exit(1) +} + +if (!Array.isArray(jsonObj)) { + jsonArr = [jsonObj] +} else { + jsonArr = jsonObj +} + +function assertNoContext (jsonObj) { + if (Array.isArray(jsonObj)) { + jsonObj.forEach((obj) => { + if ('@context' in obj) { + console.error('Error: @contex found in json-object. Only fully expanded objects without @context can be compacted. Exiting!') + process.exit(1) + } + }) + } else { + if ('@context' in jsonObj) { + console.error('Error: @context found in json-object. Only fully expanded objects without @context can be compacted. Exiting!') + process.exit(1) + } + } +} + +// Check if every object in array whether it has Context +function hasContexts (jsonArr) { + jsonArr.forEach(jsonObj => { + if (!('@context' in jsonObj)) { + return false + } + }) + return true +} + +function loadContextFromFile (fileName) { + const context = fs.readFileSync(fileName, 'utf8') + const contextParsed = JSON.parse(context) + return contextParsed +} + +// Merge Contexts +function mergeContexts (jsonArr, context) { + function mergeContext (localContext, context) { + let mergedContext = [] + if (!Array.isArray(localContext) && localContext !== undefined) { + mergedContext = [localContext] + } + if (context === undefined) { + if (mergedContext.length === 0) { + return null + } + return mergedContext + } else if (!Array.isArray(context)) { + context = [context] + } + context.forEach(c => { + if (typeof (c) !== 'string' || mergedContext.find(x => c === x) === undefined) { + mergedContext.push(c) + } + }) + return mergedContext + } + if (context !== undefined) { + const parseContextUrl = new url.parse(context, true) + if (parseContextUrl.protocol === 'file:') { + context = loadContextFromFile(parseContextUrl.path) + } + } + return jsonArr.map(jsonObj => { + const localContext = jsonObj['@context'] + return mergeContext(localContext, context) + }) +} + +function conciseExpandedForm (expanded) { + function filterAttribute (attr) { + if (typeof (attr) === 'object') { + if ('@type' in attr && (attr['@type'][0] === 'https://uri.etsi.org/ngsi-ld/Property' || + attr['@type'][0] === 'https://uri.etsi.org/ngsi-ld/Relationship')) { + delete attr['@type'] + } + if ('https://uri.etsi.org/ngsi-ld/hasValue' in attr) { + attr['@value'] = attr['https://uri.etsi.org/ngsi-ld/hasValue'][0]['@value'] + delete attr['https://uri.etsi.org/ngsi-ld/hasValue'] + } + } + } + expanded.forEach(c => { + Object.keys(c).forEach(key => { + if (Array.isArray(c[key])) { + c[key].forEach(a => filterAttribute(a)) + } else { + filterAttribute(c[key]) + } + }) + }) + return expanded +} + +function normalizeExpandedForm (expanded) { + function extendAttribute (attr) { + if (typeof (attr) === 'object') { + if (!('@type' in attr)) { + if ('https://uri.etsi.org/ngsi-ld/hasValue' in attr || '@value' in attr || '@id' in attr) { + attr['@type'] = 'https://uri.etsi.org/ngsi-ld/Property' + } else if ('https://uri.etsi.org/ngsi-ld/hasObject' in attr) { + attr['@type'] = 'https://uri.etsi.org/ngsi-ld/Relationship' + } + if ('@value' in attr) { + attr['https://uri.etsi.org/ngsi-ld/hasValue'] = attr['@value'] + delete attr['@value'] + } else if ('@id' in attr) { + attr['https://uri.etsi.org/ngsi-ld/hasValue'] = { '@id': attr['@id'] } + delete attr['@id'] + } + } + } + } + expanded.forEach(c => { + Object.keys(c).forEach(key => { + if (Array.isArray(c[key])) { + c[key].forEach(a => extendAttribute(a)) + } else { + extendAttribute(c[key]) + } + }) + }) + return expanded +} + +async function expand (objArr, contextArr) { + const expanded = await Promise.all(objArr.map(async (jsonObj, index) => { + jsonObj['@context'] = contextArr[index] + const res = await jsonld.expand(jsonObj) + return res[0] + })) + return expanded +} + +async function compact (objArr, contextArr) { + return await Promise.all(objArr.map(async (jsonObj, index) => jsonld.compact(jsonObj, contextArr[index]))) +} + +(async (jsonArr) => { + if (!(argv.n === undefined)) { + const mergedContexts = mergeContexts(jsonArr, argv.c) + if (mergedContexts !== undefined && mergedContexts.find(x => x === null)) { + console.error('Error: For Compaction, context must be either defined in all objects or externally. Exiting!') + process.exit(1) + } + // Compaction to find Properties in compacted form + const expanded = await expand(jsonArr, mergedContexts) + const concised = conciseExpandedForm(expanded) + const compacted = await compact(concised, mergedContexts) + console.log(JSON.stringify(compacted, null, 2)) + } + if (!(argv.x === undefined)) { + const mergedContexts = mergeContexts(jsonArr, argv.c) + if (mergedContexts !== undefined && mergedContexts.find(x => x === null)) { + console.error('Error: For Extraction, context must be either defined in all objects or externally. Exiting!') + process.exit(1) + } + const expanded = await expand(jsonArr, mergedContexts) + console.log(JSON.stringify(expanded, null, 2)) + } + if (!(argv.r === undefined)) { + const mergedContexts = mergeContexts(jsonArr, argv.c) + if (mergedContexts !== undefined && mergedContexts.find(x => x === null)) { + console.error('Error: For Normalization, context must be either defined in all objects or externally. Exiting!') + process.exit(1) + } + const expanded = await expand(jsonArr, mergedContexts) + const normalized = normalizeExpandedForm(expanded) + const compacted = await compact(normalized, mergedContexts) + console.log(JSON.stringify(compacted, null, 2)) + } +})(jsonArr) diff --git a/semantic-model/datamodel/tools/package.json b/semantic-model/datamodel/tools/package.json index 72638297..60da19c7 100644 --- a/semantic-model/datamodel/tools/package.json +++ b/semantic-model/datamodel/tools/package.json @@ -4,13 +4,16 @@ "description": "This package contains tools for validating NGSI-LD data for the Process Data Twin (PDT)", "main": "validate.js", "scripts": { - "test": "cd tests && bash ./test_validation.sh && exit 0", + "test": "cd tests && for dir in $(ls); do (cd $dir; bash ./test.sh); done && exit 0", "lint": "./node_modules/.bin/eslint *.js" }, "author": "Marcel Wagner", "license": "Apache-2.0", "dependencies": { "ajv": "^8.12.0", + "json-schema-ref-parser": "^9.0.9", + "jsonld-context-parser": "^2.4.0", + "rdflib": "^2.2.32", "yargs": "^17.7.2" }, "devDependencies": { diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/c0 b/semantic-model/datamodel/tools/tests/schema2shacl/c0 new file mode 100644 index 00000000..6d7e122e --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/c0 @@ -0,0 +1,38 @@ +[ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v0.1/", + "eclass": { + "@id": "https://industry-fusion.org/eclass#", + "@prefix": true + }, + "xsd": { + "@id": "http://www.w3.org/2001/XMLSchema#", + "@prefix": true + }, + "iffb": { + "@id": "https://industry-fusion.org/base/v0.1/", + "@prefix": true + }, + "iffk": { + "@id": "https://industry-fusion.org/knowledge/v0.1/", + "@prefix": true + }, + "rdf": { + "@id": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "@prefix": true + }, + "rdfs": { + "@id": "http://www.w3.org/2000/01/rdf-schema#", + "@prefix": true + }, + "schema": { + "@id": "http://schema.org/", + "@prefix": true + }, + "sh": { + "@id": "http://www.w3.org/ns/shacl#", + "@prefix": true + } + } +] diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json b/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json new file mode 100644 index 00000000..35c93de9 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json @@ -0,0 +1,17 @@ +[{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base/v0.1/Plasmacutter", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "plasmacutter" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"] +}] \ No newline at end of file diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_id b/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_id new file mode 100644 index 00000000..1c63c300 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/base/v0.1/Plasmacutter diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_result b/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_result new file mode 100644 index 00000000..a1107ad7 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_result @@ -0,0 +1,7 @@ +@prefix iffb: . +@prefix iffk: . +@prefix sh: . + +iffk:Shape a sh:NodeShape; sh:targetClass iffb:Plasmacutter. + + diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json b/semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json new file mode 100644 index 00000000..72149bdd --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json @@ -0,0 +1,17 @@ +[{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/eclass%230173-1%2301-AKJ975%23017", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "eclass:0173-1#01-AKJ975#017" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"] +}] \ No newline at end of file diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json_id b/semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json_result b/semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json_result new file mode 100644 index 00000000..7722ec68 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema2_c0.json_result @@ -0,0 +1,6 @@ +@prefix sh: . + + + a sh:NodeShape; + sh:targetClass . + diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json b/semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json new file mode 100644 index 00000000..ebc29cba --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json @@ -0,0 +1,79 @@ +[ + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/eclass%230173-1%2301-AKJ975%23017", + "title": "Cutter", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "type": { + "const": "eclass:0173-1#01-AKJ975#017" + }, + "id": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + }, + "required": ["type", "id"], + "allOf": [ + { + "$ref": "https://industry-fusion.org/base-objects/v0.1/cutter/properties" + }, + { + "$ref": "https://industry-fusion.org/base-objects/v0.1/cutter/relationships" + } + ] + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/cutter/relationships", + "title": "IFF template for cutter relationship", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "hasFilter": { + "relationship": "eclass:0173-1#01-ACK991#016", + "$ref": "https://industry-fusion.org/base-objects/v0.1/link" + } + } + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/link", + "title": "IFF template for cutter relationship", + "description": "Cutter template for IFF", + "type": "object", + "properties": { + "object": { + "type": "string", + "pattern": "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}:[a-zA-Z0-9()+,\\-.:=@;$_!*']*[a-zA-Z0-9()+,\\-.:=@;$_!*']$" + } + }, + "required": ["object"] + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://industry-fusion.org/base-objects/v0.1/cutter/properties", + "title": "Cutter properties", + "description": "Properties for class cutter", + "type": "object", + "properties": { + "machine_state": { + "type": "string", + "title": "Machine Status", + "description": "Current status of the machine (Online_Idle, Run, Online_Error, Online_Maintenance, Setup, Testing)", + "enum": [ + "Online_Idle", + "Run", + "Online_Error", + "Online_Maintenance", + "Setup", + "Testing" + ] + } + }, + "required": [ + "machine_state" + ] + } +] \ No newline at end of file diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json_id b/semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json_id new file mode 100644 index 00000000..a1064ee6 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json_id @@ -0,0 +1 @@ +https://industry-fusion.org/eclass#0173-1#01-AKJ975#017 diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json_result b/semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json_result new file mode 100644 index 00000000..4e5cf580 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json_result @@ -0,0 +1,40 @@ +@prefix iffb: . +@prefix sh: . +@prefix ngsi-ld: . + + + a sh:NodeShape; + sh:property + [ + sh:maxCount 1; + sh:minCount 0; + sh:nodeKind sh:BlankNode; + sh:path iffb:hasFilter; + sh:property + [ + sh:class + ; + sh:maxCount 1; + sh:minCount 1; + sh:nodeKind sh:IRI; + sh:path ngsi-ld:hasObject + ] + ], + [ + sh:maxCount 1; + sh:minCount 1; + sh:nodeKind sh:BlankNode; + sh:path iffb:machine_state; + sh:property + [ + sh:in + ( "Online_Idle" "Run" "Online_Error" + "Online_Maintenance" "Setup" "Testing" ); + sh:maxCount 1; + sh:minCount 1; + sh:nodeKind sh:Literal; + sh:path ngsi-ld:hasValue + ] + ]; + sh:targetClass . + diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/test.sh b/semantic-model/datamodel/tools/tests/schema2shacl/test.sh new file mode 100644 index 00000000..34bbfd9b --- /dev/null +++ b/semantic-model/datamodel/tools/tests/schema2shacl/test.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#for testdir in $(ls payload*.jsonld_?_c*); do +while IFS='_' read -ra ADDR; do + schemaname=${ADDR[0]} + context=${ADDR[1]%.json} + file=${schemaname}_${context}.json + comparewith=${file}_result + id=${file}_id + command="node ../../jsonschema2shacl.js -s $file -c file://$PWD/$context -i $(cat $id)" + echo Executing: $command + $command | diff ${comparewith} - +done <<< $(ls schema*_*.json) diff --git a/semantic-model/datamodel/tools/tests/validation/test.sh b/semantic-model/datamodel/tools/tests/validation/test.sh index 50fcd9b7..31be7beb 100644 --- a/semantic-model/datamodel/tools/tests/validation/test.sh +++ b/semantic-model/datamodel/tools/tests/validation/test.sh @@ -21,7 +21,7 @@ while IFS='_' read -ra ADDR; do dataname=${testname}_${ADDR[1]} resultname=${dataname}_result idname=${dataname}_id - command="node ../validate.js -s ${schemaname} -d ${dataname} -i $(cat ${idname})" + command="node ../../validate.js -s ${schemaname} -d ${dataname} -i $(cat ${idname})" echo -n "Executing test $command ... " $command | diff ${resultname} - echo OK From f47f100c32f7c0d2852af062579c3591882ec17a Mon Sep 17 00:00:00 2001 From: marcel Date: Wed, 8 Nov 2023 20:57:13 +0100 Subject: [PATCH 18/32] linting --- .../datamodel/tools/jsonldConverter.js | 30 ++----------------- .../tools/tests/jsonld2jsonld/test.sh | 14 +++------ .../tools/tests/schema2shacl/test.sh | 9 +++--- .../datamodel/tools/tests/validation/test.sh | 6 ++-- 4 files changed, 13 insertions(+), 46 deletions(-) diff --git a/semantic-model/datamodel/tools/jsonldConverter.js b/semantic-model/datamodel/tools/jsonldConverter.js index 770df5b4..7a2ec71f 100644 --- a/semantic-model/datamodel/tools/jsonldConverter.js +++ b/semantic-model/datamodel/tools/jsonldConverter.js @@ -99,32 +99,6 @@ if (!Array.isArray(jsonObj)) { jsonArr = jsonObj } -function assertNoContext (jsonObj) { - if (Array.isArray(jsonObj)) { - jsonObj.forEach((obj) => { - if ('@context' in obj) { - console.error('Error: @contex found in json-object. Only fully expanded objects without @context can be compacted. Exiting!') - process.exit(1) - } - }) - } else { - if ('@context' in jsonObj) { - console.error('Error: @context found in json-object. Only fully expanded objects without @context can be compacted. Exiting!') - process.exit(1) - } - } -} - -// Check if every object in array whether it has Context -function hasContexts (jsonArr) { - jsonArr.forEach(jsonObj => { - if (!('@context' in jsonObj)) { - return false - } - }) - return true -} - function loadContextFromFile (fileName) { const context = fs.readFileSync(fileName, 'utf8') const contextParsed = JSON.parse(context) @@ -154,9 +128,9 @@ function mergeContexts (jsonArr, context) { return mergedContext } if (context !== undefined) { - const parseContextUrl = new url.parse(context, true) + const parseContextUrl = new url.URL(context) if (parseContextUrl.protocol === 'file:') { - context = loadContextFromFile(parseContextUrl.path) + context = loadContextFromFile(parseContextUrl.pathname) } } return jsonArr.map(jsonObj => { diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh b/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh index 54810e1c..4e06e7b6 100644 --- a/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh @@ -15,19 +15,13 @@ # limitations under the License. # -#for testdir in $(ls payload*.jsonld_?_c*); do while IFS='_' read -ra ADDR; do payload=${ADDR[0]} switch=${ADDR[1]} context=${ADDR[2]} comparewith=${payload}_${switch}_${context} - echo comparewith $comparewith + echo comparewith "$comparewith" command="node ../../jsonldConverter.js $payload -$switch -c file://$PWD/$context" - echo Executing: $command - $command | diff ${comparewith} - - #$command - #for i in "${ADDR[@]}"; do - # process "$i" - # echo $i - #done -done <<< $(ls payload*.jsonld_?_c*) + echo Executing: "$command" + $command | diff "${comparewith}" - +done <<< "$(ls payload*.jsonld_?_c*)" diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/test.sh b/semantic-model/datamodel/tools/tests/schema2shacl/test.sh index 34bbfd9b..5ab51311 100644 --- a/semantic-model/datamodel/tools/tests/schema2shacl/test.sh +++ b/semantic-model/datamodel/tools/tests/schema2shacl/test.sh @@ -15,14 +15,13 @@ # limitations under the License. # -#for testdir in $(ls payload*.jsonld_?_c*); do while IFS='_' read -ra ADDR; do schemaname=${ADDR[0]} context=${ADDR[1]%.json} file=${schemaname}_${context}.json comparewith=${file}_result id=${file}_id - command="node ../../jsonschema2shacl.js -s $file -c file://$PWD/$context -i $(cat $id)" - echo Executing: $command - $command | diff ${comparewith} - -done <<< $(ls schema*_*.json) + command="node ../../jsonschema2shacl.js -s $file -c file://$PWD/$context -i $(cat "$id")" + echo Executing: "$command" + $command | diff "${comparewith}" - +done <<< "$(ls schema*_*.json)" diff --git a/semantic-model/datamodel/tools/tests/validation/test.sh b/semantic-model/datamodel/tools/tests/validation/test.sh index 31be7beb..a21b36fc 100644 --- a/semantic-model/datamodel/tools/tests/validation/test.sh +++ b/semantic-model/datamodel/tools/tests/validation/test.sh @@ -21,8 +21,8 @@ while IFS='_' read -ra ADDR; do dataname=${testname}_${ADDR[1]} resultname=${dataname}_result idname=${dataname}_id - command="node ../../validate.js -s ${schemaname} -d ${dataname} -i $(cat ${idname})" + command="node ../../validate.js -s ${schemaname} -d ${dataname} -i $(cat "${idname}")" echo -n "Executing test $command ... " - $command | diff ${resultname} - + $command | diff "${resultname}" - echo OK -done <<< $(ls validation*_data*.json) +done <<< "$(ls validation*_data*.json)" From fb959a473ab179f76f415dadbbbd3458085db4c5 Mon Sep 17 00:00:00 2001 From: marcel Date: Wed, 8 Nov 2023 23:51:53 +0100 Subject: [PATCH 19/32] started library --- semantic-model/datamodel/tools/Makefile | 5 +- .../datamodel/tools/jsonldConverter.js | 62 ++--------------- .../datamodel/tools/lib/jsonldUtils.js | 69 +++++++++++++++++++ semantic-model/datamodel/tools/package.json | 10 ++- .../tools/tests/jsonld2jsonld/test.sh | 2 +- 5 files changed, 87 insertions(+), 61 deletions(-) create mode 100644 semantic-model/datamodel/tools/lib/jsonldUtils.js diff --git a/semantic-model/datamodel/tools/Makefile b/semantic-model/datamodel/tools/Makefile index f828b05d..ccdd4aff 100644 --- a/semantic-model/datamodel/tools/Makefile +++ b/semantic-model/datamodel/tools/Makefile @@ -24,4 +24,7 @@ lint: shellcheck tests/*/test.sh test: - npm run test \ No newline at end of file + #npm run test + cd tests/validation && bash ./test.sh + cd tests/jsonld2jsonld && bash ./test.sh + cd tests/schema2shacl && bash ./test.sh \ No newline at end of file diff --git a/semantic-model/datamodel/tools/jsonldConverter.js b/semantic-model/datamodel/tools/jsonldConverter.js index 7a2ec71f..2e4018c3 100644 --- a/semantic-model/datamodel/tools/jsonldConverter.js +++ b/semantic-model/datamodel/tools/jsonldConverter.js @@ -15,25 +15,11 @@ */ 'use strict' -// const $RefParser = require('json-schema-ref-parser'); -// const $rdf = require('rdflib') const fs = require('fs') const yargs = require('yargs') -// const path = require('path') -// const N3 = require('n3') const url = require('url') -// const { DataFactory } = N3 -// const { namedNode, literal, blankNode, defaultGraph, quad } = DataFactory; -// const { URL } = require('url'); // Import the URL module -// const ContextParser = require('jsonld-context-parser').ContextParser -// const ContextUtil = require('jsonld-context-parser').Util; -// const myParser = new ContextParser() const jsonld = require('jsonld') -// const exp = require('constants'); - -// const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#'); -// const SHACL = $rdf.Namespace('http://www.w3.org/ns/shacl#'); -// const IFFK = $rdf.Namespace('https://industry-fusion.org/knowledge/v0.1/'); +const jsonldUtils = require('./lib/jsonldUtils') let jsonFileName const argv = yargs .option('concise', { @@ -99,46 +85,6 @@ if (!Array.isArray(jsonObj)) { jsonArr = jsonObj } -function loadContextFromFile (fileName) { - const context = fs.readFileSync(fileName, 'utf8') - const contextParsed = JSON.parse(context) - return contextParsed -} - -// Merge Contexts -function mergeContexts (jsonArr, context) { - function mergeContext (localContext, context) { - let mergedContext = [] - if (!Array.isArray(localContext) && localContext !== undefined) { - mergedContext = [localContext] - } - if (context === undefined) { - if (mergedContext.length === 0) { - return null - } - return mergedContext - } else if (!Array.isArray(context)) { - context = [context] - } - context.forEach(c => { - if (typeof (c) !== 'string' || mergedContext.find(x => c === x) === undefined) { - mergedContext.push(c) - } - }) - return mergedContext - } - if (context !== undefined) { - const parseContextUrl = new url.URL(context) - if (parseContextUrl.protocol === 'file:') { - context = loadContextFromFile(parseContextUrl.pathname) - } - } - return jsonArr.map(jsonObj => { - const localContext = jsonObj['@context'] - return mergeContext(localContext, context) - }) -} - function conciseExpandedForm (expanded) { function filterAttribute (attr) { if (typeof (attr) === 'object') { @@ -210,7 +156,7 @@ async function compact (objArr, contextArr) { (async (jsonArr) => { if (!(argv.n === undefined)) { - const mergedContexts = mergeContexts(jsonArr, argv.c) + const mergedContexts = jsonldUtils.mergeContexts(jsonArr, argv.c) if (mergedContexts !== undefined && mergedContexts.find(x => x === null)) { console.error('Error: For Compaction, context must be either defined in all objects or externally. Exiting!') process.exit(1) @@ -222,7 +168,7 @@ async function compact (objArr, contextArr) { console.log(JSON.stringify(compacted, null, 2)) } if (!(argv.x === undefined)) { - const mergedContexts = mergeContexts(jsonArr, argv.c) + const mergedContexts = jsonldUtils.mergeContexts(jsonArr, argv.c) if (mergedContexts !== undefined && mergedContexts.find(x => x === null)) { console.error('Error: For Extraction, context must be either defined in all objects or externally. Exiting!') process.exit(1) @@ -231,7 +177,7 @@ async function compact (objArr, contextArr) { console.log(JSON.stringify(expanded, null, 2)) } if (!(argv.r === undefined)) { - const mergedContexts = mergeContexts(jsonArr, argv.c) + const mergedContexts = jsonldUtils.mergeContexts(jsonArr, argv.c) if (mergedContexts !== undefined && mergedContexts.find(x => x === null)) { console.error('Error: For Normalization, context must be either defined in all objects or externally. Exiting!') process.exit(1) diff --git a/semantic-model/datamodel/tools/lib/jsonldUtils.js b/semantic-model/datamodel/tools/lib/jsonldUtils.js new file mode 100644 index 00000000..3fd112fe --- /dev/null +++ b/semantic-model/datamodel/tools/lib/jsonldUtils.js @@ -0,0 +1,69 @@ + +/** +* Copyright (c) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +'use strict' +const url = require('url') +const fs = require('fs') + +function loadContextFromFile (fileName) { + const context = fs.readFileSync(fileName, 'utf8') + const contextParsed = JSON.parse(context) + return contextParsed +} + +/** + * Merge local context from jsonld and external given context + * into a joint array + * @param {object} jsonArr + * @param {array or string} context + * @returns mergedContext + */ +function mergeContexts (jsonArr, context) { + function mergeContext (localContext, context) { + let mergedContext = [] + if (!Array.isArray(localContext) && localContext !== undefined) { + mergedContext = [localContext] + } + if (context === undefined) { + if (mergedContext.length === 0) { + return null + } + return mergedContext + } else if (!Array.isArray(context)) { + context = [context] + } + context.forEach(c => { + if (typeof (c) !== 'string' || mergedContext.find(x => c === x) === undefined) { + mergedContext.push(c) + } + }) + return mergedContext + } + if (context !== undefined) { + const parseContextUrl = new url.URL(context) + if (parseContextUrl.protocol === 'file:') { + context = loadContextFromFile(parseContextUrl.pathname) + } + } + return jsonArr.map(jsonObj => { + const localContext = jsonObj['@context'] + return mergeContext(localContext, context) + }) +} + +module.exports = { + mergeContexts +} diff --git a/semantic-model/datamodel/tools/package.json b/semantic-model/datamodel/tools/package.json index 60da19c7..b9a218a5 100644 --- a/semantic-model/datamodel/tools/package.json +++ b/semantic-model/datamodel/tools/package.json @@ -4,19 +4,27 @@ "description": "This package contains tools for validating NGSI-LD data for the Process Data Twin (PDT)", "main": "validate.js", "scripts": { - "test": "cd tests && for dir in $(ls); do (cd $dir; bash ./test.sh); done && exit 0", + "test": "./node_modules/.bin/nyc --check-coverage ./node_modules/.bin/mocha --lines 80", "lint": "./node_modules/.bin/eslint *.js" }, "author": "Marcel Wagner", "license": "Apache-2.0", "dependencies": { "ajv": "^8.12.0", + "json-schema-ref-parser": "^9.0.9", "jsonld-context-parser": "^2.4.0", + + "rdflib": "^2.2.32", + "yargs": "^17.7.2" }, "devDependencies": { + "rewire": "^7.0.0", + "nyc": "^15.1.0", + "mocha": "^10.2.0", + "chai": "^4.3.10", "eslint": "^8.53.0", "eslint-config-standard": "^17.1.0", "eslint-plugin-import": "^2.29.0", diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh b/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh index 4e06e7b6..8757a973 100644 --- a/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh @@ -23,5 +23,5 @@ while IFS='_' read -ra ADDR; do echo comparewith "$comparewith" command="node ../../jsonldConverter.js $payload -$switch -c file://$PWD/$context" echo Executing: "$command" - $command | diff "${comparewith}" - + $command | diff "${comparewith}" - || exit 1 done <<< "$(ls payload*.jsonld_?_c*)" From e31c913828df064f872c68ce74ebb6af658bf6fd Mon Sep 17 00:00:00 2001 From: marcel Date: Thu, 9 Nov 2023 12:05:35 +0100 Subject: [PATCH 20/32] moving everything which must have unittests into library --- .../datamodel/tools/jsonldConverter.js | 61 +------ .../datamodel/tools/jsonschema2shacl.js | 137 +-------------- .../datamodel/tools/lib/jsonldUtils.js | 70 +++++++- .../datamodel/tools/lib/shaclUtils.js | 160 ++++++++++++++++++ 4 files changed, 233 insertions(+), 195 deletions(-) create mode 100644 semantic-model/datamodel/tools/lib/shaclUtils.js diff --git a/semantic-model/datamodel/tools/jsonldConverter.js b/semantic-model/datamodel/tools/jsonldConverter.js index 2e4018c3..60d1b77c 100644 --- a/semantic-model/datamodel/tools/jsonldConverter.js +++ b/semantic-model/datamodel/tools/jsonldConverter.js @@ -17,7 +17,6 @@ const fs = require('fs') const yargs = require('yargs') -const url = require('url') const jsonld = require('jsonld') const jsonldUtils = require('./lib/jsonldUtils') let jsonFileName @@ -85,62 +84,6 @@ if (!Array.isArray(jsonObj)) { jsonArr = jsonObj } -function conciseExpandedForm (expanded) { - function filterAttribute (attr) { - if (typeof (attr) === 'object') { - if ('@type' in attr && (attr['@type'][0] === 'https://uri.etsi.org/ngsi-ld/Property' || - attr['@type'][0] === 'https://uri.etsi.org/ngsi-ld/Relationship')) { - delete attr['@type'] - } - if ('https://uri.etsi.org/ngsi-ld/hasValue' in attr) { - attr['@value'] = attr['https://uri.etsi.org/ngsi-ld/hasValue'][0]['@value'] - delete attr['https://uri.etsi.org/ngsi-ld/hasValue'] - } - } - } - expanded.forEach(c => { - Object.keys(c).forEach(key => { - if (Array.isArray(c[key])) { - c[key].forEach(a => filterAttribute(a)) - } else { - filterAttribute(c[key]) - } - }) - }) - return expanded -} - -function normalizeExpandedForm (expanded) { - function extendAttribute (attr) { - if (typeof (attr) === 'object') { - if (!('@type' in attr)) { - if ('https://uri.etsi.org/ngsi-ld/hasValue' in attr || '@value' in attr || '@id' in attr) { - attr['@type'] = 'https://uri.etsi.org/ngsi-ld/Property' - } else if ('https://uri.etsi.org/ngsi-ld/hasObject' in attr) { - attr['@type'] = 'https://uri.etsi.org/ngsi-ld/Relationship' - } - if ('@value' in attr) { - attr['https://uri.etsi.org/ngsi-ld/hasValue'] = attr['@value'] - delete attr['@value'] - } else if ('@id' in attr) { - attr['https://uri.etsi.org/ngsi-ld/hasValue'] = { '@id': attr['@id'] } - delete attr['@id'] - } - } - } - } - expanded.forEach(c => { - Object.keys(c).forEach(key => { - if (Array.isArray(c[key])) { - c[key].forEach(a => extendAttribute(a)) - } else { - extendAttribute(c[key]) - } - }) - }) - return expanded -} - async function expand (objArr, contextArr) { const expanded = await Promise.all(objArr.map(async (jsonObj, index) => { jsonObj['@context'] = contextArr[index] @@ -163,7 +106,7 @@ async function compact (objArr, contextArr) { } // Compaction to find Properties in compacted form const expanded = await expand(jsonArr, mergedContexts) - const concised = conciseExpandedForm(expanded) + const concised = jsonldUtils.conciseExpandedForm(expanded) const compacted = await compact(concised, mergedContexts) console.log(JSON.stringify(compacted, null, 2)) } @@ -183,7 +126,7 @@ async function compact (objArr, contextArr) { process.exit(1) } const expanded = await expand(jsonArr, mergedContexts) - const normalized = normalizeExpandedForm(expanded) + const normalized = jsonldUtils.normalizeExpandedForm(expanded) const compacted = await compact(normalized, mergedContexts) console.log(JSON.stringify(compacted, null, 2)) } diff --git a/semantic-model/datamodel/tools/jsonschema2shacl.js b/semantic-model/datamodel/tools/jsonschema2shacl.js index 666a0671..585f1602 100644 --- a/semantic-model/datamodel/tools/jsonschema2shacl.js +++ b/semantic-model/datamodel/tools/jsonschema2shacl.js @@ -19,11 +19,8 @@ const $RefParser = require('json-schema-ref-parser') const $rdf = require('rdflib') const fs = require('fs') const yargs = require('yargs') -// const N3 = require('n3') const url = require('url') -// const { DataFactory } = N3 -// const { namedNode, literal, blankNode, defaultGraph, quad } = DataFactory -// const { URL } = require('url') +const ShaclUtils = require('./lib/shaclUtils') const ContextParser = require('jsonld-context-parser').ContextParser const ContextUtil = require('jsonld-context-parser').Util const myParser = new ContextParser() @@ -61,136 +58,6 @@ const jsonSchema = JSON.parse(jsonSchemaText) let globalContext let globalPrefixHash -class NodeShape { - constructor (targetClass) { - this.targetClass = targetClass - this.properties = [] - } - - addPropertyShape (propertyShape) { - this.properties.push(propertyShape) - } - - get properties () { - return this._properties - } - - set properties (prop) { - this._properties = prop - } -} - -class PropertyShape { - constructor (mincount, maxcount, nodeKind, path, isProperty) { - this.mincount = mincount - this.maxcount = maxcount - this.nodeKind = nodeKind - this.path = path - this.constraints = [] - this.isProperty = isProperty - } - - addConstraint (constraint) { - this.constraints.push(constraint) - } - - set propertyNode (node) { - this._propertyNode = node - } - - get propertyNode () { - return this._propertyNode - } -} - -class Constraint { - constructor (type, params) { - this.type = type - this.params = params - } -} - -function scanNodeShape (typeschema) { - const id = typeschema.$id - - const nodeShape = new NodeShape(id) - scanProperties(nodeShape, typeschema) - return nodeShape -} - -function scanProperties (nodeShape, typeschema) { - let required = [] - if ('required' in typeschema) { - required = typeschema.required - } - if ('properties' in typeschema) { - Object.keys(typeschema.properties).forEach( - (property) => { - if (property === 'type' || property === 'id') { - return - } - let nodeKind = SHACL('Literal') - let klass = null - let isProperty = true - if ('relationship' in typeschema.properties[property]) { - nodeKind = SHACL('IRI') - klass = typeschema.properties[property].relationship - klass = globalContext.expandTerm(klass, true) - isProperty = false - } - let mincount = 0 - const maxcount = 1 - if (required.includes(property)) { - mincount = 1 - } - let path = property - if (!ContextUtil.isValidIri(path)) { - path = globalContext.expandTerm(path, true) - } - const propertyShape = new PropertyShape(mincount, maxcount, nodeKind, $rdf.sym(path), isProperty) - nodeShape.addPropertyShape(propertyShape) - if (klass !== null) { - propertyShape.addConstraint(new Constraint(SHACL('class'), $rdf.sym(klass))) - } - scanConstraints(propertyShape, typeschema.properties[property]) - }) - } - if ('allOf' in typeschema) { - typeschema.allOf.forEach((elem) => { - scanProperties(nodeShape, elem) - }) - } -} - -function scanConstraints (propertyShape, typeschema) { - if ('enum' in typeschema) { - propertyShape.addConstraint(new Constraint(SHACL('in'), typeschema.enum)) - } - if ('datatype' in typeschema) { - // datatype constraints are not used actively. It is not testing the value but only checks if the formal - // datatype "tag" conforms - // propertyShape.addConstraint(new Constraint(SHACL('datatype'), typeschema.datatype)) - } - if ('maxiumum' in typeschema) { - propertyShape.addConstraint(new Constraint(SHACL('maxInclusive'), typeschema.maximum)) - } - if ('miniumum' in typeschema) { - propertyShape.addConstraint(new Constraint(SHACL('minInclusive'), typeschema.minimum)) - } - if ('exclusiveMiniumum' in typeschema) { - propertyShape.addConstraint(new Constraint(SHACL('minExclusive'), typeschema.exclusiveMinimum)) - } - if ('exclusiveMaxiumum' in typeschema) { - propertyShape.addConstraint(new Constraint(SHACL('maxExclusive'), typeschema.exclusiveMaximum)) - } - if ('maxLength' in typeschema) { - propertyShape.addConstraint(new Constraint(SHACL('maxLength'), typeschema.maxLength)) - } - if ('minLength' in typeschema) { - propertyShape.addConstraint(new Constraint(SHACL('minLength'), typeschema.minLength)) - } -} - function dumpShacl (nodeShape, store) { dumpNodeShape(nodeShape, store) } @@ -244,7 +111,7 @@ function shaclize (schemas, id) { id = encodeHash(id) const store = new $rdf.IndexedFormula() const typeschema = schemas.find((schema) => schema.$id === id) - const nodeShape = scanNodeShape(typeschema) + const nodeShape = ShaclUtils.scanNodeShape(typeschema, globalContext) dumpShacl(nodeShape, store) const serializer = new $rdf.Serializer(store) serializer.setFlags('u') diff --git a/semantic-model/datamodel/tools/lib/jsonldUtils.js b/semantic-model/datamodel/tools/lib/jsonldUtils.js index 3fd112fe..9ffb35ec 100644 --- a/semantic-model/datamodel/tools/lib/jsonldUtils.js +++ b/semantic-model/datamodel/tools/lib/jsonldUtils.js @@ -64,6 +64,74 @@ function mergeContexts (jsonArr, context) { }) } +/** + * Expects NGSI-LD object in expanded form and transforms to NGSI-LD concise form + * @param {object} expanded + * @returns concise and expanded form + */ +function conciseExpandedForm (expanded) { + function filterAttribute (attr) { + if (typeof (attr) === 'object') { + if ('@type' in attr && (attr['@type'][0] === 'https://uri.etsi.org/ngsi-ld/Property' || + attr['@type'][0] === 'https://uri.etsi.org/ngsi-ld/Relationship')) { + delete attr['@type'] + } + if ('https://uri.etsi.org/ngsi-ld/hasValue' in attr) { + attr['@value'] = attr['https://uri.etsi.org/ngsi-ld/hasValue'][0]['@value'] + delete attr['https://uri.etsi.org/ngsi-ld/hasValue'] + } + } + } + expanded.forEach(c => { + Object.keys(c).forEach(key => { + if (Array.isArray(c[key])) { + c[key].forEach(a => filterAttribute(a)) + } else { + filterAttribute(c[key]) + } + }) + }) + return expanded +} + +/** + * Expects NGSI-LD object in expanded form and transforms to NGSI-LD normalized + * @param {object} expanded + * @returns normalized NGSI-LD and expanded form + */ +function normalizeExpandedForm (expanded) { + function extendAttribute (attr) { + if (typeof (attr) === 'object') { + if (!('@type' in attr)) { + if ('https://uri.etsi.org/ngsi-ld/hasValue' in attr || '@value' in attr || '@id' in attr) { + attr['@type'] = 'https://uri.etsi.org/ngsi-ld/Property' + } else if ('https://uri.etsi.org/ngsi-ld/hasObject' in attr) { + attr['@type'] = 'https://uri.etsi.org/ngsi-ld/Relationship' + } + if ('@value' in attr) { + attr['https://uri.etsi.org/ngsi-ld/hasValue'] = attr['@value'] + delete attr['@value'] + } else if ('@id' in attr) { + attr['https://uri.etsi.org/ngsi-ld/hasValue'] = { '@id': attr['@id'] } + delete attr['@id'] + } + } + } + } + expanded.forEach(c => { + Object.keys(c).forEach(key => { + if (Array.isArray(c[key])) { + c[key].forEach(a => extendAttribute(a)) + } else { + extendAttribute(c[key]) + } + }) + }) + return expanded +} + module.exports = { - mergeContexts + mergeContexts, + conciseExpandedForm, + normalizeExpandedForm } diff --git a/semantic-model/datamodel/tools/lib/shaclUtils.js b/semantic-model/datamodel/tools/lib/shaclUtils.js new file mode 100644 index 00000000..5ac8c5ee --- /dev/null +++ b/semantic-model/datamodel/tools/lib/shaclUtils.js @@ -0,0 +1,160 @@ +/** +* Copyright (c) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +'use strict' +const $rdf = require('rdflib') +const ContextParser = require('jsonld-context-parser').ContextParser +const ContextUtil = require('jsonld-context-parser').Util + +const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') +const SHACL = $rdf.Namespace('http://www.w3.org/ns/shacl#') +const IFFK = $rdf.Namespace('https://industry-fusion.org/knowledge/v0.1/') + +class NodeShape { + constructor (targetClass) { + this.targetClass = targetClass + this.properties = [] + } + + addPropertyShape (propertyShape) { + this.properties.push(propertyShape) + } + + get properties () { + return this._properties + } + + set properties (prop) { + this._properties = prop + } +} + +class PropertyShape { + constructor (mincount, maxcount, nodeKind, path, isProperty) { + this.mincount = mincount + this.maxcount = maxcount + this.nodeKind = nodeKind + this.path = path + this.constraints = [] + this.isProperty = isProperty + } + + addConstraint (constraint) { + this.constraints.push(constraint) + } + + set propertyNode (node) { + this._propertyNode = node + } + + get propertyNode () { + return this._propertyNode + } +} + +class Constraint { + constructor (type, params) { + this.type = type + this.params = params + } +} + +function scanNodeShape (typeschema, globalContext) { + const id = typeschema.$id + + const nodeShape = new NodeShape(id) + scanProperties(nodeShape, typeschema, globalContext) + return nodeShape +} + +function scanProperties (nodeShape, typeschema, globalContext) { + let required = [] + if ('required' in typeschema) { + required = typeschema.required + } + if ('properties' in typeschema) { + Object.keys(typeschema.properties).forEach( + (property) => { + if (property === 'type' || property === 'id') { + return + } + let nodeKind = SHACL('Literal') + let klass = null + let isProperty = true + if ('relationship' in typeschema.properties[property]) { + nodeKind = SHACL('IRI') + klass = typeschema.properties[property].relationship + klass = globalContext.expandTerm(klass, true) + isProperty = false + } + let mincount = 0 + const maxcount = 1 + if (required.includes(property)) { + mincount = 1 + } + let path = property + if (!ContextUtil.isValidIri(path)) { + path = globalContext.expandTerm(path, true) + } + const propertyShape = new PropertyShape(mincount, maxcount, nodeKind, $rdf.sym(path), isProperty) + nodeShape.addPropertyShape(propertyShape) + if (klass !== null) { + propertyShape.addConstraint(new Constraint(SHACL('class'), $rdf.sym(klass))) + } + scanConstraints(propertyShape, typeschema.properties[property]) + }) + } + if ('allOf' in typeschema) { + typeschema.allOf.forEach((elem) => { + scanProperties(nodeShape, elem, globalContext) + }) + } +} + +function scanConstraints (propertyShape, typeschema) { + if ('enum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('in'), typeschema.enum)) + } + if ('datatype' in typeschema) { + // datatype constraints are not used actively. It is not testing the value but only checks if the formal + // datatype "tag" conforms + // propertyShape.addConstraint(new Constraint(SHACL('datatype'), typeschema.datatype)) + } + if ('maxiumum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('maxInclusive'), typeschema.maximum)) + } + if ('miniumum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('minInclusive'), typeschema.minimum)) + } + if ('exclusiveMiniumum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('minExclusive'), typeschema.exclusiveMinimum)) + } + if ('exclusiveMaxiumum' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('maxExclusive'), typeschema.exclusiveMaximum)) + } + if ('maxLength' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('maxLength'), typeschema.maxLength)) + } + if ('minLength' in typeschema) { + propertyShape.addConstraint(new Constraint(SHACL('minLength'), typeschema.minLength)) + } +} + +module.exports = { + NodeShape, + PropertyShape, + Constraint, + scanNodeShape +} From fcf297a5a1d71027da07a8a79abb64d94cd0c83f Mon Sep 17 00:00:00 2001 From: marcel Date: Thu, 9 Nov 2023 20:25:03 +0100 Subject: [PATCH 21/32] cleaned up shacltools --- .../datamodel/tools/jsonschema2shacl.js | 101 +---------------- .../datamodel/tools/lib/shaclUtils.js | 106 +++++++++++++++++- 2 files changed, 103 insertions(+), 104 deletions(-) diff --git a/semantic-model/datamodel/tools/jsonschema2shacl.js b/semantic-model/datamodel/tools/jsonschema2shacl.js index 585f1602..93dc8c79 100644 --- a/semantic-model/datamodel/tools/jsonschema2shacl.js +++ b/semantic-model/datamodel/tools/jsonschema2shacl.js @@ -16,18 +16,10 @@ 'use strict' const $RefParser = require('json-schema-ref-parser') -const $rdf = require('rdflib') const fs = require('fs') const yargs = require('yargs') -const url = require('url') const ShaclUtils = require('./lib/shaclUtils') -const ContextParser = require('jsonld-context-parser').ContextParser -const ContextUtil = require('jsonld-context-parser').Util -const myParser = new ContextParser() -const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') -const SHACL = $rdf.Namespace('http://www.w3.org/ns/shacl#') -const IFFK = $rdf.Namespace('https://industry-fusion.org/knowledge/v0.1/') const argv = yargs .command('$0', 'Converting an IFF Schema file for NGSI-LD objects into a SHACL constraint.') .option('schema', { @@ -54,94 +46,7 @@ const argv = yargs // Read in an array of JSON-Schemas const jsonSchemaText = fs.readFileSync(argv.s, 'utf8') -const jsonSchema = JSON.parse(jsonSchemaText) -let globalContext -let globalPrefixHash - -function dumpShacl (nodeShape, store) { - dumpNodeShape(nodeShape, store) -} - -function dumpNodeShape (nodeShape, store) { - const nodeName = decodeURIComponent(globalContext.expandTerm(nodeShape.targetClass)) - const parsedUrl = new url.URL(nodeName) - const fragment = parsedUrl.hash.substring(1) - const shapeName = fragment + 'Shape' - store.add(IFFK(shapeName), RDF('type'), SHACL('NodeShape')) - store.add(IFFK(shapeName), SHACL('targetClass'), $rdf.sym(nodeName)) - nodeShape.properties.forEach((property) => { - const propNode = $rdf.blankNode() - property.propertyNode = propNode - store.add(IFFK(shapeName), SHACL('property'), propNode) - dumpPropertyShape(property, store) - }) -} - -function dumpPropertyShape (propertyShape, store) { - const propNode = propertyShape.propertyNode - store.add(propNode, SHACL('minCount'), propertyShape.mincount) - store.add(propNode, SHACL('maxCount'), propertyShape.maxcount) - store.add(propNode, SHACL('nodeKind'), SHACL('BlankNode')) - store.add(propNode, SHACL('path'), propertyShape.path) - const attributeNode = $rdf.blankNode() - store.add(propNode, SHACL('property'), attributeNode) - const ngsildPrefix = globalPrefixHash['ngsi-ld'] - const NGSILD = $rdf.Namespace(ngsildPrefix) - if (propertyShape.isProperty) { - store.add(attributeNode, SHACL('path'), NGSILD('hasValue')) - } else { - store.add(attributeNode, SHACL('path'), NGSILD('hasObject')) - } - store.add(attributeNode, SHACL('minCount'), 1) - store.add(attributeNode, SHACL('maxCount'), 1) - store.add(attributeNode, SHACL('nodeKind'), propertyShape.nodeKind) - const constraints = propertyShape.constraints - constraints.forEach((constraint) => { - store.add(attributeNode, constraint.type, constraint.params) - }) -} - -function encodeHash (id) { - const url = new URL(id) - const hash = encodeURIComponent(url.hash) - return `${url.protocol}//${url.hostname}${url.pathname}${hash}` -} - -function shaclize (schemas, id) { - id = encodeHash(id) - const store = new $rdf.IndexedFormula() - const typeschema = schemas.find((schema) => schema.$id === id) - const nodeShape = ShaclUtils.scanNodeShape(typeschema, globalContext) - dumpShacl(nodeShape, store) - const serializer = new $rdf.Serializer(store) - serializer.setFlags('u') - serializer.setNamespaces(globalPrefixHash) - const turtle = serializer.statementsToN3(store.statementsMatching(undefined, undefined, undefined, undefined)) - console.log(turtle) -} - -async function loadContext (uriOrContext) { - const parseUrl = new url.URL(uriOrContext) - if (parseUrl.protocol === 'file:') { - uriOrContext = JSON.parse(fs.readFileSync(parseUrl.pathname, 'utf-8')) - } - const context = await myParser.parse(uriOrContext) - globalContext = context - const prefixHash = {} - Object.keys(context.getContextRaw()).filter((key) => key !== '@vocab').forEach((key) => { - const value = context.getContextRaw()[key] - if (typeof value === 'string') { - if (ContextUtil.isPrefixIriEndingWithGenDelim(value)) { - prefixHash[key] = value - } - } else if (typeof value === 'object') { - if (ContextUtil.isPrefixIriEndingWithGenDelim(value['@id'])) { - prefixHash[key] = value['@id'] - } - } - }) - globalPrefixHash = prefixHash -} +const jsonSchema = JSON.parse(jsonSchemaText); (async (jsconSchema) => { const myResolver = { @@ -170,9 +75,9 @@ async function loadContext (uriOrContext) { } })(jsonSchema) .then(async (schema) => { - await loadContext(argv.c) + await ShaclUtils.loadContext(argv.c) return schema }) .then(schema => { - shaclize(schema, argv.i) + ShaclUtils.shaclize(schema, argv.i) }) diff --git a/semantic-model/datamodel/tools/lib/shaclUtils.js b/semantic-model/datamodel/tools/lib/shaclUtils.js index 5ac8c5ee..78d0369f 100644 --- a/semantic-model/datamodel/tools/lib/shaclUtils.js +++ b/semantic-model/datamodel/tools/lib/shaclUtils.js @@ -15,13 +15,19 @@ */ 'use strict' const $rdf = require('rdflib') -const ContextParser = require('jsonld-context-parser').ContextParser const ContextUtil = require('jsonld-context-parser').Util +const url = require('url') +const fs = require('fs') +const ContextParser = require('jsonld-context-parser').ContextParser +const myParser = new ContextParser() const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') const SHACL = $rdf.Namespace('http://www.w3.org/ns/shacl#') const IFFK = $rdf.Namespace('https://industry-fusion.org/knowledge/v0.1/') +let globalContext +let globalPrefixHash + class NodeShape { constructor (targetClass) { this.targetClass = targetClass @@ -71,15 +77,54 @@ class Constraint { } } -function scanNodeShape (typeschema, globalContext) { +function dumpPropertyShape (propertyShape, store) { + const propNode = propertyShape.propertyNode + store.add(propNode, SHACL('minCount'), propertyShape.mincount) + store.add(propNode, SHACL('maxCount'), propertyShape.maxcount) + store.add(propNode, SHACL('nodeKind'), SHACL('BlankNode')) + store.add(propNode, SHACL('path'), propertyShape.path) + const attributeNode = $rdf.blankNode() + store.add(propNode, SHACL('property'), attributeNode) + const ngsildPrefix = globalPrefixHash['ngsi-ld'] + const NGSILD = $rdf.Namespace(ngsildPrefix) + if (propertyShape.isProperty) { + store.add(attributeNode, SHACL('path'), NGSILD('hasValue')) + } else { + store.add(attributeNode, SHACL('path'), NGSILD('hasObject')) + } + store.add(attributeNode, SHACL('minCount'), 1) + store.add(attributeNode, SHACL('maxCount'), 1) + store.add(attributeNode, SHACL('nodeKind'), propertyShape.nodeKind) + const constraints = propertyShape.constraints + constraints.forEach((constraint) => { + store.add(attributeNode, constraint.type, constraint.params) + }) +} + +function dumpNodeShape (nodeShape, store) { + const nodeName = decodeURIComponent(globalContext.expandTerm(nodeShape.targetClass)) + const parsedUrl = new url.URL(nodeName) + const fragment = parsedUrl.hash.substring(1) + const shapeName = fragment + 'Shape' + store.add(IFFK(shapeName), RDF('type'), SHACL('NodeShape')) + store.add(IFFK(shapeName), SHACL('targetClass'), $rdf.sym(nodeName)) + nodeShape.properties.forEach((property) => { + const propNode = $rdf.blankNode() + property.propertyNode = propNode + store.add(IFFK(shapeName), SHACL('property'), propNode) + dumpPropertyShape(property, store) + }) +} + +function scanNodeShape (typeschema) { const id = typeschema.$id const nodeShape = new NodeShape(id) - scanProperties(nodeShape, typeschema, globalContext) + scanProperties(nodeShape, typeschema) return nodeShape } -function scanProperties (nodeShape, typeschema, globalContext) { +function scanProperties (nodeShape, typeschema) { let required = [] if ('required' in typeschema) { required = typeschema.required @@ -118,7 +163,7 @@ function scanProperties (nodeShape, typeschema, globalContext) { } if ('allOf' in typeschema) { typeschema.allOf.forEach((elem) => { - scanProperties(nodeShape, elem, globalContext) + scanProperties(nodeShape, elem) }) } } @@ -152,9 +197,58 @@ function scanConstraints (propertyShape, typeschema) { } } +function shaclize (schemas, id) { + id = encodeHash(id) + const store = new $rdf.IndexedFormula() + const typeschema = schemas.find((schema) => schema.$id === id) + const nodeShape = scanNodeShape(typeschema) + dumpShacl(nodeShape, store) + const serializer = new $rdf.Serializer(store) + serializer.setFlags('u') + serializer.setNamespaces(globalPrefixHash) + const turtle = serializer.statementsToN3(store.statementsMatching(undefined, undefined, undefined, undefined)) + console.log(turtle) +} + +function encodeHash (id) { + const url = new URL(id) + const hash = encodeURIComponent(url.hash) + return `${url.protocol}//${url.hostname}${url.pathname}${hash}` +} + +function dumpShacl (nodeShape, store) { + dumpNodeShape(nodeShape, store) +} + +async function loadContext (uriOrContext) { + const parseUrl = new url.URL(uriOrContext) + if (parseUrl.protocol === 'file:') { + uriOrContext = JSON.parse(fs.readFileSync(parseUrl.pathname, 'utf-8')) + } + const context = await myParser.parse(uriOrContext) + globalContext = context + const prefixHash = {} + Object.keys(context.getContextRaw()).filter((key) => key !== '@vocab').forEach((key) => { + const value = context.getContextRaw()[key] + if (typeof value === 'string') { + if (ContextUtil.isPrefixIriEndingWithGenDelim(value)) { + prefixHash[key] = value + } + } else if (typeof value === 'object') { + if (ContextUtil.isPrefixIriEndingWithGenDelim(value['@id'])) { + prefixHash[key] = value['@id'] + } + } + }) + globalPrefixHash = prefixHash +} + module.exports = { NodeShape, PropertyShape, Constraint, - scanNodeShape + scanNodeShape, + dumpNodeShape, + shaclize, + loadContext } From 87b05caf44a83f5f0c4941842037287109fc2387 Mon Sep 17 00:00:00 2001 From: marcel Date: Fri, 10 Nov 2023 00:41:26 +0100 Subject: [PATCH 22/32] added jsonldUtils test --- .../datamodel/tools/lib/jsonldUtils.js | 3 ++ .../datamodel/tools/tests/testJsonldUtils.js | 47 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 semantic-model/datamodel/tools/tests/testJsonldUtils.js diff --git a/semantic-model/datamodel/tools/lib/jsonldUtils.js b/semantic-model/datamodel/tools/lib/jsonldUtils.js index 9ffb35ec..e0ced535 100644 --- a/semantic-model/datamodel/tools/lib/jsonldUtils.js +++ b/semantic-model/datamodel/tools/lib/jsonldUtils.js @@ -58,6 +58,9 @@ function mergeContexts (jsonArr, context) { context = loadContextFromFile(parseContextUrl.pathname) } } + if (jsonArr === undefined) { + return mergeContext(undefined, context) + } return jsonArr.map(jsonObj => { const localContext = jsonObj['@context'] return mergeContext(localContext, context) diff --git a/semantic-model/datamodel/tools/tests/testJsonldUtils.js b/semantic-model/datamodel/tools/tests/testJsonldUtils.js new file mode 100644 index 00000000..e25487e2 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/testJsonldUtils.js @@ -0,0 +1,47 @@ +/** +* Copyright (c) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +'use strict' + +const { assert } = require('chai') +const chai = require('chai') +global.should = chai.should() + +const rewire = require('rewire') +const ToTest = rewire('../lib/jsonldUtils.js') + +describe('Test loadContextFromFile', function () { + it('Should load context from filename', function () { + const fs = { + readFileSync: (filename, coding) => { + filename.should.equal('filename') + return '["context"]' + } + } + const revert = ToTest.__set__('fs', fs) + //ToTest.__set__('process', process) + const loadContextFromFile = ToTest.__get__('loadContextFromFile') + const result = loadContextFromFile('filename') + result.should.deep.equal(['context']) + revert() + }) +}) + +describe('Test mergeContexts', function() { + it('should return null with undefined contexts', function() { + const result = ToTest.mergeContexts(undefined, undefined) + assert(result === null) + }) +}) From 21a36916545b226b3f59c353f2b3a4c4e7ec8123 Mon Sep 17 00:00:00 2001 From: marcel Date: Fri, 10 Nov 2023 01:33:30 +0100 Subject: [PATCH 23/32] continued to work on testJsonldUtils --- .../datamodel/tools/tests/testJsonldUtils.js | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/semantic-model/datamodel/tools/tests/testJsonldUtils.js b/semantic-model/datamodel/tools/tests/testJsonldUtils.js index e25487e2..abfddfe5 100644 --- a/semantic-model/datamodel/tools/tests/testJsonldUtils.js +++ b/semantic-model/datamodel/tools/tests/testJsonldUtils.js @@ -44,4 +44,32 @@ describe('Test mergeContexts', function() { const result = ToTest.mergeContexts(undefined, undefined) assert(result === null) }) + it('should return only the global context', function() { + const context = 'http://context' + const result = ToTest.mergeContexts(undefined, context) + result.should.deep.equal([context]) + }) + it('should return only the local context', function() { + const context = [{ '@context': 'http://context' }] + const result = ToTest.mergeContexts(context, undefined) + result.should.deep.equal([['http://context']]) + }) + it('should merge both contexts', function() { + const globalcontext = 'http://context2' + const context = [{ '@context': 'http://context' }] + const result = ToTest.mergeContexts(context, globalcontext) + result.should.deep.equal([['http://context', 'http://context2']]) + }) + it('should merge object with string contexts', function() { + const globalcontext = 'http://context2' + const context = [{ '@context': { ns: 'http://context' } }] + const result = ToTest.mergeContexts(context, globalcontext) + result.should.deep.equal([[{ns: 'http://context'}, 'http://context2']]) + }) + it('should remove double context', function() { + const globalcontext = 'http://context2' + const context = [{ '@context': { ns: 'http://context' } }, {'@context': 'http://context2'}] + const result = ToTest.mergeContexts(context, globalcontext) + result.should.deep.equal([[{ns: 'http://context'}, 'http://context2'], ['http://context2']]) + }) }) From 7a7ec10d84309b66639c775b42e6d043ea8e3be9 Mon Sep 17 00:00:00 2001 From: marcel Date: Fri, 10 Nov 2023 23:40:05 +0100 Subject: [PATCH 24/32] Added further test for conciseexpandedform --- .../datamodel/tools/tests/testJsonldUtils.js | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/semantic-model/datamodel/tools/tests/testJsonldUtils.js b/semantic-model/datamodel/tools/tests/testJsonldUtils.js index abfddfe5..adf30f4b 100644 --- a/semantic-model/datamodel/tools/tests/testJsonldUtils.js +++ b/semantic-model/datamodel/tools/tests/testJsonldUtils.js @@ -31,7 +31,7 @@ describe('Test loadContextFromFile', function () { } } const revert = ToTest.__set__('fs', fs) - //ToTest.__set__('process', process) + // ToTest.__set__('process', process) const loadContextFromFile = ToTest.__get__('loadContextFromFile') const result = loadContextFromFile('filename') result.should.deep.equal(['context']) @@ -39,37 +39,56 @@ describe('Test loadContextFromFile', function () { }) }) -describe('Test mergeContexts', function() { - it('should return null with undefined contexts', function() { +describe('Test mergeContexts', function () { + it('should return null with undefined contexts', function () { const result = ToTest.mergeContexts(undefined, undefined) assert(result === null) }) - it('should return only the global context', function() { + it('should return only the global context', function () { const context = 'http://context' const result = ToTest.mergeContexts(undefined, context) result.should.deep.equal([context]) }) - it('should return only the local context', function() { + it('should return only the local context', function () { const context = [{ '@context': 'http://context' }] const result = ToTest.mergeContexts(context, undefined) result.should.deep.equal([['http://context']]) }) - it('should merge both contexts', function() { + it('should merge both contexts', function () { const globalcontext = 'http://context2' const context = [{ '@context': 'http://context' }] const result = ToTest.mergeContexts(context, globalcontext) result.should.deep.equal([['http://context', 'http://context2']]) }) - it('should merge object with string contexts', function() { + it('should merge object with string contexts', function () { const globalcontext = 'http://context2' const context = [{ '@context': { ns: 'http://context' } }] const result = ToTest.mergeContexts(context, globalcontext) - result.should.deep.equal([[{ns: 'http://context'}, 'http://context2']]) + result.should.deep.equal([[{ ns: 'http://context' }, 'http://context2']]) }) - it('should remove double context', function() { + it('should remove double context', function () { const globalcontext = 'http://context2' - const context = [{ '@context': { ns: 'http://context' } }, {'@context': 'http://context2'}] + const context = [{ '@context': { ns: 'http://context' } }, { '@context': 'http://context2' }] const result = ToTest.mergeContexts(context, globalcontext) - result.should.deep.equal([[{ns: 'http://context'}, 'http://context2'], ['http://context2']]) + result.should.deep.equal([[{ ns: 'http://context' }, 'http://context2'], ['http://context2']]) + }) +}) +describe('Test conciseExpandedForm', function () { + it('should concise a property', function () { + const expandedForm = [{ + '@id': 'id', + '@type': 'type', + property: { + '@type': ['https://uri.etsi.org/ngsi-ld/Property'], + 'https://uri.etsi.org/ngsi-ld/hasValue': [{ '@value': 'value' }] + } + }] + const expectedConciseForm = [{ + '@id': 'id', + '@type': 'type', + property: { '@value': 'value' } + }] + const result = ToTest.conciseExpandedForm(expandedForm) + expectedConciseForm.should.deep.equal(result) }) }) From 75095a71cb203da244bb18e8bcd546f1518cc617 Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 12 Nov 2023 16:31:52 +0100 Subject: [PATCH 25/32] extended test --- .../datamodel/tools/lib/jsonldUtils.js | 4 +- semantic-model/datamodel/tools/package.json | 2 +- .../datamodel/tools/tests/testJsonldUtils.js | 63 ++++++++++++++++++- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/semantic-model/datamodel/tools/lib/jsonldUtils.js b/semantic-model/datamodel/tools/lib/jsonldUtils.js index e0ced535..bd4a503d 100644 --- a/semantic-model/datamodel/tools/lib/jsonldUtils.js +++ b/semantic-model/datamodel/tools/lib/jsonldUtils.js @@ -107,9 +107,9 @@ function normalizeExpandedForm (expanded) { if (typeof (attr) === 'object') { if (!('@type' in attr)) { if ('https://uri.etsi.org/ngsi-ld/hasValue' in attr || '@value' in attr || '@id' in attr) { - attr['@type'] = 'https://uri.etsi.org/ngsi-ld/Property' + attr['@type'] = ['https://uri.etsi.org/ngsi-ld/Property'] } else if ('https://uri.etsi.org/ngsi-ld/hasObject' in attr) { - attr['@type'] = 'https://uri.etsi.org/ngsi-ld/Relationship' + attr['@type'] = ['https://uri.etsi.org/ngsi-ld/Relationship'] } if ('@value' in attr) { attr['https://uri.etsi.org/ngsi-ld/hasValue'] = attr['@value'] diff --git a/semantic-model/datamodel/tools/package.json b/semantic-model/datamodel/tools/package.json index b9a218a5..d9dead78 100644 --- a/semantic-model/datamodel/tools/package.json +++ b/semantic-model/datamodel/tools/package.json @@ -4,7 +4,7 @@ "description": "This package contains tools for validating NGSI-LD data for the Process Data Twin (PDT)", "main": "validate.js", "scripts": { - "test": "./node_modules/.bin/nyc --check-coverage ./node_modules/.bin/mocha --lines 80", + "test": "./node_modules/.bin/nyc --check-coverage ./node_modules/.bin/mocha tests/*.js --lines 80", "lint": "./node_modules/.bin/eslint *.js" }, "author": "Marcel Wagner", diff --git a/semantic-model/datamodel/tools/tests/testJsonldUtils.js b/semantic-model/datamodel/tools/tests/testJsonldUtils.js index adf30f4b..a8c83184 100644 --- a/semantic-model/datamodel/tools/tests/testJsonldUtils.js +++ b/semantic-model/datamodel/tools/tests/testJsonldUtils.js @@ -78,17 +78,74 @@ describe('Test conciseExpandedForm', function () { const expandedForm = [{ '@id': 'id', '@type': 'type', - property: { + property: [{ '@type': ['https://uri.etsi.org/ngsi-ld/Property'], 'https://uri.etsi.org/ngsi-ld/hasValue': [{ '@value': 'value' }] - } + }] + }] + const expectedConciseForm = [{ + '@id': 'id', + '@type': 'type', + property: [{ '@value': 'value' }] + }] + const result = ToTest.conciseExpandedForm(expandedForm) + expectedConciseForm.should.deep.equal(result) + }) + it('should concise a relationship', function () { + const expandedForm = [{ + '@id': 'id', + '@type': 'type', + hasRelationship: [{ + '@type': ['https://uri.etsi.org/ngsi-ld/Relationship'], + 'https://uri.etsi.org/ngsi-ld/hasObject': [{ '@id': 'iri' }] + }] }] const expectedConciseForm = [{ '@id': 'id', '@type': 'type', - property: { '@value': 'value' } + hasRelationship: [{ 'https://uri.etsi.org/ngsi-ld/hasObject': [{ '@id': 'iri' }] }] }] const result = ToTest.conciseExpandedForm(expandedForm) expectedConciseForm.should.deep.equal(result) }) }) +describe('Test normalizeExpandedForm', function () { + it('should normalize a property', function () { + const expandedForm = [{ + '@id': 'id', + '@type': 'type', + property: [{ + 'https://uri.etsi.org/ngsi-ld/hasValue': [{ '@value': 'value' }] + }] + }] + const expectedNormalizedForm = [{ + '@id': 'id', + '@type': 'type', + property: [{ + '@type': ['https://uri.etsi.org/ngsi-ld/Property'], + 'https://uri.etsi.org/ngsi-ld/hasValue': [{ '@value': 'value' }] + }] + }] + const result = ToTest.normalizeExpandedForm(expandedForm) + expectedNormalizedForm.should.deep.equal(result) + }) + it('should normalize a property', function () { + const expandedForm = [{ + '@id': 'id', + '@type': 'type', + hasRelationship: [{ + 'https://uri.etsi.org/ngsi-ld/hasObject': [{ '@id': 'iri' }] + }] + }] + const expectedNormalizedForm = [{ + '@id': 'id', + '@type': 'type', + hasRelationship: [{ + '@type': ['https://uri.etsi.org/ngsi-ld/Relationship'], + 'https://uri.etsi.org/ngsi-ld/hasObject': [{ '@id': 'iri' }] + }] + }] + const result = ToTest.normalizeExpandedForm(expandedForm) + expectedNormalizedForm.should.deep.equal(result) + }) +}) \ No newline at end of file From 79648f1ff26e6d786587e2b6b097ff4a3847164f Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 12 Nov 2023 17:38:25 +0100 Subject: [PATCH 26/32] first verstion of testShaclUtils --- .../datamodel/tools/tests/testShaclUtils.js | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 semantic-model/datamodel/tools/tests/testShaclUtils.js diff --git a/semantic-model/datamodel/tools/tests/testShaclUtils.js b/semantic-model/datamodel/tools/tests/testShaclUtils.js new file mode 100644 index 00000000..c969d0f3 --- /dev/null +++ b/semantic-model/datamodel/tools/tests/testShaclUtils.js @@ -0,0 +1,54 @@ +/** +* Copyright (c) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +'use strict' + +const { assert } = require('chai') +const chai = require('chai') +global.should = chai.should() + +const rewire = require('rewire') +const ToTest = rewire('../lib/shaclUtils.js') + +describe('Test class NodeShape', function () { + it('Should manage properties', function () { + const nodeShape = new ToTest.NodeShape('targetClass') + nodeShape.properties = 'property' + const properties = nodeShape.properties + properties.should.deep.equal('property') + const nodeShape2 = new ToTest.NodeShape('targetClass') + nodeShape2.properties = ['property1'] + nodeShape2.addPropertyShape('propertyShape') + nodeShape2.addPropertyShape('propertyShape2') + const properties2 = nodeShape2.properties + properties2.should.deep.equal(['property1', 'propertyShape', 'propertyShape2']) + }) +}) +describe('Test class PropertyShape', function () { + it('Should manage properties', function () { + const propertyShape = new ToTest.PropertyShape(0, 1, 'nodeKind', 'path', true) + propertyShape.addConstraint('property') + propertyShape.addConstraint('property2') + const constraints = propertyShape.constraints + constraints.should.deep.equal(['property', 'property2']) + propertyShape.mincount.should.equal(0) + propertyShape.maxcount.should.equal(1) + propertyShape.nodeKind.should.equal('nodeKind') + propertyShape.path.should.equal('path') + propertyShape.isProperty.should.equal(true) + propertyShape.propertyNode = 'node' + propertyShape.propertyNode.should.equal('node') + }) +}) From 8facfe1e7da31ff1a8657e785236bf59aee9d29a Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 12 Nov 2023 20:38:36 +0100 Subject: [PATCH 27/32] continued shaclUtils unit tests --- .../datamodel/tools/lib/shaclUtils.js | 9 +- .../datamodel/tools/tests/testShaclUtils.js | 159 ++++++++++++++++++ 2 files changed, 166 insertions(+), 2 deletions(-) diff --git a/semantic-model/datamodel/tools/lib/shaclUtils.js b/semantic-model/datamodel/tools/lib/shaclUtils.js index 78d0369f..e5b9f523 100644 --- a/semantic-model/datamodel/tools/lib/shaclUtils.js +++ b/semantic-model/datamodel/tools/lib/shaclUtils.js @@ -18,6 +18,7 @@ const $rdf = require('rdflib') const ContextUtil = require('jsonld-context-parser').Util const url = require('url') const fs = require('fs') +const path = require('path') const ContextParser = require('jsonld-context-parser').ContextParser const myParser = new ContextParser() @@ -104,8 +105,12 @@ function dumpPropertyShape (propertyShape, store) { function dumpNodeShape (nodeShape, store) { const nodeName = decodeURIComponent(globalContext.expandTerm(nodeShape.targetClass)) const parsedUrl = new url.URL(nodeName) - const fragment = parsedUrl.hash.substring(1) - const shapeName = fragment + 'Shape' + let shapeNamePrefix = parsedUrl.hash.substring(1) + const pathname = parsedUrl.pathname + if (shapeNamePrefix === '') { + shapeNamePrefix = path.basename(pathname) + } + const shapeName = shapeNamePrefix + 'Shape' store.add(IFFK(shapeName), RDF('type'), SHACL('NodeShape')) store.add(IFFK(shapeName), SHACL('targetClass'), $rdf.sym(nodeName)) nodeShape.properties.forEach((property) => { diff --git a/semantic-model/datamodel/tools/tests/testShaclUtils.js b/semantic-model/datamodel/tools/tests/testShaclUtils.js index c969d0f3..0f7cd9a8 100644 --- a/semantic-model/datamodel/tools/tests/testShaclUtils.js +++ b/semantic-model/datamodel/tools/tests/testShaclUtils.js @@ -52,3 +52,162 @@ describe('Test class PropertyShape', function () { propertyShape.propertyNode.should.equal('node') }) }) +describe('Test class Constraint', function () { + it('Should construct', function () { + const constraint = new ToTest.Constraint('type', 'param') + constraint.type.should.equal('type') + constraint.params.should.equal('param') + }) +}) +describe('Test dumpPropertyShape', function () { + it('Should dump property without constraints', function () { + const propertyShape = new ToTest.PropertyShape(1, 2, 'nodeKind', 'path', true) + propertyShape.propertyNode = 'propertyNode' + const storeAdds = [] + const store = { + add: (s, p, o) => { storeAdds.push([s, p, o]) } + } + const $rdf = { + blankNode: () => { return {} }, + Namespace: () => (x) => 'ngsild:' + x + } + const SHACL = (x) => 'shacl:' + x + const revert = ToTest.__set__('$rdf', $rdf) + ToTest.__set__('SHACL', SHACL) + ToTest.__set__('globalPrefixHash', { 'ngsi-ld': 'ngsi-ld' }) + const dumpPropertyShape = ToTest.__get__('dumpPropertyShape') + dumpPropertyShape(propertyShape, store) + storeAdds.should.deep.equal([ + ['propertyNode', 'shacl:minCount', 1], + ['propertyNode', 'shacl:maxCount', 2], + ['propertyNode', 'shacl:nodeKind', 'shacl:BlankNode'], + ['propertyNode', 'shacl:path', 'path'], + ['propertyNode', 'shacl:property', {}], + [{}, 'shacl:path', 'ngsild:hasValue'], + [{}, 'shacl:minCount', 1], + [{}, 'shacl:maxCount', 1], + [{}, 'shacl:nodeKind', 'nodeKind']]) + revert() + }) + it('Should dump property with constraints', function () { + const propertyShape = new ToTest.PropertyShape(0, 2, 'nodeKind', 'path', true) + propertyShape.propertyNode = 'propertyNode' + propertyShape.addConstraint(new ToTest.Constraint('type', 'params')) + propertyShape.addConstraint(new ToTest.Constraint('type2', ['p1', 'p2'])) + const storeAdds = [] + const store = { + add: (s, p, o) => { storeAdds.push([s, p, o]) } + } + const $rdf = { + blankNode: () => { return {} }, + Namespace: () => (x) => 'ngsild:' + x + } + const SHACL = (x) => 'shacl:' + x + const revert = ToTest.__set__('$rdf', $rdf) + ToTest.__set__('SHACL', SHACL) + ToTest.__set__('globalPrefixHash', { 'ngsi-ld': 'ngsi-ld' }) + const dumpPropertyShape = ToTest.__get__('dumpPropertyShape') + dumpPropertyShape(propertyShape, store) + storeAdds.should.deep.equal([ + ['propertyNode', 'shacl:minCount', 0], + ['propertyNode', 'shacl:maxCount', 2], + ['propertyNode', 'shacl:nodeKind', 'shacl:BlankNode'], + ['propertyNode', 'shacl:path', 'path'], + ['propertyNode', 'shacl:property', {}], + [{}, 'shacl:path', 'ngsild:hasValue'], + [{}, 'shacl:minCount', 1], + [{}, 'shacl:maxCount', 1], + [{}, 'shacl:nodeKind', 'nodeKind'], + [{}, 'type', 'params'], + [{}, 'type2', ['p1', 'p2']]]) + revert() + }) + it('Should dump relationship without constraints', function () { + const propertyShape = new ToTest.PropertyShape(1, 1, 'nodeKind', 'relationship', false) + propertyShape.propertyNode = 'propertyNode' + const storeAdds = [] + const store = { + add: (s, p, o) => { storeAdds.push([s, p, o]) } + } + const $rdf = { + blankNode: () => { return {} }, + Namespace: () => (x) => 'ngsild:' + x + } + const SHACL = (x) => 'shacl:' + x + const revert = ToTest.__set__('$rdf', $rdf) + ToTest.__set__('SHACL', SHACL) + ToTest.__set__('globalPrefixHash', { 'ngsi-ld': 'ngsi-ld' }) + const dumpPropertyShape = ToTest.__get__('dumpPropertyShape') + dumpPropertyShape(propertyShape, store) + storeAdds.should.deep.equal([ + ['propertyNode', 'shacl:minCount', 1], + ['propertyNode', 'shacl:maxCount', 1], + ['propertyNode', 'shacl:nodeKind', 'shacl:BlankNode'], + ['propertyNode', 'shacl:path', 'relationship'], + ['propertyNode', 'shacl:property', {}], + [{}, 'shacl:path', 'ngsild:hasObject'], + [{}, 'shacl:minCount', 1], + [{}, 'shacl:maxCount', 1], + [{}, 'shacl:nodeKind', 'nodeKind']]) + revert() + }) +}) +describe('Test dumpNodeShape', function () { + it('Should dump without properties', function () { + const nodeShape = new ToTest.NodeShape('http://example.com/targetClass') + const storeAdds = [] + const store = { + add: (s, p, o) => { storeAdds.push([s, p, o]) } + } + const $rdf = { + blankNode: () => { return {} }, + sym: (x) => 'sym:' + x + } + const globalContext = { + expandTerm: (x) => x + } + const revert = ToTest.__set__('SHACL', (x) => 'shacl:' + x) + ToTest.__set__('IFFK', (x) => 'iffk:' + x) + ToTest.__set__('RDF', (x) => 'rdf:' + x) + ToTest.__set__('globalContext', globalContext) + ToTest.__set__('$rdf', $rdf) + const dumpNodeShape = ToTest.__get__('dumpNodeShape') + dumpNodeShape(nodeShape, store) + storeAdds.should.deep.equal([ + ['iffk:targetClassShape', 'rdf:type', 'shacl:NodeShape'], + ['iffk:targetClassShape', 'shacl:targetClass', 'sym:http://example.com/targetClass'] + ]) + revert() + }) + it('Should dump relationship with properties', function () { + const nodeShape = new ToTest.NodeShape('http://example.com/targetClass') + const propertyShape = new ToTest.PropertyShape(0, 2, 'nodeKind', 'path', true) + nodeShape.addPropertyShape(propertyShape) + const storeAdds = [] + const store = { + add: (s, p, o) => { storeAdds.push([s, p, o]) } + } + const $rdf = { + blankNode: () => { return {} }, + sym: (x) => 'sym:' + x + } + const globalContext = { + expandTerm: (x) => x + } + const dumpPropertyShape = (x, y) => { x.propertyNode.should.deep.equal({}) } + const revert = ToTest.__set__('SHACL', (x) => 'shacl:' + x) + ToTest.__set__('IFFK', (x) => 'iffk:' + x) + ToTest.__set__('RDF', (x) => 'rdf:' + x) + ToTest.__set__('globalContext', globalContext) + ToTest.__set__('dumpPropertyShape', dumpPropertyShape) + ToTest.__set__('$rdf', $rdf) + const dumpNodeShape = ToTest.__get__('dumpNodeShape') + dumpNodeShape(nodeShape, store) + storeAdds.should.deep.equal([ + ['iffk:targetClassShape', 'rdf:type', 'shacl:NodeShape'], + ['iffk:targetClassShape', 'shacl:targetClass', 'sym:http://example.com/targetClass'], + ['iffk:targetClassShape', 'shacl:property', {}] + ]) + revert() + }) +}) From fd325d360d2d2b161e2c05bd93e036f2d5026c84 Mon Sep 17 00:00:00 2001 From: marcel Date: Sun, 12 Nov 2023 20:41:29 +0100 Subject: [PATCH 28/32] continued shaclUtils unit tests --- .../datamodel/tools/tests/testShaclUtils.js | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/semantic-model/datamodel/tools/tests/testShaclUtils.js b/semantic-model/datamodel/tools/tests/testShaclUtils.js index 0f7cd9a8..f53867db 100644 --- a/semantic-model/datamodel/tools/tests/testShaclUtils.js +++ b/semantic-model/datamodel/tools/tests/testShaclUtils.js @@ -210,4 +210,30 @@ describe('Test dumpNodeShape', function () { ]) revert() }) + it('Should dump with hash type', function () { + const nodeShape = new ToTest.NodeShape('http://example.com/example#targetClass#1#2') + const storeAdds = [] + const store = { + add: (s, p, o) => { storeAdds.push([s, p, o]) } + } + const $rdf = { + blankNode: () => { return {} }, + sym: (x) => 'sym:' + x + } + const globalContext = { + expandTerm: (x) => x + } + const revert = ToTest.__set__('SHACL', (x) => 'shacl:' + x) + ToTest.__set__('IFFK', (x) => 'iffk:' + x) + ToTest.__set__('RDF', (x) => 'rdf:' + x) + ToTest.__set__('globalContext', globalContext) + ToTest.__set__('$rdf', $rdf) + const dumpNodeShape = ToTest.__get__('dumpNodeShape') + dumpNodeShape(nodeShape, store) + storeAdds.should.deep.equal([ + ['iffk:targetClass#1#2Shape', 'rdf:type', 'shacl:NodeShape'], + ['iffk:targetClass#1#2Shape', 'shacl:targetClass', 'sym:http://example.com/example#targetClass#1#2'] + ]) + revert() + }) }) From acac8fe5345c892724b945ae219b53f1316abc07 Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 13 Nov 2023 00:01:53 +0100 Subject: [PATCH 29/32] added more unit tests --- .../datamodel/tools/tests/testShaclUtils.js | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/semantic-model/datamodel/tools/tests/testShaclUtils.js b/semantic-model/datamodel/tools/tests/testShaclUtils.js index f53867db..7d68fb2b 100644 --- a/semantic-model/datamodel/tools/tests/testShaclUtils.js +++ b/semantic-model/datamodel/tools/tests/testShaclUtils.js @@ -237,3 +237,165 @@ describe('Test dumpNodeShape', function () { revert() }) }) +describe('Test scanProperties', function () { + it('Should dump without properties', function () { + const scanProperties = ToTest.__get__('scanProperties') + const typeSchema = { + properties: { + type: { + const: 'Plasmacutter' + }, + id: { + type: 'string', + pattern: "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + + }, + required: ['type', 'id'] + } + const expectedNodeShape = { + _properties: [], + targetClass: 'targetClass' + } + const nodeShape = new ToTest.NodeShape('targetClass') + scanProperties(nodeShape, typeSchema) + nodeShape.should.deep.equal(expectedNodeShape) + }) +}) +describe('Test scanProperties', function () { + it('Should dump without properties', function () { + const scanProperties = ToTest.__get__('scanProperties') + const typeSchema = { + properties: { + type: { + const: 'Plasmacutter' + }, + id: { + type: 'string', + pattern: "^urn:[a-zA-Z0-9][a-zA-Z0-9-]{1,31}:([a-zA-Z0-9()+,.:=@;$_!*'-]|%[0-9a-fA-F]{2})*$" + } + + }, + required: ['type', 'id'] + } + const expectedNodeShape = { + _properties: [], + targetClass: 'targetClass' + } + const nodeShape = new ToTest.NodeShape('targetClass') + scanProperties(nodeShape, typeSchema) + nodeShape.should.deep.equal(expectedNodeShape) + }) + it('Should dump with property', function () { + const scanProperties = ToTest.__get__('scanProperties') + const typeSchema = { + properties: { + machine_state: { + type: 'string', + title: 'Machine Status', + description: 'Current status of the machine (Online_Idle, Run, Online_Error, Online_Maintenance, Setup, Testing)', + enum: [ + 'Testing' + ] + } + } + } + const expectedNodeShape = { + targetClass: 'targetClass', + _properties: [ + { + mincount: 0, + maxcount: 1, + nodeKind: 'shacl:Literal', + path: 'sym:machine_state', + constraints: [ + { + type: 'shacl:in', + params: [ + 'Testing'] + }], + isProperty: true + } + ] + } + const nodeShape = new ToTest.NodeShape('targetClass') + scanProperties(nodeShape, typeSchema) + nodeShape.should.deep.equal(expectedNodeShape) + }) + it('Should dump with relationship', function () { + const scanProperties = ToTest.__get__('scanProperties') + const typeSchema = { + properties: { + hasFilter: { + relationship: 'eclass:0173-1#01-ACK991#016', + $ref: 'https://industry-fusion.org/base-objects/v0.1/link' + } + } + } + const expectedNodeShape = { + targetClass: 'targetClass', + _properties: [ + { + mincount: 0, + maxcount: 1, + nodeKind: 'shacl:IRI', + path: 'sym:hasFilter', + constraints: [ + { + type: 'shacl:class', + params: 'sym:eclass:0173-1#01-ACK991#016' + } + ], + isProperty: false + }] + } + const nodeShape = new ToTest.NodeShape('targetClass') + scanProperties(nodeShape, typeSchema) + nodeShape.should.deep.equal(expectedNodeShape) + }) + it('Should dump with allOf', function () { + const scanProperties = ToTest.__get__('scanProperties') + const typeSchema = { + allOf: [ + { + properties: { + machine_state: { + type: 'string', + title: 'Machine Status', + description: 'Current status of the machine (Online_Idle, Run, Online_Error, Online_Maintenance, Setup, Testing)', + enum: [ + 'Setup', + 'Testing' + ] + } + } + } + ] + } + const expectedNodeShape = { + targetClass: 'targetClass', + _properties: [ + { + mincount: 0, + maxcount: 1, + nodeKind: 'shacl:Literal', + path: 'sym:machine_state', + constraints: [ + { + type: 'shacl:in', + params: + [ + 'Setup', + 'Testing' + ] + } + ], + isProperty: true + } + ] + } + const nodeShape = new ToTest.NodeShape('targetClass') + scanProperties(nodeShape, typeSchema) + nodeShape.should.deep.equal(expectedNodeShape) + }) +}) From c2f47130b0745904a040e0ebf19f869e12e28a02 Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 13 Nov 2023 02:02:25 +0100 Subject: [PATCH 30/32] improved test --- .../datamodel/tools/lib/shaclUtils.js | 8 +- .../datamodel/tools/tests/testShaclUtils.js | 83 ++++++++++++++++++- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/semantic-model/datamodel/tools/lib/shaclUtils.js b/semantic-model/datamodel/tools/lib/shaclUtils.js index e5b9f523..fe3a7717 100644 --- a/semantic-model/datamodel/tools/lib/shaclUtils.js +++ b/semantic-model/datamodel/tools/lib/shaclUtils.js @@ -182,16 +182,16 @@ function scanConstraints (propertyShape, typeschema) { // datatype "tag" conforms // propertyShape.addConstraint(new Constraint(SHACL('datatype'), typeschema.datatype)) } - if ('maxiumum' in typeschema) { + if ('maximum' in typeschema) { propertyShape.addConstraint(new Constraint(SHACL('maxInclusive'), typeschema.maximum)) } - if ('miniumum' in typeschema) { + if ('minimum' in typeschema) { propertyShape.addConstraint(new Constraint(SHACL('minInclusive'), typeschema.minimum)) } - if ('exclusiveMiniumum' in typeschema) { + if ('exclusiveMinimum' in typeschema) { propertyShape.addConstraint(new Constraint(SHACL('minExclusive'), typeschema.exclusiveMinimum)) } - if ('exclusiveMaxiumum' in typeschema) { + if ('exclusiveMaximum' in typeschema) { propertyShape.addConstraint(new Constraint(SHACL('maxExclusive'), typeschema.exclusiveMaximum)) } if ('maxLength' in typeschema) { diff --git a/semantic-model/datamodel/tools/tests/testShaclUtils.js b/semantic-model/datamodel/tools/tests/testShaclUtils.js index 7d68fb2b..8fbaa902 100644 --- a/semantic-model/datamodel/tools/tests/testShaclUtils.js +++ b/semantic-model/datamodel/tools/tests/testShaclUtils.js @@ -263,7 +263,7 @@ describe('Test scanProperties', function () { }) }) describe('Test scanProperties', function () { - it('Should dump without properties', function () { + it('Should scan without properties', function () { const scanProperties = ToTest.__get__('scanProperties') const typeSchema = { properties: { @@ -286,7 +286,7 @@ describe('Test scanProperties', function () { scanProperties(nodeShape, typeSchema) nodeShape.should.deep.equal(expectedNodeShape) }) - it('Should dump with property', function () { + it('Should scan with property', function () { const scanProperties = ToTest.__get__('scanProperties') const typeSchema = { properties: { @@ -322,7 +322,7 @@ describe('Test scanProperties', function () { scanProperties(nodeShape, typeSchema) nodeShape.should.deep.equal(expectedNodeShape) }) - it('Should dump with relationship', function () { + it('Should scan with relationship', function () { const scanProperties = ToTest.__get__('scanProperties') const typeSchema = { properties: { @@ -353,7 +353,7 @@ describe('Test scanProperties', function () { scanProperties(nodeShape, typeSchema) nodeShape.should.deep.equal(expectedNodeShape) }) - it('Should dump with allOf', function () { + it('Should scan with allOf', function () { const scanProperties = ToTest.__get__('scanProperties') const typeSchema = { allOf: [ @@ -399,3 +399,78 @@ describe('Test scanProperties', function () { nodeShape.should.deep.equal(expectedNodeShape) }) }) +describe('Test scanConstraints', function () { + it('Should dump without properties', function () { + const scanConstraints = ToTest.__get__('scanConstraints') + const propertyShape = new ToTest.PropertyShape(0, 2, 'nodeKind', 'path', true) + const typeSchema = { + type: 'string', + title: 'Machine Status', + description: 'Current status of the machine (Online_Idle, Run, Online_Error, Online_Maintenance, Setup, Testing)', + enum: [ + 'Online_Idle' + ], + maximum: 2, + minimum: 1, + exclusiveMinimum: 0, + exclusiveMaximum: 3, + maxLength: 100, + minLength: 10 + } + const expectedConstraints = [ + { type: 'shacl:in', params: ['Online_Idle'] }, + { type: 'shacl:maxInclusive', params: 2 }, + { type: 'shacl:minInclusive', params: 1 }, + { type: 'shacl:minExclusive', params: 0 }, + { type: 'shacl:maxExclusive', params: 3 }, + { type: 'shacl:maxLength', params: 100 }, + { type: 'shacl:minLength', params: 10 } + ] + scanConstraints(propertyShape, typeSchema) + propertyShape.constraints.should.deep.equal(expectedConstraints) + }) +}) +describe('Test encodeHash', function () { + it('Should uri-encode hash', function () { + const encodeHash = ToTest.__get__('encodeHash') + const result = encodeHash('https://example.com/test#1#2#3') + result.should.equal('https://example.com/test%231%232%233') + }) +}) +describe('Test loadContext', function () { + it('Should resolve https uri', async function () { + const loadContext = ToTest.__get__('loadContext') + const context = { + getContextRaw: () => { + return { + '@vocab': 'https://industry-fusion.org/base/v0.1/', + eclass: { + '@id': 'https://industry-fusion.org/eclass#', + '@prefix': true + }, + xsd: { + '@id': 'http://www.w3.org/2001/XMLSchema#', + '@prefix': true + }, + iffb: { + '@id': 'https://industry-fusion.org/base/v0.1/', + '@prefix': true + } + } + } + } + const myParser = { + parse: async (x) => { return context } + } + const expectedResult = { + eclass: 'https://industry-fusion.org/eclass#', + xsd: 'http://www.w3.org/2001/XMLSchema#', + iffb: 'https://industry-fusion.org/base/v0.1/' + } + const revert = ToTest.__set__('myParser', myParser) + await loadContext('https://example.com/context') + const globalPrefixHash = ToTest.__get__('globalPrefixHash') + globalPrefixHash.should.deep.equal(expectedResult) + revert() + }) +}) From 68fe4372c29253e07d486589d8d5676397707e33 Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 13 Nov 2023 11:22:50 +0100 Subject: [PATCH 31/32] added tests to workflow --- .github/workflows/build.yaml | 5 ++++- semantic-model/datamodel/tools/Makefile | 2 +- semantic-model/datamodel/tools/package.json | 2 +- semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh | 2 +- .../tools/tests/schema2shacl/schema1_c0.json_result | 2 +- semantic-model/datamodel/tools/tests/schema2shacl/test.sh | 2 +- semantic-model/datamodel/tools/tests/validation/test.sh | 2 +- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9833e87d..cc95d20c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,4 +17,7 @@ jobs: - name: Build shacl2flink run: | sudo apt install sqlite3 sqlite3-pcre - cd semantic-model/shacl2flink && make setup && make lint test build test-kms \ No newline at end of file + cd semantic-model/shacl2flink && make setup && make lint test build test-kms + - name: Build datamodel tools + run: | + cd semantic-model/datamodel && make setup && make lint test \ No newline at end of file diff --git a/semantic-model/datamodel/tools/Makefile b/semantic-model/datamodel/tools/Makefile index ccdd4aff..c62793ec 100644 --- a/semantic-model/datamodel/tools/Makefile +++ b/semantic-model/datamodel/tools/Makefile @@ -24,7 +24,7 @@ lint: shellcheck tests/*/test.sh test: - #npm run test + npm run test cd tests/validation && bash ./test.sh cd tests/jsonld2jsonld && bash ./test.sh cd tests/schema2shacl && bash ./test.sh \ No newline at end of file diff --git a/semantic-model/datamodel/tools/package.json b/semantic-model/datamodel/tools/package.json index d9dead78..a4a6d39c 100644 --- a/semantic-model/datamodel/tools/package.json +++ b/semantic-model/datamodel/tools/package.json @@ -4,7 +4,7 @@ "description": "This package contains tools for validating NGSI-LD data for the Process Data Twin (PDT)", "main": "validate.js", "scripts": { - "test": "./node_modules/.bin/nyc --check-coverage ./node_modules/.bin/mocha tests/*.js --lines 80", + "test": "./node_modules/.bin/nyc --lines 80 --check-coverage ./node_modules/.bin/mocha tests/*.js", "lint": "./node_modules/.bin/eslint *.js" }, "author": "Marcel Wagner", diff --git a/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh b/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh index 8757a973..3bfc9d5d 100644 --- a/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh +++ b/semantic-model/datamodel/tools/tests/jsonld2jsonld/test.sh @@ -23,5 +23,5 @@ while IFS='_' read -ra ADDR; do echo comparewith "$comparewith" command="node ../../jsonldConverter.js $payload -$switch -c file://$PWD/$context" echo Executing: "$command" - $command | diff "${comparewith}" - || exit 1 + $command | diff "${comparewith}" - || exit 1 done <<< "$(ls payload*.jsonld_?_c*)" diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_result b/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_result index a1107ad7..dcadd9f6 100644 --- a/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_result +++ b/semantic-model/datamodel/tools/tests/schema2shacl/schema1_c0.json_result @@ -2,6 +2,6 @@ @prefix iffk: . @prefix sh: . -iffk:Shape a sh:NodeShape; sh:targetClass iffb:Plasmacutter. +iffk:PlasmacutterShape a sh:NodeShape; sh:targetClass iffb:Plasmacutter. diff --git a/semantic-model/datamodel/tools/tests/schema2shacl/test.sh b/semantic-model/datamodel/tools/tests/schema2shacl/test.sh index 5ab51311..a0508fb7 100644 --- a/semantic-model/datamodel/tools/tests/schema2shacl/test.sh +++ b/semantic-model/datamodel/tools/tests/schema2shacl/test.sh @@ -23,5 +23,5 @@ while IFS='_' read -ra ADDR; do id=${file}_id command="node ../../jsonschema2shacl.js -s $file -c file://$PWD/$context -i $(cat "$id")" echo Executing: "$command" - $command | diff "${comparewith}" - + $command | diff "${comparewith}" - || exit 1 done <<< "$(ls schema*_*.json)" diff --git a/semantic-model/datamodel/tools/tests/validation/test.sh b/semantic-model/datamodel/tools/tests/validation/test.sh index a21b36fc..abbdbcdd 100644 --- a/semantic-model/datamodel/tools/tests/validation/test.sh +++ b/semantic-model/datamodel/tools/tests/validation/test.sh @@ -23,6 +23,6 @@ while IFS='_' read -ra ADDR; do idname=${dataname}_id command="node ../../validate.js -s ${schemaname} -d ${dataname} -i $(cat "${idname}")" echo -n "Executing test $command ... " - $command | diff "${resultname}" - + $command | diff "${resultname}" - || exit 1 echo OK done <<< "$(ls validation*_data*.json)" From 9cff7488e4bf6acf9e1c5b13dc3cfe12cbedee61 Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 13 Nov 2023 11:44:30 +0100 Subject: [PATCH 32/32] Makefile --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cc95d20c..88bf1df1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,4 +20,4 @@ jobs: cd semantic-model/shacl2flink && make setup && make lint test build test-kms - name: Build datamodel tools run: | - cd semantic-model/datamodel && make setup && make lint test \ No newline at end of file + cd semantic-model/datamodel/tools && make setup && make lint test \ No newline at end of file