From 3e8310dcca10a12caf6ddb09ca1c23845771a839 Mon Sep 17 00:00:00 2001 From: marcel Date: Sat, 28 Dec 2024 01:57:57 +0100 Subject: [PATCH] Add tutorial, implementation matrix and execution instructions to shacl2flink directory Signed-off-by: marcel --- KafkaBridge/config/config.json | 3 +- KafkaBridge/config/knowledge.ttl | 1 - semantic-model/shacl2flink/Makefile | 1 - semantic-model/shacl2flink/README.md | 105 ++- .../shacl2flink/create_rdf_table.py | 6 - semantic-model/shacl2flink/docs/examples.md | 321 +++++++ .../files/cutter-filter-full-high-temp.jsonld | 40 + .../files/cutter-filter-full-low-temp.jsonld | 40 + .../files/cutter-filter-full-no-filter.jsonld | 20 + .../files/cutter-filter-full-no-temp.jsonld | 36 + ...tter-filter-full-not-aligned-states.jsonld | 40 + .../cutter-filter-full-wrong-filter.jsonld | 40 + .../cutter-filter-full-wrong-state.jsonld | 40 + .../docs/files/cutter-filter-full.jsonld | 40 + .../files/cutter-filter-relationship.jsonld | 25 + .../docs/files/cutter-filter-relationship.ttl | 9 + .../docs/files/cutter-temperature.jsonld | 14 + .../docs/files/cutter-temperature.ttl | 8 + .../shacl2flink/docs/files/knowledge.ttl | 23 + .../shacl2flink/docs/files/shacl.ttl | 74 ++ .../shacl2flink/docs/supported-features.md | 858 ++++++++++++++++++ 21 files changed, 1722 insertions(+), 22 deletions(-) delete mode 120000 KafkaBridge/config/knowledge.ttl create mode 100644 semantic-model/shacl2flink/docs/examples.md create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-full-high-temp.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-full-low-temp.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-full-no-filter.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-full-no-temp.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-full-not-aligned-states.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-full-wrong-filter.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-full-wrong-state.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-full.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-relationship.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-filter-relationship.ttl create mode 100644 semantic-model/shacl2flink/docs/files/cutter-temperature.jsonld create mode 100644 semantic-model/shacl2flink/docs/files/cutter-temperature.ttl create mode 100644 semantic-model/shacl2flink/docs/files/knowledge.ttl create mode 100644 semantic-model/shacl2flink/docs/files/shacl.ttl create mode 100644 semantic-model/shacl2flink/docs/supported-features.md diff --git a/KafkaBridge/config/config.json b/KafkaBridge/config/config.json index c6fc5093..5e6ec5a0 100644 --- a/KafkaBridge/config/config.json +++ b/KafkaBridge/config/config.json @@ -54,8 +54,7 @@ "debeziumBridge": { "topic": "iff.ngsild.public.entity", "entityTopicPrefix": "iff.ngsild.entities", - "attributesTopic": "iff.ngsild.attributes", - "rdfSources": ["../config/knowledge.ttl"] + "attributesTopic": "iff.ngsild.attributes" }, "bridgeCommon": { "kafkaSyncOnAttribute": "https://industry-fusion.com/types/v0.9/metadata/kafkaSyncOn" diff --git a/KafkaBridge/config/knowledge.ttl b/KafkaBridge/config/knowledge.ttl deleted file mode 120000 index 533e04be..00000000 --- a/KafkaBridge/config/knowledge.ttl +++ /dev/null @@ -1 +0,0 @@ -../../semantic-model/kms/knowledge.ttl \ No newline at end of file diff --git a/semantic-model/shacl2flink/Makefile b/semantic-model/shacl2flink/Makefile index 8fa882ed..66227857 100644 --- a/semantic-model/shacl2flink/Makefile +++ b/semantic-model/shacl2flink/Makefile @@ -114,7 +114,6 @@ flink-deploy: clean # make build || make enable-strimzi kubectl -n $(NAMESPACE) delete -f output/ngsild-kafka.yaml --ignore-not-found || make enable-strimzi kubectl -n $(NAMESPACE) delete -f output/rdf-kafka.yaml --ignore-not-found || make enable-strimzi - kubectl -n $(NAMESPACE) delete -f output/knowledge-configmap.yaml --ignore-not-found cd ../../helm && ./helmfile -f helmfile-shacl.yaml apply || make enable-strimzi make enable-strimzi make test-flink-is-deployed diff --git a/semantic-model/shacl2flink/README.md b/semantic-model/shacl2flink/README.md index 2ffbf3f6..0fc963aa 100644 --- a/semantic-model/shacl2flink/README.md +++ b/semantic-model/shacl2flink/README.md @@ -1,10 +1,32 @@ -# Digital Twin Shacl2Flink +# Digital Twin Shacl to Flink Transformation + +This directory contains the translation mechanism from SHACL basee constraints and rules to SQL/Flink. +There are always three ingredients to such a translation, called KMS (Knowledge, Model-instance, SHACL) + +- **K**nowledge contains OWL/RDF data, preferable serialized in Turtle +- **M**odel-instance describes the actual instances/objects of the setup. These are described in JSON-LD/NGSI-LD. +- **S**HACL is the W3C standard describing the constraints and rules for the model with respect to the Knowledge. + +A first [overview](../datamodel/README.md) and [tutorial](../datamodel/Tutorial.md) can be found in the [datamodel](../datamodel/) directory. + +# Table of Contents + +1. [Quick Setup](#quick-setup) +2. [KMS Examples & Tutorial](./docs/examples.md) +3. [Supported SHACL Features](./docs/supported-features.md) +4. User Defined Functions +3. [Build and test KMS](#build-and-test-kms) +4. [Deploy Flink-Jobs](#deploy-flink-jobs) +5. [References](#references) + +# Quick Setup ## Requirements -- You need to have Python > 3.8 -- Virtualenv needs to be installed -- `sqlite3` and `sqlite3-pcre` need to be installed +- Python > 3.8 +- Virtualenv installed + +In addition, install `sqlite3` and `sqlite3-pcre` ```bash sudo apt install sqlite3 sqlite3-pcre @@ -23,12 +45,6 @@ source venv/bin/activate make setup ``` -Then run: - -```bash -make build -``` - ## VS Code Normally VS Code should recognize the virtual environment and ask you if you want to use the virtual environment as you Python interpreter. @@ -49,7 +65,7 @@ pip install -r requirements-dev.txt Run with ```bash -python -m pytest +make test ``` ## Linting @@ -57,4 +73,69 @@ Run with ```bash make lint -``` \ No newline at end of file +``` + + +# Build and Test KMS +## Build KMS directory + +There are three files expected in the `../kms` directory: + +- shacl.ttl +- knowledge.ttl +- model-instance.ttl + +To build: + +```bash +make build +``` + +As a result, there must be a new directory `output` with the following files included: + +- **core.yaml** - SQL-Tables for Flink (Core tables are used independent of concrete SHACL rules) +- **core.sqlite** - SQL-Tables for SQLite (Core tables are used independent of concrete SHACL rules) +- **shacl-validation.yaml** - From SHACL compiled SQL scripts for Flink +- **shacl-validation.sqlite** - From SHACL compiled SQL scripts for SQLite +- **shacl-validation-maps.yaml** - Additional SQL scripts when result is too large to store in **shacl-validation.sqlite** directly +- **rdf.sqlite** - Knowledge translated to RDF triples for SQLite +- **rdf.yaml** - Knowledge translated to RDF triples for Flink +- **ngsild-kafka.yaml** - Kafka topics used by Flink +- **ngsild-models.sqlite** - translated model-instance.ttl for SQLite (only for SQLite needed) +- **ngsild.sqlite** - SQL tables for the concrete SHACL rules generated for SQLite +- **ngsild.yaml** - SQL tables for the concrete SHACL rules generated for Flink +- **rdf-kafka.yaml** - Kafka topic for rdf data +- **rdf-maps.yaml** - RDF data add-on when data is too much to fit into **rdf.yaml** +- **udf.yaml** - User Defined Functions (UDF) for Flink SQL + + +## Test locally with SQLite + +```bash +make test-sqlite +``` + +# Deploy Flink Jobs + +## Deploy SHACL rules to Flink + +```bash +make flink-deploy +``` + +## Undeploy SHACL rules to Flink + +```bash +make flink-undeploy +``` + +# References + +[RDF] RDF +[RDFS] RDFS +[TURTLE] TURTLE +[OWL] OWL +[SHACL] SHACL +[JSONLD] JSON-LD +[XSD] XSD +[SPARQL] diff --git a/semantic-model/shacl2flink/create_rdf_table.py b/semantic-model/shacl2flink/create_rdf_table.py index 2a1c873c..0ee3aa0f 100644 --- a/semantic-model/shacl2flink/create_rdf_table.py +++ b/semantic-model/shacl2flink/create_rdf_table.py @@ -166,12 +166,6 @@ def main(knowledgefile, namespace, output_folder='output'): configs.rdf_topic, configs.kafka_topic_object_label, config), fk) - with open(os.path.join(output_folder, "knowledge-configmap.yaml"), "w") as fp: - qres = g.query(filter_out) - class_ttl = {} - class_ttl['knowledge.ttl'] = qres.serialize(format='turtle').decode("utf-8") - fp.write("---\n") - yaml.dump(utils.create_configmap_generic('knowledge', class_ttl), fp) if __name__ == '__main__': diff --git a/semantic-model/shacl2flink/docs/examples.md b/semantic-model/shacl2flink/docs/examples.md new file mode 100644 index 00000000..b9bef48e --- /dev/null +++ b/semantic-model/shacl2flink/docs/examples.md @@ -0,0 +1,321 @@ +# KMS Examples & Tutorial + +## Introduction to KMS + +*KMS* combines three essential ingredients for semantic data modeling: + +1. **K (Knowledge):** Encodes domain taxonomies and ontologies in OWL/RDF, typically serialized in Turtle format. This represents the conceptual structure of the domain. +2. **M (Model-Instance):** Describes real-world instances and their relationships in NGSI-LD/JSON-LD format. +3. **S (SHACL):** Specifies constraints and rules to validate instances against the Knowledge model. + +This tutorial demonstrates how to combine these elements to ensure data integrity and semantic consistency. By the end, you will understand how to define NGSI-LD instances, create an OWL knowledge model, and validate data using SHACL. + +--- + +## NGSI-LD Instance Model + +### Defining Instances + +Let’s define two entities, a **Cutter** and a **Filter**, and their relationship: + +1. The Cutter (`urn:iff:cutter:1`) has: + - A **relationship**: `hasFilter` pointing to the Filter (`urn:iff:filter:1`). + - Two **properties**: + - `hasState` with the value `executingState`. + - `hasTemperature` with the value `30.5`. + +2. The Filter (`urn:iff:filter:1`) has: + - One **property**: `hasState` with the value `executingState`. + +Here’s the JSON-LD representation: + +```json +[ + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { "@vocab": "https://industry-fusion.org/base/v1/" } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:iff:filter:1" + }, + "hasState": { + "type": "Property", + "value": { "@id": "https://industry-fusion.org/ontology/v1/executingState" } + }, + "hasTemperature": { + "type": "Property", + "value": 30.5 + } + }, + { + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { "@vocab": "https://industry-fusion.org/base/v1/" } + ], + "id": "urn:iff:filter:1", + "type": "Filter", + "hasState": { + "type": "Property", + "value": "https://industry-fusion.org/ontology/v1/executingState" + } + } +] +``` + +### Key Concepts: + +- **Properties**: Attributes like `hasState` and `hasTemperature` provide data about an entity. +- **Relationships**: Connections between entities, such as `hasFilter`. +- **Unique Identifiers**: Each entity has a unique `id`. +- **International Resource Identifiers (IRIs)**: IRIs are encoded in JSON-LD by `"@id": "iri"` + +--- +## OWL Knowledge Model + +The Knowledge model defines the conceptual structure and relationships between types. For this example: + +1. **Machine Taxonomy**: + - `Cutter` and `Filter` are subtypes of `Machine`. + +2. **States**: + - The `MachineState` class includes: + - `executingState` + - `errorState` + - `notExecutingState` + +Here’s the OWL representation in Turtle syntax: + +```turtle +@prefix rdfs: . +@prefix owl: . +@prefix iffbase: . +@prefix ontology: . + +iffbase:Machine a owl:Class . + +iffbase:Cutter a owl:Class ; + rdfs:subClassOf iffbase:Machine . + +iffbase:Filter a owl:Class ; + rdfs:subClassOf iffbase:Machine . + +ontology:MachineState a owl:Class . + +ontology:executingState a ontology:MachineState . +ontology:errorState a ontology:MachineState . +ontology:notExecutingState a ontology:MachineState . +``` + +--- + +## SHACL Shapes and Validation + +### Understanding SHACL + +SHACL shapes define constraints and rules for validating data against the Knowledge model. They operate on graph structures and can validate both properties and relationships. + +### Example: Cutter Temperature Constraint + +We want to validate that: + +**Every Cutter must have exactly one `hasTemperature` property of type `double`.** + +#### SHACL Shape Definition: + +```turtle +@prefix sh: . +@prefix iffbase: . +@prefix xsd: . + +cutterTemperatureShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:nodeKind sh:BlankNode ; + sh:property [ + sh:path ngsi-ld:hasValue ; + sh:datatype xsd:double ; + ] + ] . +``` + +--- + +### Adding Min/Max Constraints + +To specify acceptable temperature ranges, we extend the shape: + +**Constraint**: Temperature must be between 20.0 (inclusive) and 50.0 (exclusive). + +```turtle +cutterTemperatureWithMinMaxShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:nodeKind sh:BlankNode ; + sh:property [ + sh:path ngsi-ld:hasValue ; + sh:datatype xsd:double ; + sh:minInclusive 20.0 ; + sh:maxExclusive 50.0 ; + ] + ] . +``` + +--- + +### Relationship Constraints + +**Constraint**: Every Cutter must have at most one `hasFilter` relationship pointing to an entity of type `Filter`. + +```turtle +cutterLinkedToFilterShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; + sh:property [ + sh:path iffbase:hasFilter ; + sh:maxCount 1 ; + sh:nodeKind sh:BlankNode ; + sh:property [ + sh:path ngsi-ld:hasObject ; + sh:class iffbase:Filter ; + ] + ] . +``` + +--- + +### Using Ontology Links + +**Constraint**: Every Machine must have at least one `hasState` property of class `MachineState`. + +```turtle +machineStateShape a sh:NodeShape ; + sh:targetClass iffbase:Machine ; + sh:property [ + sh:path iffbase:hasState ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:nodeKind sh:BlankNode ; + sh:property [ + sh:path ngsi-ld:hasValue ; + sh:class ontology:MachineState ; + ] + ] . +``` + +--- + +## Advanced Constraints with SPARQL + +SHACL supports custom constraints using SPARQL. For example: + +**Constraint**: A Cutter in `executingState` must have a linked Filter also in `executingState`. + +#### SPARQL Query: + +```sparql +SELECT ?this ?filter WHERE { + ?this iffbase:hasFilter [ ngsi-ld:hasObject ?filter ] . + ?this iffbase:hasState [ ngsi-ld:hasValue ?cstate ] . + ?filter iffbase:hasState [ ngsi-ld:hasValue ?fstate ] . + FILTER (?cstate = ontology:executingState && ?fstate != ontology:executingState) +} +``` + +#### SHACL Shape with SPARQL: + +```turtle +:cutterStateInLineWithFilterShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; + sh:sparql [ + a sh:SPARQLConstraint ; + sh:message "Cutter {?this} executing without executing filter {?filter}" ; + sh:select """ + PREFIX ontology: + PREFIX iffbase: + PREFIX ngsi-ld: + + SELECT ?this ?filter WHERE { + ?this iffbase:hasFilter [ ngsi-ld:hasObject ?filter ] . + ?this iffbase:hasState [ ngsi-ld:hasValue ?cstate ] . + ?filter iffbase:hasState [ ngsi-ld:hasValue ?fstate ] . + FILTER (?cstate = ontology:executingState && ?fstate != ontology:executingState) + } + """ ; + ] . +``` + + + +## Construct Nodes with SHACL Rules + +TBD + +--- + +## Validation with PyShacl + +### Precondition + +- Python >= 3.9 +- PyShacl is installed `pip install pyshacl` +- Working directory is `docs` + +### Files needed for validation + +#### NGSI-LD Instances +- [./files/cutter-filter-full.jsonld](./files/cutter-filter-full.jsonld) - Correct Cutter and Filter instances +- [./files/cutter-filter-full-no-temp.jsonld](cutter-filter-full-no-temp.jsonld) - Missing Temperature in Cutter +- [./files/cutter-filter-full-high-temp.jsonld](./files/cutter-filter-full-high-temp.jsonld) - Too temperature +- [./files/cutter-filter-full-low-temp.jsonld](./files/cutter-filter-full-low-temp.jsonld]) - Too low temperature +- [./files/cutter-filter-full-no-filter.jsonld](./files/cutter-filter-full-no-filter.jsonld) - Cutter does not have a `hasFilter` Relationship +- [./files/cutter-filter-full-wrong-filter.jsonld](./files/cutter-filter-full-wrong-filter.jsonld) - Cutter is linked with Filter of wrong type +- [./files/cutter-filter-full-wrong-state.jsonld](./files/cutter-filter-full-wrong-state.jsonld) - Cutter state is not of class `ontology:MachineState` +- [./files/cutter-filter-full-not-aligned-states.jsonld](./files/cutter-filter-full-not-aligned-states.jsonld) - Cutter state in on `ontology:executingState` but connected Filter has wrong state + +#### SHACL file + +[./files/shacl.ttl](./files/shacl.ttl) + +#### Knowledge/Ontology File + +[./files/knowledge.ttl](./files/knowledge.ttl) + +### Validation Examples + +Validate an NGSI-LD Instace with + + pyshacl -s ./files/shacl.ttl -df json-ld ./files/ -e files/knowledge.ttl + +To simplify the output we append `| egrep 'Conforms|Message'` to the validation. For instance, validation of the correct instances is done by + + pyshacl -s ./files/shacl.ttl -df json-ld ./files/cutter-filter-full.jsonld -e ./files/knowledge.ttl | egrep 'Conforms|Message' +Output: + + Conforms: True + +Validate the low temparature example: + + pyshacl -s ./files/shacl.ttl -df json-ld ./files/cutter-filter-full-low-temp.jsonld -e ./files/knowledge.ttl | egrep 'Conforms|Message' + +Output: + + Conforms: False + Message: Value is not >= Literal("20.0", datatype=xsd:decimal) + +Validate the SPARQL query example: + + pyshacl -s ./files/shacl.ttl -df json-ld ./files/cutter-filter-full-not-aligned-states.jsonld -e ./files/knowledge.ttl | egrep 'Conforms|Message' + +Output: + + Conforms: False + Message: Cutter urn:iff:cutter:1 running without running filter urn:iff:filter:1 diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-full-high-temp.jsonld b/semantic-model/shacl2flink/docs/files/cutter-filter-full-high-temp.jsonld new file mode 100644 index 00000000..9f29335b --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-full-high-temp.jsonld @@ -0,0 +1,40 @@ +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:iff:filter:1" + }, + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + }, + "hasTemperature": { + "type": "Property", + "value": 50.0 + } +}, +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:filter:1", + "type": "Filter", + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + } +}] \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-full-low-temp.jsonld b/semantic-model/shacl2flink/docs/files/cutter-filter-full-low-temp.jsonld new file mode 100644 index 00000000..b7aa55d9 --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-full-low-temp.jsonld @@ -0,0 +1,40 @@ +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:iff:filter:1" + }, + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + }, + "hasTemperature": { + "type": "Property", + "value": 19.99 + } +}, +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:filter:1", + "type": "Filter", + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + } +}] \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-full-no-filter.jsonld b/semantic-model/shacl2flink/docs/files/cutter-filter-full-no-filter.jsonld new file mode 100644 index 00000000..e3d5f902 --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-full-no-filter.jsonld @@ -0,0 +1,20 @@ +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + }, + "hasTemperature": { + "type": "Property", + "value": 30.5 + } +}] \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-full-no-temp.jsonld b/semantic-model/shacl2flink/docs/files/cutter-filter-full-no-temp.jsonld new file mode 100644 index 00000000..6196186f --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-full-no-temp.jsonld @@ -0,0 +1,36 @@ +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:iff:filter:1" + }, + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + } +}, +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:filter:1", + "type": "Filter", + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + } +}] \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-full-not-aligned-states.jsonld b/semantic-model/shacl2flink/docs/files/cutter-filter-full-not-aligned-states.jsonld new file mode 100644 index 00000000..3f88e568 --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-full-not-aligned-states.jsonld @@ -0,0 +1,40 @@ +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:iff:filter:1" + }, + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + }, + "hasTemperature": { + "type": "Property", + "value": 30.5 + } +}, +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:filter:1", + "type": "Filter", + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/errorState" + } + } +}] \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-full-wrong-filter.jsonld b/semantic-model/shacl2flink/docs/files/cutter-filter-full-wrong-filter.jsonld new file mode 100644 index 00000000..87f6449f --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-full-wrong-filter.jsonld @@ -0,0 +1,40 @@ +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:iff:filter:1" + }, + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + }, + "hasTemperature": { + "type": "Property", + "value": 30.5 + } +}, +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:filter:1", + "type": "WrongFilter", + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + } +}] \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-full-wrong-state.jsonld b/semantic-model/shacl2flink/docs/files/cutter-filter-full-wrong-state.jsonld new file mode 100644 index 00000000..8cd40953 --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-full-wrong-state.jsonld @@ -0,0 +1,40 @@ +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:iff:filter:1" + }, + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/wrongState" + } + }, + "hasTemperature": { + "type": "Property", + "value": 30.5 + } +}, +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:filter:1", + "type": "Filter", + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + } +}] \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-full.jsonld b/semantic-model/shacl2flink/docs/files/cutter-filter-full.jsonld new file mode 100644 index 00000000..1c7a362d --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-full.jsonld @@ -0,0 +1,40 @@ +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:iff:filter:1" + }, + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + }, + "hasTemperature": { + "type": "Property", + "value": 30.5 + } +}, +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:filter:1", + "type": "Filter", + "hasState": { + "type": "Property", + "value": { + "@id": "https://industry-fusion.org/ontology/v1/executingState" + } + } +}] \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-relationship.jsonld b/semantic-model/shacl2flink/docs/files/cutter-filter-relationship.jsonld new file mode 100644 index 00000000..f1d8903f --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-relationship.jsonld @@ -0,0 +1,25 @@ +[{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasFilter": { + "type": "Relationship", + "object": "urn:iff:filter:1" + } +}, +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:filter:1", + "type": "Filter" +} +] \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-filter-relationship.ttl b/semantic-model/shacl2flink/docs/files/cutter-filter-relationship.ttl new file mode 100644 index 00000000..a656a541 --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-filter-relationship.ttl @@ -0,0 +1,9 @@ +@prefix ns1: . +@prefix ns2: . + + a ns1:Cutter ; + ns1:hasFilter [ a ns2:Relationship ; + ns2:hasObject ] . + + a ns1:Filter . + diff --git a/semantic-model/shacl2flink/docs/files/cutter-temperature.jsonld b/semantic-model/shacl2flink/docs/files/cutter-temperature.jsonld new file mode 100644 index 00000000..0aa75129 --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-temperature.jsonld @@ -0,0 +1,14 @@ +{ + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + { + "@vocab": "https://industry-fusion.org/base/v1/" + } + ], + "id": "urn:iff:cutter:1", + "type": "Cutter", + "hasTemperature": { + "type": "Property", + "value": 30.5 + } + } \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/cutter-temperature.ttl b/semantic-model/shacl2flink/docs/files/cutter-temperature.ttl new file mode 100644 index 00000000..e2f2f078 --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/cutter-temperature.ttl @@ -0,0 +1,8 @@ +@prefix : . +@prefix ngsi-ld: . +@prefix xsd: . + + a :Cutter ; + :hasTemperature [ a ngsi-ld:Property ; + ngsi-ld:hasValue 3.05e+01 ] . + diff --git a/semantic-model/shacl2flink/docs/files/knowledge.ttl b/semantic-model/shacl2flink/docs/files/knowledge.ttl new file mode 100644 index 00000000..9b253085 --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/knowledge.ttl @@ -0,0 +1,23 @@ +@prefix rdfs: . +@prefix owl: . +@prefix iffbase: . +@prefix ontology: . + + +iffbase:Machine a owl:NamedIndividual, + owl:Class . +iffbase:Cutter a owl:NamedIndividual, + owl:Class ; + rdfs:subClassOf iffbase:Machine . +iffbase:Filter a owl:NamedIndividual, + owl:Class ; + rdfs:subClassOf iffbase:Machine . + +ontology:MachineState a owl:NamedIndividual, + owl:Class . +ontology:executingState a owl:NamedIndividual, + ontology:MachineState . +ontology:errorState a owl:NamedIndividual, + ontology:MachineState . +ontology:notExecutingState a owl:NamedIndividual, + ontology:MachineState . \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/files/shacl.ttl b/semantic-model/shacl2flink/docs/files/shacl.ttl new file mode 100644 index 00000000..7ff04149 --- /dev/null +++ b/semantic-model/shacl2flink/docs/files/shacl.ttl @@ -0,0 +1,74 @@ +@prefix : . +@prefix ngsi-ld: . +@prefix sh: . +@prefix xsd: . +@prefix ontology: . +@prefix iffbase: . + +:cutterTemperatureWithMinMaxShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; + sh:property [ + sh:maxCount 1 ; + sh:minCount 1 ; + sh:nodeKind sh:BlankNode ; + sh:path iffbase:hasTemperature ; + sh:property [ + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:Literal ; + sh:path ngsi-ld:hasValue ; + sh:datatype xsd:double ; + sh:minInclusive 20.0 ; + sh:maxExclusive 50.0 ; + ] ; + ] . + +:cutterLinkedToFilterShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; + sh:property [ + sh:maxCount 1; + sh:nodeKind sh:BlankNode; + sh:path iffbase:hasFilter; + sh:property [ + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI; + sh:path ngsi-ld:hasObject; + sh:class iffbase:Filter + ] + ] +. + +:machineStateShape a sh:NodeShape ; + sh:targetClass iffbase:Machine ; + sh:property [ + sh:maxCount 1; + sh:nodeKind sh:BlankNode; + sh:path iffbase:hasState; + sh:property [ + sh:minCount 1 ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI; + sh:path ngsi-ld:hasValue; + sh:class ontology:MachineState + ] + ] +. + +:cutterStateInLineWithFilterShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; + sh:sparql [ a sh:SPARQLConstraints ; + sh:message "Cutter {?this} running without running filter {?filter}" ; + sh:select """ + PREFIX ontology: + PREFIX iffbase: + PREFIX ngsi-ld: + + SELECT ?this ?filter WHERE { + ?this iffbase:hasFilter [ ngsi-ld:hasObject ?filter] . + ?this iffbase:hasState [ ngsi-ld:hasValue ?cstate ] . + ?filter iffbase:hasState [ngsi-ld:hasValue ?fstate] . + FILTER( ?cstate = ontology:executingState && ?fstate != ontology:executingState) + } + """ ; + ]. \ No newline at end of file diff --git a/semantic-model/shacl2flink/docs/supported-features.md b/semantic-model/shacl2flink/docs/supported-features.md new file mode 100644 index 00000000..c16b9d27 --- /dev/null +++ b/semantic-model/shacl2flink/docs/supported-features.md @@ -0,0 +1,858 @@ +# Supported Features for the SHACL to Flink transformation + +## Target Nodes + +SHACL defines a mechanism to select the node which is validated. The following mechanisms are supported + + + + + + + + + + + + + +
Feature Example Implemented
+ +```turtle +sh:targetNode +``` + + +```turtle +cutterTemperatureShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; +``` + +✓
+ + +## Constraint Components + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Feature Example Implemented
+ +```turtle +sh:class +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasState ; + sh:property [ + sh:class ontology:MachineState ; + ] + ] . +``` + +✓
+ +```turtle +sh:datatype +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:datatype xsd:double ; + ] + ] . +``` + +✗
+ +```turtle +sh:nodeKind +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:nodeKind sh:Literal ; + ] + ] . +``` + +✓
+ +```turtle +sh:minCount +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:minCount 1 ; + ] . +``` + +✓
+ +```turtle +sh:maxCount +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:maxCount 1 ; + ] . +``` + +✓
+ +```turtle +sh:minExclusive +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:minExclusive 20.0 ; + ] + ] . +``` + +✓
+ +```turtle +sh:minInclusive +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:minExclusive 20.0 ; + ] + ] . +``` + +✓
+ +```turtle +sh:maxExclusive +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:maxExclusive 50.0 ; + ] + ] . +``` + +✓
+ +```turtle +sh:maxInclusive +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:maxInclusive 50.0 ; + ] + ] . +``` + +✓
+ +```turtle +sh:minLength +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasStringExample ; + sh:property [ + sh:minLength 5 ; + ] + ] . +``` + +✓
+ +```turtle +sh:maxLength +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasStringExample ; + sh:property [ + sh:maxLength 5 ; + ] + ] . +``` + +✓
+ +```turtle +sh:pattern +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasStringExample ; + sh:property [ + sh:pattern "^1\\.\\d{4,5}" ; + ] + ] . +``` + +✓
+ +```turtle +sh:in +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasStringExample ; + sh:property [ + sh:in ("Hello" "World") ; + ] + ] . +``` + +✓
+ +```turtle +sh:hasValue +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:path iffbase:hasStringExample ; + sh:property [ + sh:hasValue "Hello World" ; + ] + ] . +``` + +✗
+ +```turtle +sh:not +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:not + [ + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:maxInclusive 50.0 ; + ] + ] + ] . +``` + +✗
+ +```turtle +sh:or +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:or ( + [ + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:maxInclusive 50.0 ; + ] + ] + ] + [ + sh:property [ + sh:path iffbase:hasTemperature2 ; + sh:property [ + sh:minInclusive 20.0 ; + ] + ] + ] + ) . +``` + +✗
+ +```turtle +sh:and +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:and ( + [ + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:maxInclusive 50.0 ; + ] + ] + ] + [ + sh:property [ + sh:path iffbase:hasTemperature2 ; + sh:property [ + sh:minInclusive 20.0 ; + ] + ] + ] + ) . +``` + +✗
+ +```turtle +sh:xone +``` + + + +```turtle +:demoShape a sh:NodeShape ; + sh:xone ( + [ + sh:property [ + sh:path iffbase:hasTemperature ; + sh:property [ + sh:maxInclusive 50.0 ; + ] + ] + ] + [ + sh:property [ + sh:path iffbase:hasTemperature2 ; + sh:property [ + sh:minInclusive 20.0 ; + ] + ] + ] + ) . +``` + +✗
+ +```turtle +sh:sparql +``` + + +```turtle +:demoShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; + sh:sparql [ + a sh:SPARQLConstraint ; + sh:select """ + """ + ] . +``` + +✓
+ +```turtle +sh:message +``` + + +```turtle +:demoShape a sh:NodeShape ; + sh:targetClass iffbase:Cutter ; + sh:sparql [ + sh:message "Cutter {?this} executing without executing filter {?filter}" ; + ] . +``` + +✓
+ +```turtle +sh:severity +``` + + +```turtle +:demoShape a sh:NodeShape ; + sh:property [ + sh:severity iffbase:severityCritical ] ; + sh:path iffbase:hasAttribute ; + ] . +``` + +✓
+ +## Sparql based constraints + +### Sparql Query + +The following SPARQL features are supported + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Feature Example Implemented
+ +``` +Basic Graph Pattern (BGP) +``` + + +```sparql + ?this iffbase:hasFilter [ ngsi-ld:hasObject ?filter ] . + ?this iffbase:hasState [ ngsi-ld:hasValue ?cstate ] . + ?filter iffbase:hasState [ ngsi-ld:hasValue ?fstate ] . +``` + +✓
+ +``` + +OPTIONAL {} +(only single triple supported) +``` + + +```sparql +OPTIONAL{ ?this iffbase:hasFilter [ ngsi-ld:hasObject ?filter ] } +``` + +✓
+ +``` +BIND +``` + + +```turtle +BIND("hello world") as ?value +``` + +✓
+ +``` +BOUND +``` + + +```turtle +BOUND(?value) +``` + +✓
+ +``` +IF +``` + + +```turtle +IF(condition, true, false) +``` + +✓
+ +``` +FILTER +``` + + +```turtle +FILTER (?cstate = ontology:executingState && ?fstate != ontology:executingState) +``` + +✓
+ +``` +ConditionalOrExpression +``` + + +```turtle +?cstate = ontology:executingState || ?fstate != ontology:executingState +``` + +✓
+ +``` +ConditionalAndExpression +``` + + +```turtle +?cstate = ontology:executingState && ?fstate != ontology:executingState +``` + +✓
+ +``` +RelationalExpression +(=, !=, <,>, <=, >=, IN, NOT IN) +``` + + +```turtle +?x > 5 +``` + +✓
+ +``` +JOIN +``` + + +```sparql +{BGP} +{BGP} +``` + +✓
+ +``` +NOT EXISTS +``` + + +```turtle + FILTER NOT EXISTS{ + BGP + } + +``` + +✓
+ +``` +DISTINCT +``` + + +```turtle +SELECT DISTINCT ?value + WHERE {} +``` + +✓
+ +``` +Now +``` + + +```turtle +NOW() +``` + +✓
+ +``` +CAST +(xsd:integer, xsd:float, xsd:dateTime, xsd:string) +``` + + +```turtle +xsd:integer(?value) +``` + +✓
+ +``` +Additive Expression +``` + + +```turtle +?value1 + ?value2 +``` + +✓
+ +``` +UnaryNot +``` + + +```turtle +!(?value) +``` + +✓
+ +``` +Multiplicative Expression +(*, /) +``` + + +```turtle +?value1 * ?value2 +``` + +✓
+ +### Construct +TBD \ No newline at end of file