Add JSON schema validator originally developed in OERSI (#443)

metafacture-json/build.gradle

@@ -22,6 +22,7 @@ dependencies {
   implementation 'com.fasterxml.jackson.core:jackson-core:2.13.0'
   implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.0'
   implementation 'com.jayway.jsonpath:json-path:2.6.0'
+  implementation 'com.github.erosb:everit-json-schema:1.14.1'
   testImplementation 'junit:junit:4.12'
   testImplementation 'org.mockito:mockito-core:2.5.5'
   testRuntimeOnly 'org.slf4j:slf4j-simple:1.7.21'

metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java

@@ -0,0 +1,176 @@
+/*
+ * Copyright 2021, 2022 Fabian Steeg, hbz
+ *
+ * Licensed under the Apache License, Version 2.0 the "License";
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.metafacture.json;
+
+import org.metafacture.framework.FluxCommand;
+import org.metafacture.framework.MetafactureException;
+import org.metafacture.framework.ObjectReceiver;
+import org.metafacture.framework.annotations.Description;
+import org.metafacture.framework.annotations.In;
+import org.metafacture.framework.annotations.Out;
+import org.metafacture.framework.helpers.DefaultObjectPipe;
+
+import org.everit.json.schema.Schema;
+import org.everit.json.schema.ValidationException;
+import org.everit.json.schema.loader.SchemaClient;
+import org.everit.json.schema.loader.SchemaLoader;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Validate JSON against a given schema, pass only valid input to the receiver.
+ *
+ * @author Fabian Steeg (fsteeg)
+ */
+@Description("Validate JSON against a given schema, send only valid input to the receiver. Pass the schema location to validate against. " +
+        "Set 'schemaRoot' for resolving sub-schemas referenced in '$id' or '$ref' (defaults to the classpath root: '/'). " +
+        "Write valid and/or invalid output to locations specified with 'writeValid' and 'writeInvalid'.")
+@In(String.class)
+@Out(String.class)
+@FluxCommand("validate-json")
+public final class JsonValidator extends DefaultObjectPipe<String, ObjectReceiver<String>> {
+
+    private static final Logger LOG = LoggerFactory.getLogger(JsonValidator.class);
+    private static final String DEFAULT_SCHEMA_ROOT = "/";
+    private String schemaUrl;
+    private Schema schema;
+    private long fail;
+    private long success;
+    private FileWriter writeInvalid;
+    private FileWriter writeValid;
+    private String schemaRoot = DEFAULT_SCHEMA_ROOT;
+
+    /**
+     * @param url The URL of the schema to validate against.
+     */
+    public JsonValidator(final String url) {
+        this.schemaUrl = url;
+    }
+
+    /**
+     * @param schemaRoot The root location for resolving sub-schemas referenced in '$id' or '$ref'.
+     */
+    public void setSchemaRoot(final String schemaRoot) {
+        this.schemaRoot = schemaRoot;
+    }
+
+    /**
+     * @param writeValid The location to write valid data to.
+     */
+    public void setWriteValid(final String writeValid) {
+        this.writeValid = fileWriter(writeValid);
+    }
+
+    /**
+     * @param writeInvalid The location to write invalid data to.
+     */
+    public void setWriteInvalid(final String writeInvalid) {
+        this.writeInvalid = fileWriter(writeInvalid);
+    }
+
+    @Override
+    public void process(final String json) {
+        JSONObject object = null;
+        try {
+            object = new JSONObject(json); // throws JSONException on syntax error
+        }
+        catch (final JSONException e) {
+            handleInvalid(json, null, e.getMessage());
+        }
+        try {
+            initSchema();
+            schema.validate(object); // throws ValidationException if invalid
+            getReceiver().process(json);
+            ++success;
+            write(json, writeValid);
+        }
+        catch (final ValidationException e) {
+            handleInvalid(json, object, e.getAllMessages().toString());
+        }
+    }
+
+    @Override
+    protected void onCloseStream() {
+        close(writeInvalid);
+        close(writeValid);
+        LOG.debug("Success: {}, Fail: {}", success, fail);
+        super.onCloseStream();
+    }
+
+    private void initSchema() {
+        if (schema != null) {
+            return;
+        }
+        try (InputStream inputStream = getClass().getResourceAsStream(schemaUrl)) {
+            schema = SchemaLoader.builder()
+                    .schemaJson(new JSONObject(new JSONTokener(inputStream)))
+                    .schemaClient(SchemaClient.classPathAwareClient())
+                    .resolutionScope("classpath://" + schemaRoot)
+                    .build().load().build();
+        }
+        catch (final IOException | JSONException e) {
+            throw new MetafactureException(e.getMessage(), e);
+        }
+    }
+
+    private FileWriter fileWriter(final String fileLocation) {
+        try {
+            return new FileWriter(fileLocation);
+        }
+        catch (final IOException e) {
+            throw new MetafactureException(e.getMessage(), e);
+        }
+    }
+
+    private void handleInvalid(final String json, final JSONObject object,
+            final String errorMessage) {
+        LOG.info("Invalid JSON: {} in {}", errorMessage, object != null ? object.opt("id") : json);
+        ++fail;
+        write(json, writeInvalid);
+    }
+
+    private void write(final String json, final FileWriter fileWriter) {
+        if (fileWriter != null) {
+            try {
+                fileWriter.append(json);
+                fileWriter.append("\n");
+            }
+            catch (final IOException e) {
+                throw new MetafactureException(e.getMessage(), e);
+            }
+        }
+    }
+
+    private void close(final FileWriter fileWriter) {
+        if (fileWriter != null) {
+            try {
+                fileWriter.close();
+            }
+            catch (final IOException e) {
+                throw new MetafactureException(e.getMessage(), e);
+            }
+        }
+    }
+
+}

metafacture-json/src/main/resources/flux-commands.properties

@@ -15,3 +15,4 @@
 #
 encode-json org.metafacture.json.JsonEncoder
 decode-json org.metafacture.json.JsonDecoder
+validate-json org.metafacture.json.JsonValidator

metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2021, 2022 Fabian Steeg, hbz
+ *
+ * Licensed under the Apache License, Version 2.0 the "License";
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.metafacture.json;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.metafacture.framework.MetafactureException;
+import org.metafacture.framework.ObjectReceiver;
+import org.mockito.InOrder;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+/**
+ * Tests for {@link JsonValidator}.
+ *
+ * @author Fabian Steeg
+ *
+ */
+public final class JsonValidatorTest {
+
+    private static final String SCHEMA = "/schemas/schema.json";
+    private static final String JSON_VALID = "{\"id\":\"http://example.org/\"}";
+    private static final String JSON_INVALID_MISSING_REQUIRED = "{}";
+    private static final String JSON_INVALID_URI_FORMAT= "{\"id\":\"example.org/\"}";
+    private static final String JSON_INVALID_DUPLICATE_KEY = "{\"id\":\"val\",\"id\":\"val\"}";
+    private static final String JSON_INVALID_SYNTAX_ERROR = "{\"id1\":\"val\",\"id2\":\"val\"";
+
+    private JsonValidator validator;
+
+    @Mock
+    private ObjectReceiver<String> receiver;
+    private InOrder inOrder;
+
+    @Before
+    public void setup() {
+        MockitoAnnotations.initMocks(this);
+        validator = new JsonValidator(SCHEMA);
+        validator.setSchemaRoot("/schemas/");
+        validator.setReceiver(receiver);
+        inOrder = Mockito.inOrder(receiver);
+    }
+
+    @Test
+    public void testShouldValidate() {
+        validator.process(JSON_VALID);
+        inOrder.verify(receiver, Mockito.calls(1)).process(JSON_VALID);
+    }
+
+    @Test
+    public void testShouldInvalidateMissingRequired() {
+        validator.process(JSON_INVALID_MISSING_REQUIRED);
+        inOrder.verifyNoMoreInteractions();
+    }
+
+    @Test
+    public void testShouldInvalidateUriFormat() {
+        validator.process(JSON_INVALID_URI_FORMAT);
+        inOrder.verifyNoMoreInteractions();
+    }
+
+    @Test
+    public void testShouldInvalidateDuplicateKey() {
+        validator.process(JSON_INVALID_DUPLICATE_KEY);
+        inOrder.verifyNoMoreInteractions();
+    }
+
+    @Test
+    public void testShouldInvalidateSyntaxError() {
+        validator.process(JSON_INVALID_SYNTAX_ERROR);
+        inOrder.verifyNoMoreInteractions();
+    }
+
+    @Test(expected = MetafactureException.class)
+    public void testShouldCatchMissingSchemaFile() {
+        new JsonValidator("").process("");
+    }
+
+    @Test(expected = MetafactureException.class)
+    public void testShouldCatchMissingValidOutputFile() {
+        validator.setWriteValid("");
+        validator.process(JSON_INVALID_MISSING_REQUIRED);
+    }
+
+    @Test(expected = MetafactureException.class)
+    public void testShouldCatchMissingInvalidOutputFile() {
+        validator.setWriteInvalid("");
+        validator.process(JSON_INVALID_MISSING_REQUIRED);
+    }
+
+    @After
+    public void cleanup() {
+        validator.closeStream();
+    }
+
+}

metafacture-json/src/test/resources/schemas/id.json

@@ -0,0 +1,8 @@
+{
+  "$id": "id.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "URL",
+  "description": "The URL/URI of the resource",
+  "type": "string",
+  "format": "uri"
+}

metafacture-json/src/test/resources/schemas/schema.json

@@ -0,0 +1,13 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "schema.json",
+  "type": "object",
+  "properties": {
+    "id": {
+      "$ref": "id.json"
+    }
+  },
+  "required": [
+    "id"
+  ]
+}