Skip to content

Commit

Permalink
gh-2008 View validation experiment (#2376)
Browse files Browse the repository at this point in the history
* Added Test

* gh-2008 - Don't run view validation against FederatedStore - proxy - FederatedStore

* Updated import

* gh-2008 Added more tests

* Improved efficiency
  • Loading branch information
d47853 authored Jan 14, 2021
1 parent 3fba005 commit 95cd5b9
Show file tree
Hide file tree
Showing 4 changed files with 217 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,12 @@ public enum StoreTrait {
* {@link uk.gov.gchq.gaffer.data.element.Edge}s
* for {@link uk.gov.gchq.gaffer.operation.impl.get.GetElements} operations.
*/
MATCHED_VERTEX;
MATCHED_VERTEX,

/**
* Stores with this trait have schemas that may change and should do less view validation
*/
DYNAMIC_SCHEMA;

public static final Set<StoreTrait> ALL_TRAITS = Collections.unmodifiableSet(Sets.newHashSet(StoreTrait.values()));
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.gchq.gaffer.federatedstore.operation;

import uk.gov.gchq.gaffer.federatedstore.FederatedStore;
import uk.gov.gchq.gaffer.federatedstore.FederatedStoreConstants;
import uk.gov.gchq.gaffer.graph.Graph;
import uk.gov.gchq.gaffer.operation.Operation;
import uk.gov.gchq.gaffer.store.Store;
import uk.gov.gchq.gaffer.store.StoreTrait;
import uk.gov.gchq.gaffer.store.operation.OperationChainValidator;
import uk.gov.gchq.gaffer.store.schema.Schema;
import uk.gov.gchq.gaffer.store.schema.ViewValidator;
Expand All @@ -34,7 +35,6 @@

import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;

/**
* Validation class for validating {@link uk.gov.gchq.gaffer.operation.OperationChain}s against {@link ViewValidator}s using the Federated Store schemas.
* Extends {@link OperationChainValidator} and uses the {@link FederatedStore} to get
Expand All @@ -52,7 +52,7 @@ protected Schema getSchema(final Operation operation, final User user, final Sto

@Override
protected void validateViews(final Operation op, final User user, final Store store, final ValidationResult validationResult) {
validateAllGraphsIdViews(op, user, store, validationResult, getGraphIds(op, user, (FederatedStore) store));
validateAllGraphsIdViews(op, user, store, validationResult, getGraphIdsCSV(op, user, (FederatedStore) store));
}

/**
Expand All @@ -63,23 +63,26 @@ protected void validateViews(final Operation op, final User user, final Store st
* @param user The requesting user
* @param store The current store
* @param validationResult The result of validation
* @param graphIds The graphs to test the view against
* @param graphIdsCSV The graphs to test the view against
*/
private void validateAllGraphsIdViews(final Operation op, final User user, final Store store, final ValidationResult validationResult, final Collection<String> graphIds) {
private void validateAllGraphsIdViews(final Operation op, final User user, final Store store, final ValidationResult validationResult, final String graphIdsCSV) {
ValidationResult savedResult = new ValidationResult();
ValidationResult currentResult = null;

final Operation clonedOp = shallowCloneWithDeepOptions(op);

for (final String graphId : graphIds) {
Collection<Graph> graphs = ((FederatedStore) store).getGraphs(user, graphIdsCSV, clonedOp);
for (final Graph graph : graphs) {
String graphId = graph.getGraphId();
final boolean graphIdValid = ((FederatedStore) store).getAllGraphIds(user).contains(graphId);
//If graphId is not valid, then there is no schema to validate a view against.
// If graphId is not valid, then there is no schema to validate a view against.
if (graphIdValid) {
currentResult = new ValidationResult();
clonedOp.addOption(FederatedStoreConstants.KEY_OPERATION_OPTIONS_GRAPH_IDS, graphId);
super.validateViews(clonedOp, user, store, currentResult);
if (!graph.hasTrait(StoreTrait.DYNAMIC_SCHEMA)) {
super.validateViews(clonedOp, user, store, currentResult);
}
if (currentResult.isValid()) {
//If any graph has a valid View, break with valid current result
// If any graph has a valid View, break with valid current result
break;
} else {
ValidationResult prependGraphId = new ValidationResult();
Expand All @@ -91,15 +94,15 @@ private void validateAllGraphsIdViews(final Operation op, final User user, final

//What state did the for loop exit with?
if (currentResult != null && !currentResult.isValid()) {
validationResult.addError("View is not valid for graphIds:" + graphIds.stream().collect(Collectors.joining(",", "[", "]")));
validationResult.addError("View is not valid for graphIds:" + getGraphIds(op, user, (FederatedStore) store).stream().collect(Collectors.joining(",", "[", "]")));
//If invalid, no graphs views where valid, so add all saved errors.
validationResult.add(savedResult);
}
}

/**
* Return a clone of the given operations with a deep clone of options.
*
* <p>
* Because op.shallowClone() is used it can't be guaranteed that original options won't be modified.
* So a deep clone of the options is made for the shallow clone of the operation.
*
Expand All @@ -115,12 +118,17 @@ private Operation shallowCloneWithDeepOptions(final Operation op) {
}

private Collection<String> getGraphIds(final Operation op, final User user, final FederatedStore store) {
return nonNull(op) && nonNull(getGraphIds(op)) && !getGraphIds(op).isEmpty()
? Arrays.asList(getGraphIds(op).split(","))
: store.getAllGraphIds(user);
return Arrays.asList(getGraphIdsCSV(op, user, store).split(","));
}

private String getGraphIdsCSV(final Operation op, final User user, final FederatedStore store) {
String graphIds = getGraphIds(op);
return nonNull(graphIds) && !graphIds.isEmpty()
? graphIds
: String.join(",", store.getAllGraphIds(user));
}

private String getGraphIds(final Operation op) {
return op.getOption(FederatedStoreConstants.KEY_OPERATION_OPTIONS_GRAPH_IDS);
return op == null ? null : op.getOption(FederatedStoreConstants.KEY_OPERATION_OPTIONS_GRAPH_IDS);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import uk.gov.gchq.gaffer.operation.Operations;
import uk.gov.gchq.gaffer.operation.graph.OperationView;
import uk.gov.gchq.gaffer.operation.impl.add.AddElements;
import uk.gov.gchq.gaffer.store.StoreTrait;
import uk.gov.gchq.gaffer.store.schema.Schema;

import java.util.ArrayList;
Expand Down Expand Up @@ -126,7 +127,7 @@ public static <OP extends Operation> OP updateOperationForGraph(final OP operati
resultOp = (OP) operation.shallowClone();
if (validView.hasGroups()) {
((OperationView) resultOp).setView(validView);
} else {
} else if (!graph.hasTrait(StoreTrait.DYNAMIC_SCHEMA)) {
// The view has no groups so the operation would return
// nothing, so we shouldn't execute the operation.
resultOp = null;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
/*
* Copyright 2020 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.gchq.gaffer.federatedstore;

import com.google.common.collect.Lists;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import uk.gov.gchq.gaffer.data.element.Edge;
import uk.gov.gchq.gaffer.data.element.Element;
import uk.gov.gchq.gaffer.data.element.Entity;
import uk.gov.gchq.gaffer.data.elementdefinition.view.View;
import uk.gov.gchq.gaffer.federatedstore.operation.AddGraph;
import uk.gov.gchq.gaffer.graph.Graph;
import uk.gov.gchq.gaffer.graph.GraphConfig;
import uk.gov.gchq.gaffer.mapstore.MapStoreProperties;
import uk.gov.gchq.gaffer.operation.OperationException;
import uk.gov.gchq.gaffer.operation.impl.add.AddElements;
import uk.gov.gchq.gaffer.operation.impl.get.GetAllElements;
import uk.gov.gchq.gaffer.proxystore.ProxyProperties;
import uk.gov.gchq.gaffer.store.schema.Schema;
import uk.gov.gchq.gaffer.user.User;

import java.util.List;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static uk.gov.gchq.gaffer.federatedstore.integration.FederatedViewsIT.BASIC_EDGE;
import static uk.gov.gchq.gaffer.federatedstore.integration.FederatedViewsIT.BASIC_ENTITY;

/**
* The FederatedStoreToFederatedStore Test works as follows:
* --------------------
* FederatedStore | GAFFER REST API |
* -> Proxy Store --------------> | |
* | FederatedStore |
* | -> MapStore |
* --------------------
*/
public class FederatedStoreToFederatedStoreTest {

private Graph federatedStoreGraph;
private Graph restApiFederatedGraph;

@BeforeEach
public void setUpStores() throws OperationException {
ProxyProperties proxyProperties = new ProxyProperties();
proxyProperties.setStoreClass(SingleUseFederatedStore.class);

restApiFederatedGraph = new Graph.Builder()
.storeProperties(proxyProperties)
.config(new GraphConfig("RestApiGraph"))
.addSchema(new Schema())
.build();

federatedStoreGraph = new Graph.Builder()
.config(new GraphConfig("federatedStoreGraph"))
.storeProperties(new FederatedStoreProperties())
.build();

connectGraphs();
addMapStore();
}

private void addMapStore() throws OperationException {
restApiFederatedGraph.execute(new AddGraph.Builder()
.storeProperties(new MapStoreProperties())
.graphId("mapStore")
.schema(Schema.fromJson(getClass().getResourceAsStream("/schema/basicEntitySchema.json")))
.build(), new User());
}

private void connectGraphs() throws OperationException {
federatedStoreGraph.execute(new AddGraph.Builder()
.storeProperties(new ProxyProperties())
.graphId("RestProxy")
.schema(new Schema())
.build(), new User());
}

@Test
public void shouldErrorIfViewIsInvalid() throws OperationException {
// Given
Entity entity = new Entity.Builder()
.group(BASIC_ENTITY)
.vertex("myVertex")
.property("property1", 1)
.build();

restApiFederatedGraph.execute(new AddElements.Builder()
.input(entity)
.build(), new User());

// When
OperationException e = assertThrows(OperationException.class, () -> federatedStoreGraph.execute(new GetAllElements.Builder()
.view(new View.Builder()
.edge(BASIC_EDGE)
.build())
.build(), new User()));

assertTrue(e.getMessage().contains("View is not valid for graphIds:[mapStore]"));
}

@Test
public void shouldMaintainView() throws OperationException {
// Given
final String mapStoreGraphId = "mapStoreWithFullSchema";
restApiFederatedGraph.execute(new AddGraph.Builder()
.storeProperties(new MapStoreProperties())
.graphId(mapStoreGraphId)
.schema(Schema.fromJson(getClass().getResourceAsStream("/schema/basicEntitySchema.json"),
getClass().getResourceAsStream("/schema/basicEdgeSchema.json")))
.build(), new User());

Entity entity = new Entity.Builder()
.group(BASIC_ENTITY)
.vertex("myVertex")
.property("property1", 1)
.build();

Edge edge = new Edge.Builder()
.source("mySource")
.dest("myDest")
.group(BASIC_EDGE)
.property("columnQualifier", 2)
.property("prooperty1", 1)
.build();

restApiFederatedGraph.execute(new AddElements.Builder()
.input(entity, edge)
.option(FederatedStoreConstants.KEY_OPERATION_OPTIONS_GRAPH_IDS, mapStoreGraphId)
.build(), new User());

// When
List<? extends Element> results = Lists.newArrayList(federatedStoreGraph.execute(new GetAllElements.Builder()
.view(new View.Builder()
.entity(BASIC_ENTITY)
.build())
.build(), new User()));

// Then
assertEquals(1, results.size());

}

@Test
public void shouldBeAbleToSendViewedQueries() throws OperationException {
// Given
Entity entity = new Entity.Builder()
.group(BASIC_ENTITY)
.vertex("myVertex")
.property("property1", 1)
.build();

restApiFederatedGraph.execute(new AddElements.Builder()
.input(entity)
.build(), new User());

// When
List<? extends Element> results = Lists.newArrayList(federatedStoreGraph.execute(new GetAllElements.Builder()
.view(new View.Builder()
.entity(BASIC_ENTITY)
.build())
.build(), new User()));

// Then
assertEquals(1, results.size());
assertEquals(entity, results.get(0));
}
}

0 comments on commit 95cd5b9

Please sign in to comment.