Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#2072: add configuration option to configure pre-defined "extraFields" #2076

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

#2072: add configuration option to configure pre-defined "extraFields" #2076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0412a6b
#2072: add configuration option to configure pre-defined "extraFields"
thjaeckle Dec 16, 2024
c5cb75a
#2072: add unit test for caching signal enrichment with pre-defined e…
thjaeckle Jan 2, 2025
a8f4450
#2072: fixed license year of new files
thjaeckle Jan 2, 2025
46a7df3
#2072: provide another testcase for cache optimization only requestin…
thjaeckle Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,41 @@ public enum DittoHeaderDefinition implements HeaderDefinition {
JsonObject.class,
false,
true,
HeaderValueValidators.getJsonObjectValidator()),

/**
* Internal header containing the pre-defined configured {@code extraFields} as list of jsonPointers for the
* emitted thing event.
*
* @since 3.7.0
*/
PRE_DEFINED_EXTRA_FIELDS("ditto-pre-defined-extra-fields",
JsonArray.class,
false,
false,
HeaderValueValidators.getJsonArrayValidator()),

/**
* Internal header containing the pre-defined configured {@code extraFields} as keys and the allowed "read subjects"
* as array of stings - defining which "auth subjects" are allowed to read which pre-defined extra field.
*
* @since 3.7.0
*/
PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT("ditto-pre-defined-extra-fields-read-grant",
JsonObject.class,
false,
false,
HeaderValueValidators.getJsonObjectValidator()),

/**
* Internal header containing pre-defined {@code extraFields} as JSON object sent along for emitted thing event.
*
* @since 3.7.0
*/
PRE_DEFINED_EXTRA_FIELDS_OBJECT("ditto-pre-defined-extra-fields-object",
JsonObject.class,
false,
false,
HeaderValueValidators.getJsonObjectValidator());

/**
Expand Down
27 changes: 27 additions & 0 deletions base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,16 @@ public final class ImmutableDittoHeadersTest {
.set(DittoHeaderDefinition.ORIGINATOR.getKey(), "foo:bar")
.build();

private static final JsonArray KNOWN_PRE_DEFINED_EXTRA_FIELDS = JsonArray.newBuilder()
.add("foo:bar:123")
.build();
private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT = JsonObject.newBuilder()
.set("/definition", "known:subject")
.build();
private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT = JsonObject.newBuilder()
.set("definition", "foo:bar:123")
.build();


static {
KNOWN_METADATA_HEADERS = MetadataHeaders.newInstance();
Expand Down Expand Up @@ -205,6 +215,12 @@ public void settingAllKnownHeadersWorksAsExpected() {
.putHeader(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION))
.putHeader(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP))
.putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString())
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString())
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString())
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString())
.build();

assertThat(underTest).isEqualTo(expectedHeaderMap);
Expand Down Expand Up @@ -535,6 +551,11 @@ public void toJsonReturnsExpected() {
.set(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), KNOWN_AT_HISTORICAL_REVISION)
.set(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), KNOWN_AT_HISTORICAL_TIMESTAMP.toString())
.set(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS)
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(), KNOWN_PRE_DEFINED_EXTRA_FIELDS)
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT)
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT)
.build();

final Map<String, String> allKnownHeaders = createMapContainingAllKnownHeaders();
Expand Down Expand Up @@ -774,6 +795,12 @@ private static Map<String, String> createMapContainingAllKnownHeaders() {
result.put(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION));
result.put(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP));
result.put(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString());
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString());
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString());
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString());

return result;
}
Expand Down
94 changes: 92 additions & 2 deletions ...rg/eclipse/ditto/internal/models/signalenrichment/DittoCachingSignalEnrichmentFacade.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,17 @@

import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executor;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import javax.annotation.Nullable;

Expand All @@ -38,8 +41,10 @@
import org.eclipse.ditto.internal.utils.pekko.logging.ThreadSafeDittoLogger;
import org.eclipse.ditto.internal.utils.tracing.DittoTracing;
import org.eclipse.ditto.internal.utils.tracing.span.SpanOperationName;
import org.eclipse.ditto.json.JsonArray;
import org.eclipse.ditto.json.JsonFactory;
import org.eclipse.ditto.json.JsonFieldSelector;
import org.eclipse.ditto.json.JsonKey;
import org.eclipse.ditto.json.JsonObject;
import org.eclipse.ditto.json.JsonObjectBuilder;
import org.eclipse.ditto.json.JsonPointer;
Expand Down Expand Up @@ -160,6 +165,16 @@ public CompletionStage<JsonObject> retrievePartialThing(final ThingId thingId,
(concernedSignal instanceof ThingEvent) && !(ProtocolAdapter.isLiveSignal(concernedSignal)) ?
List.of((ThingEvent<?>) concernedSignal) : List.of();

final DittoHeaders signalHeaders = Optional.ofNullable(concernedSignal)
.map(Signal::getDittoHeaders)
.orElseGet(DittoHeaders::empty);
if (jsonFieldSelector != null &&
signalHeaders.containsKey(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey())
) {
return performPreDefinedExtraFieldsOptimization(
thingId, jsonFieldSelector, dittoHeaders, signalHeaders, thingEvents
);
}
// as second step only return what was originally requested as fields:
final var cachingParameters =
new CachingParameters(jsonFieldSelector, thingEvents, true, 0);
Expand Down Expand Up @@ -199,6 +214,81 @@ public CompletionStage<JsonObject> retrievePartialThing(final EntityId thingId,
.thenApply(jsonObject -> applyJsonFieldSelector(jsonObject, jsonFieldSelector));
}

private CompletionStage<JsonObject> performPreDefinedExtraFieldsOptimization(final ThingId thingId,
final JsonFieldSelector jsonFieldSelector,
final DittoHeaders dittoHeaders,
final DittoHeaders signalHeaders,
final List<ThingEvent<?>> thingEvents
) {
final JsonArray configuredPredefinedExtraFields =
JsonArray.of(signalHeaders.get(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey()));
final Set<JsonPointer> allConfiguredPredefinedExtraFields = configuredPredefinedExtraFields.stream()
.filter(JsonValue::isString)
.map(JsonValue::asString)
.map(JsonPointer::of)
.collect(Collectors.toSet());

final JsonObject preDefinedExtraFields =
JsonObject.of(signalHeaders.get(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey()));
final CompletionStage<JsonObject> filteredPreDefinedExtraFieldsReadGranted =
filterPreDefinedExtraReadGrantedObject(jsonFieldSelector, dittoHeaders, signalHeaders,
preDefinedExtraFields);

final boolean allExtraFieldsPresent =
allConfiguredPredefinedExtraFields.containsAll(jsonFieldSelector.getPointers());
if (allExtraFieldsPresent) {
return filteredPreDefinedExtraFieldsReadGranted;
} else {
// optimization to only fetch extra fields which were not pre-defined
final List<JsonPointer> missingFieldsPointers = new ArrayList<>(jsonFieldSelector.getPointers());
missingFieldsPointers.removeAll(allConfiguredPredefinedExtraFields);
final JsonFieldSelector missingFieldsSelector = JsonFactory.newFieldSelector(missingFieldsPointers);
final var cachingParameters =
new CachingParameters(missingFieldsSelector, thingEvents, true, 0);

return doRetrievePartialThing(thingId, dittoHeaders, null, cachingParameters)
.thenCompose(jsonObject -> filteredPreDefinedExtraFieldsReadGranted
.thenApply(preDefinedObject ->
JsonFactory.newObject( // merge
applyJsonFieldSelector(jsonObject, missingFieldsSelector),
preDefinedObject
)
)
);
}
}

private static CompletionStage<JsonObject> filterPreDefinedExtraReadGrantedObject(
final JsonFieldSelector jsonFieldSelector,
final DittoHeaders dittoHeaders, final DittoHeaders signalHeaders, final JsonObject preDefinedExtraFields) {
final JsonObject preDefinedExtraFieldsReadGrant = JsonObject.of(
signalHeaders.get(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey())
);
final JsonFieldSelector grantedReadJsonFieldSelector = filterAskedForFieldSelectorToGrantedFields(
jsonFieldSelector, preDefinedExtraFieldsReadGrant,
dittoHeaders.getAuthorizationContext().getAuthorizationSubjectIds()
);
return CompletableFuture.completedStage(preDefinedExtraFields.get(grantedReadJsonFieldSelector));
}

private static JsonFieldSelector filterAskedForFieldSelectorToGrantedFields(
final JsonFieldSelector jsonFieldSelector,
final JsonObject preDefinedExtraFieldsReadGrant,
final List<String> authorizationSubjectIds)
{
final List<JsonPointer> allowedPointers = StreamSupport.stream(jsonFieldSelector.spliterator(), false)
.filter(pointer -> preDefinedExtraFieldsReadGrant.getValue(JsonKey.of(pointer.toString()))
.filter(JsonValue::isArray)
.map(JsonValue::asArray)
.filter(readGrantArray -> readGrantArray.stream()
.filter(JsonValue::isString)
.map(JsonValue::asString)
.anyMatch(authorizationSubjectIds::contains)
).isPresent()
).toList();
return JsonFactory.newFieldSelector(allowedPointers);
}

protected CompletionStage<JsonObject> doRetrievePartialThing(final EntityId thingId,
final DittoHeaders dittoHeaders,
@Nullable final DittoHeaders dittoHeadersNotAddedToCacheKey,
Expand Down Expand Up @@ -369,11 +459,11 @@ private CompletionStage<JsonObject> doSmartUpdateCachedObject(final SignalEnrich
}

private static <T> T getLast(final List<T> list) {
return list.get(list.size() - 1);
return list.getLast();
}

private static <T> T getFirst(final List<T> list) {
return list.get(0);
return list.getFirst();
}

private CompletionStage<JsonObject> handleNextExpectedThingEvents(final SignalEnrichmentCacheKey cacheKey,
Expand Down
2 changes: 1 addition & 1 deletion ...pse/ditto/internal/models/signalenrichment/AbstractCachingSignalEnrichmentFacadeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
*/
abstract class AbstractCachingSignalEnrichmentFacadeTest extends AbstractSignalEnrichmentFacadeTest {

private static final String ISSUER_PREFIX = "test:";
protected static final String ISSUER_PREFIX = "test:";
private static final String CACHE_CONFIG_KEY = "my-cache";
private static final String CACHE_CONFIG = CACHE_CONFIG_KEY + """
{
Expand Down
Loading
Loading