More positive naming. More documentation. Split into Filters and Modi…

…fiers.

dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java

@@ -36,14 +36,14 @@
 import org.dpppt.backend.sdk.ws.controller.GaenController;
 import org.dpppt.backend.sdk.ws.filter.ResponseWrapperFilter;
 import org.dpppt.backend.sdk.ws.insertmanager.InsertManager;
-import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.FakeKeysFilter;
-import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.IOSLegacyProblemRPLT144Filter;
-import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.InvalidRollingPeriodFilter;
-import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.KeysNotMatchingJWTFilter;
-import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.NoBase64Filter;
-import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.OldAndroid0RPFilter;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.NonFakeKeysFilter;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.ValidRollingPeriodFilter;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.KeysMatchingJWTFilter;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.Base64Filter;
 import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RollingStartNumberAfterDayAfterTomorrowFilter;
-import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RollingStartNumberBeforeRetentionDay;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RollingStartNumberInRetentionPeriodFilter;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier.IOSLegacyProblemRPLT144Modifier;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier.OldAndroid0RPModifier;
 import org.dpppt.backend.sdk.ws.interceptor.HeaderInjector;
 import org.dpppt.backend.sdk.ws.security.KeyVault;
 import org.dpppt.backend.sdk.ws.security.NoValidateRequest;
@@ -195,32 +195,33 @@ public ProtoSignature gaenSigner() {
 	@Bean
 	public InsertManager insertManager() {
 		var manager = new InsertManager(gaenDataService(), gaenValidationUtils());
-		manager.addFilter(new NoBase64Filter(gaenValidationUtils()));
-		manager.addFilter(new KeysNotMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils()));
+		manager.addFilter(new Base64Filter(gaenValidationUtils()));
+		manager.addFilter(new KeysMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils()));
 		manager.addFilter(new RollingStartNumberAfterDayAfterTomorrowFilter());
-		manager.addFilter(new RollingStartNumberBeforeRetentionDay(gaenValidationUtils()));
-		manager.addFilter(new FakeKeysFilter());
-		manager.addFilter(new InvalidRollingPeriodFilter());
+		manager.addFilter(new RollingStartNumberInRetentionPeriodFilter(gaenValidationUtils()));
+		manager.addFilter(new NonFakeKeysFilter());
+		manager.addFilter(new ValidRollingPeriodFilter());
 		return manager;
 	}
+
 	@ConditionalOnProperty(
-    value="ws.app.gaen.insertfilter.android0rpfilter", 
+    value="ws.app.gaen.insertmanager.android0rpmodifier", 
     havingValue = "true", 
 	matchIfMissing = false)
-	@Bean public OldAndroid0RPFilter oldAndroid0RPFilter(InsertManager manager){
-		var androidFilter = new OldAndroid0RPFilter();
-		manager.addFilter(androidFilter);
-		return androidFilter;
+	@Bean public OldAndroid0RPModifier oldAndroid0RPModifier(InsertManager manager){
+		var androidModifier = new OldAndroid0RPModifier();
+		manager.addModifier(androidModifier);
+		return androidModifier;
 	}
 
 	@ConditionalOnProperty(
-    value="ws.app.gaen.insertfilter.iosrplt144filter", 
+    value="ws.app.gaen.insertmanager.iosrplt144modifier", 
     havingValue = "true", 
 	matchIfMissing = false)
-	@Bean public IOSLegacyProblemRPLT144Filter iosLegacyProblemRPLT144(InsertManager manager){
-		var iosFilter = new IOSLegacyProblemRPLT144Filter();
-		manager.addFilter(iosFilter);
-		return iosFilter;
+	@Bean public IOSLegacyProblemRPLT144Modifier iosLegacyProblemRPLT144(InsertManager manager){
+		var iosModifier = new IOSLegacyProblemRPLT144Modifier();
+		manager.addModifier(iosModifier);
+		return iosModifier;
 	}
 
 

dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java

@@ -35,7 +35,7 @@
 import org.dpppt.backend.sdk.utils.UTCInstant;
 import org.dpppt.backend.sdk.ws.insertmanager.InsertException;
 import org.dpppt.backend.sdk.ws.insertmanager.InsertManager;
-import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.NoBase64Filter.KeyIsNotBase64Exception;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.Base64Filter.KeyIsNotBase64Exception;
 import org.dpppt.backend.sdk.ws.security.ValidateRequest;
 import org.dpppt.backend.sdk.ws.security.ValidateRequest.ClaimIsBeforeOnsetException;
 import org.dpppt.backend.sdk.ws.security.ValidateRequest.InvalidDateException;

dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManager.java

@@ -7,24 +7,28 @@
 import org.dpppt.backend.sdk.model.gaen.GaenKey;
 import org.dpppt.backend.sdk.semver.Version;
 import org.dpppt.backend.sdk.utils.UTCInstant;
-import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.InsertionFilter;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.KeyInsertionFilter;
+import org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier.KeyInsertionModifier;
 import org.dpppt.backend.sdk.ws.util.ValidationUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * The insertion manager is responsible for inserting keys uploaded by clients
  * into the database. To make sure we only have valid keys in the database, a
- * list of {@Link InsertionFilter} is applied to the given list of keys. The
- * remaining keys are then inserted into the database. If any of the filters
- * throws an {@Link InsertException} the process of insertions is aborted and
- * the exception is propagated back to the caller, which is responsible for
- * handling the exception.
+ * list of {@link KeyInsertionModifier} is applied, and then a list of
+ * {@Link KeyInsertionFilter} is applied to the given list of keys. The
+ * remaining keys are then inserted into the database. If any of the modifiers
+ * filters throws an {@Link InsertException} the process of insertions is
+ * aborted and the exception is propagated back to the caller, which is
+ * responsible for handling the exception.
  *
  */
 public class InsertManager {
 
-	private final List<InsertionFilter> filterList = new ArrayList<>();
+	private final List<KeyInsertionFilter> filterList = new ArrayList<>();
+	private final List<KeyInsertionModifier> modifierList = new ArrayList<>();
+
 	private final GAENDataService dataService;
 	private final ValidationUtils validationUtils;
 
@@ -35,13 +39,17 @@ public InsertManager(GAENDataService dataService, ValidationUtils validationUtil
 		this.validationUtils = validationUtils;
 	}
 
-	public void addFilter(InsertionFilter filter) {
-		filterList.add(filter);
+	public void addFilter(KeyInsertionFilter filter) {
+		this.filterList.add(filter);
+	}
+
+	public void addModifier(KeyInsertionModifier modifier) {
+		this.modifierList.add(modifier);
 	}
 
 	/**
 	 * Inserts the keys into the database. The additional parameters are supplied to
-	 * the configured filters.
+	 * the configured modifiers and filters.
 	 * 
 	 * @param keys      the list of keys from the client
 	 * @param header    request header from client
@@ -71,7 +79,11 @@ public void insertIntoDatabase(List<GaenKey> keys, String header, Object princip
 		var osVersion = extractOsVersion(headerParts[4]);
 		var appVersion = extractAppVersion(headerParts[1], headerParts[2]);
 
-		for (InsertionFilter filter : filterList) {
+		for (KeyInsertionModifier modifier : modifierList) {
+			internalKeys = modifier.modify(now, internalKeys, osType, osVersion, appVersion, principal);
+		}
+
+		for (KeyInsertionFilter filter : filterList) {
 			internalKeys = filter.filter(now, internalKeys, osType, osVersion, appVersion, principal);
 		}
 

dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md

@@ -1,37 +1,43 @@
 # Insert-Manager
 
 ## Idea
-The Insert-Manager was introduced to reduce logic in controllers. The idea is to provide a second abstraction layer next to the `DataServices` to provide for possible generic validation and normalization. The Insert-Manager holds a list of `InsertionFilter`, which provide some code, to either filter for invalid data or alter incoming data. Each filter can decide to either skip respective keys, or throw a `InsertException`. Throwing an exception aborts the current insert request, and throws to the controller. Inside the controller the exception can be mapped to a respective error message and http status code.
+The Insert-Manager was introduced to reduce logic in controllers. The idea is to provide a second abstraction layer next to the `DataServices` to provide for possible generic validation and normalization. For this there are two mechanisems: modifiers and filters
 
-The current default only handles `KeyIsNotBase64Exception` and ignores all other exceptions (since there are none).
+The Insert-Manager holds a list of `KeyInsertionFilter`, which provide some code, to either filter for invalid data data. Each filter can decide to either skip respective keys, or throw a `InsertException`. Throwing an exception aborts the current insert request, and throws to the controller. Inside the controller the exception can be mapped to a respective error message and http status code.
 
-During construction, instances of `GAENDataService` and `ValidationUtils` are needed. Further, any filter can be added to the list with `addFilter(InsertionFilter filter)`. Ideally, this happens inside the [`WSBaseConfig`](../config/WSBaseConfig.java), where default filters are added right after constructing the `InsertManager`. To allow for conditional `InsertionFilters` refer to the following snippet:
+Additionally the Insert-Manager can be configured to hold a list of `KeyInsertModifier`. Modifiers are can be used to modify incoming keys before inserting into the database. (for example to fix buggy clients)
 
-```java
-@ConditionalOnProperty(
-value="ws.app.gaen.ioslegacy", 
-havingValue = "true", 
-matchIfMissing = true)
-@Bean public IOSLegacyProblemRPLT144 iosLegacyProblemRPLT144(InsertManager manager){
-    var iosFilter = new IOSLegacyProblemRPLT144();
-    manager.addFilter(iosFilter);
-    return iosFilter;
-}
-```
+Encapsulating the logic into smaller pieces of code, should allow for easier and better reviews of the respective filters and modifiers. Further, for each filter or modifier an extensive documentation can be provided, without cluttering the code with too many comments. 
 
-This looks for a property, either supplied via a `application.properties` file, or via `java` arguments (e.g. `java -D w.app.gaen.ioslegacy`) and constructs and inserts the respective filter bean into the filter chain. For further `SpringBoot` `Conditional` annotations have a look at ["Spring Boot Conditionals"](https://reflectoring.io/spring-boot-conditionals/)
+## Valid Keys
+A valid key is defined as follows:
+- Base64 Encoded key
+- Non Fake
+- Rolling Period in [1..144]
+- Rolling start number inside the configured retention period
+- Rolling start number not too far in the future, more precisely not after the day after tomorrow at time of insertion
+- Key date must honor the onset date which is given by the healt authority
 
-Encapsulating the logic into smaller pieces of code, should allow for easier and better reviews of the respective filters. Further, for each filter an extensive documentation can be provided, without cluttering the code with too many comments. 
 
-## InsertionFilter Interface
-The `InsertionFilter` interface has the following signature:
+## KeyInsertionFilter Interface
+The `KeyInsertionFilter` interface has the following signature:
 
 ```java
 public List<GaenKey> filter(UTCInstant now, List<GaenKey> content, OSType osType, Version osVersion, Version appVersion, Object principal) throws InsertException;
 ```
 
 It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process.
 
+## KeyInsertionModifier Interface
+The `KeyInsertionModifier` interface has the following signature:
+
+```java
+public List<GaenKey> modify(UTCInstant now, List<GaenKey> content, OSType osType, Version osVersion, Version appVersion, Object principal) throws InsertException;
+```
+
+It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process.
+
+
 ## InsertException
 
 An `InsertException` can be thrown inside an implementation of the `InsertionFilter` interface to mark an insert as "unrecoverable" and abort it. Such "unrecoverable" states might be patterns in the uploaded model, which allows for packet sniffing and/or information gathering.
@@ -44,32 +50,54 @@ Looking at the `WSBaseConfig`, we can see that during construction of the `Inser
 @Bean
 public InsertManager insertManager() {
     var manager = new InsertManager(gaenDataService(), gaenValidationUtils());
-    manager.addFilter(new NoBase64Filter(gaenValidationUtils()));
-    manager.addFilter(new KeysNotMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils()));
-    manager.addFilter(new RollingStartNumberAfterDayAfterTomorrow());
-    manager.addFilter(new RollingStartNumberBeforeRetentionDay(gaenValidationUtils()));
-    manager.addFilter(new FakeKeysFilter());
-    manager.addFilter(new InvalidRollingPeriodFilter());
+    manager.addFilter(new Base64Filter(gaenValidationUtils()));
+    manager.addFilter(new KeysMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils()));
+    manager.addFilter(new RollingStartNumberAfterDayAfterTomorrowFilter());
+    manager.addFilter(new RollingStartNumberInRetentionPeriodFilter(gaenValidationUtils()));
+    manager.addFilter(new NonFakeKeysFilter());
+    manager.addFilter(new ValidRollingPeriodFilter());
     return manager;
 }
 ```
 
-- `NoBase64Filter`
+- `Base64Filter`
     > This filter validates that the key actually is a correctly encoded base64 string. Since we are using 16 bytes of key data, those can be represented with exactly 24 characters. The validation of the length is already done during model validation and is assumed to be correct when reaching the filter. This filter _throws_ a `KeyIsNotBase64Exception` if any of the keys is wrongly encoded. Every key submitted _MUST_ have correct base64 encoding
-- `KeysNotMatchingJWTFilter`: 
+- `KeysMatchingJWTFilter`: 
     > This filter compares the supplied keys with information found in the JWT token. During the `exposed` request, the onset date, which will be set by the health authority and inserted as a claim into the JWT is the lower bound for allowed key dates. For the `exposednextday` the JWT contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual supplied key.
 - `RollingStartNumberAfterDayAfterTomorrowFilter`: 
     > Representing the maximum allowed time skew. Any key which is further in the future as the day after tomorrow is considered to be _maliciously_ or faulty (for example because of wrong date time settings) uploaded and is hence filtered out.
-- `RollingStartNumberBeforeRetentionDay`: 
-    > Any key which was valid earlier than `RetentionPeriod` is considered to be outdated and not saved in the database. The key would be removed during the next database clean anyways.
-- `FakeKeysFilter`
-    > Any key which has the `fake` flag is not inserted.
-- `InvalidRollingPeriodFilter`: 
+- `RollingStartNumberInRetentionPeriodFilter`: 
+    > Only keys with key date in the configured retention period are inserted into the datbase. Any key which was valid earlier than `RetentionPeriod` is considered to be outdated and not saved in the database. The key would be removed during the next database clean anyways.
+- `NonFakeKeysFilter`
+    > Only keys that are non-fake are inserted into the database, more precicely keys that have the fake flag set to `0`.
+- `ValidRollingPeriodFilter`: 
     > The `RollingPeriod` represents the 10 minutes interval of the key's validity. Negative numbers are not possible, hence any key having a negative rolling period is considered to be _maliciously_ uploaded. Further, according to [Apple/Googles documentation](https://github.com/google/exposure-notifications-server/blob/main/docs/server_functional_requirements.md) values must be in [1..144]
 
 
-## Additonal Filters
-- `IOSLegacyProblemRPLT144Filter`
-    > This filter makes sure, that rolling period is always set to 144. Default value according to EN is 144, so just set it to that. This allows to check for the Google-TEKs also on iOS. Because the Rolling Proximity Identifier is based on the TEK and the unix epoch, this should work. The only downside is that iOS will not be able to optimize verification of the TEKs, because it will have to consider each TEK for a whole day.
-- `OldAndroid0RPFilter`: 
-    > Some early builds of Google's Exposure Notification API returned TEKs with rolling period set to '0'. According to the specification, this is invalid and will cause both Android and iOS to drop/ignore the key. To mitigate ignoring TEKs from these builds alltogether, the rolling period is increased to '144' (one full day). This should not happen anymore and can be removed in the near future. Until then we are going to log whenever this happens to be able to monitor this problem.
+## Additonal Modifiers
+- `IOSLegacyProblemRPLT144FModifier`
+    > This modifier makes sure, that rolling period is always set to 144. Default value according to EN is 144, so just set it to that. This allows to check for the Google-TEKs also on iOS. Because the Rolling Proximity Identifier is based on the TEK and the unix epoch, this should work. The only downside is that iOS will not be able to optimize verification of the TEKs, because it will have to consider each TEK for a whole day.
+- `OldAndroid0RPModifier`: 
+    > Some early builds of Google's Exposure Notification API returned TEKs with rolling period set to '0'. According to the specification, this is invalid and will cause both Android and iOS to drop/ignore the key. To mitigate ignoring TEKs from these builds alltogether, the rolling period is increased to '144' (one full day). This should not happen anymore and can be removed in the near future. Until then we are going to log whenever this happens to be able to monitor this problem.
+
+
+
+## Configuration 
+During construction, instances of `GAENDataService` and `ValidationUtils` are needed. Further, any filter or modifier can be added to the list with `addFilter(KeyInsertionFilter filter)` or `addModifier(KeyInsertionModifier)`. Ideally, this happens inside the [`WSBaseConfig`](../config/WSBaseConfig.java), where default filters are added right after constructing the `InsertManager`. 
+
+To allow for conditional `KeyInsertionFilters` or `KeyInsertionModifiers` refer to the following snippet:
+
+```java
+@ConditionalOnProperty(
+value="ws.app.gaen.insertmanager.iosrplt144modifier", 
+havingValue = "true", 
+matchIfMissing = false)
+@Bean public IOSLegacyProblemRPLT144Modifier iosLegacyProblemRPLT144(InsertManager manager){
+    var iosModifier = new IOSLegacyProblemRPLT144Modifier();
+    manager.addModifier(iosModifier);
+    return iosModifier;
+}
+```
+
+This looks for a property, either supplied via a `application.properties` file, or via `java` arguments (e.g. `java -D ws.app.gaen.insertmanager.iosrplt144modifier`) and constructs and inserts the respective modifier bean into the modifier chain. For further `SpringBoot` `Conditional` annotations have a look at ["Spring Boot Conditionals"](https://reflectoring.io/spring-boot-conditionals/)
+

dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/NoBase64Filter.java → dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/Base64Filter.java

@@ -15,11 +15,11 @@
  * if any of the keys is not valid Base64, as this is a client error.
  *
  */
-public class NoBase64Filter implements InsertionFilter {
+public class Base64Filter implements KeyInsertionFilter {
 
 	private final ValidationUtils validationUtils;
 
-	public NoBase64Filter(ValidationUtils validationUtils) {
+	public Base64Filter(ValidationUtils validationUtils) {
 		this.validationUtils = validationUtils;
 	}