From 8387381df67fa6b619b405ed3112a4c8484edf4b Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Fri, 28 Aug 2020 12:15:35 +0200 Subject: [PATCH 01/10] Adding semver Adds the semver class with its tests. See #226 Once this is merged, #213 and #215 need to be rebased on this. --- .../org/dpppt/backend/sdk/semver/README.md | 25 ++ .../org/dpppt/backend/sdk/semver/Version.java | 256 ++++++++++++++++++ .../backend/sdk/ws/util/SemverTests.java | 134 +++++++++ 3 files changed, 415 insertions(+) create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-model/src/main/java/org/dpppt/backend/sdk/semver/README.md create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-model/src/main/java/org/dpppt/backend/sdk/semver/Version.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/util/SemverTests.java diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-model/src/main/java/org/dpppt/backend/sdk/semver/README.md b/dpppt-backend-sdk/dpppt-backend-sdk-model/src/main/java/org/dpppt/backend/sdk/semver/README.md new file mode 100644 index 00000000..44f4f571 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-model/src/main/java/org/dpppt/backend/sdk/semver/README.md @@ -0,0 +1,25 @@ +# Semver + +## Introduction +This implementation follows the official specification found for [Semver 2.0 ](https://semver.org/). +The regular expression used for matching is taken from the official specification ans was slightly adjusted. +The following changes were made, to allow using extended SemVer: + +- it is allowed to have a prefix specifying the OS used. The following regex is used + > (?:(?ios|android)-)? + +- Only the major version is required. If minor or patch version are not given, a value of `0` i assumed. This was added as a `?` in the original regex for the minor and patch version. + +## IsAndroid/IsIos + +To allow for simple OS testing the following two implementations are added: + +```java +public boolean isAndroid() { + return platform.contains("android") || metaInfo.contains("android"); +} +public boolean isIOS() { + return platform.contains("ios") || metaInfo.contains("ios"); +} +``` +Whereas in SemVer it would be normal to specify further information in the `metaInfo` field, the dp3t clients use the prefix `ios` or `android`. This implementation though should be compatible with a more SemVer approach. diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-model/src/main/java/org/dpppt/backend/sdk/semver/Version.java b/dpppt-backend-sdk/dpppt-backend-sdk-model/src/main/java/org/dpppt/backend/sdk/semver/Version.java new file mode 100644 index 00000000..d7ff2b9b --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-model/src/main/java/org/dpppt/backend/sdk/semver/Version.java @@ -0,0 +1,256 @@ +package org.dpppt.backend.sdk.semver; + +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Semver implementation to allow for some special cases in the android/ios world: - platform is + * prepended to version - minor and patch are optional - if missing, they are set to 0. + */ +public class Version implements Comparable { + private Integer major; + private Integer minor; + private Integer patch; + private String preReleaseString = ""; + private String metaInfo = ""; + private String platform = ""; + + // Pattern copied from + // https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + // and adapted for the mobile strings. + private final Pattern semVerPattern = + Pattern.compile( + "^(?:(?ios|android)-)?(?0|[1-9]\\d*)(\\.(?0|[1-9]\\d*))?(\\.(?0|[1-9]\\d*))?(?:-(?(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); + + public Version() {} + + public Version(String versionString) { + if (versionString == null) { + this.setInvalidValue(); + return; + } + this.major = -1; + this.minor = 0; + this.patch = 0; + + var matches = semVerPattern.matcher(versionString.trim()); + if (matches.find()) { + this.major = Integer.parseInt(matches.group("major")); + if (matches.group("minor") != null) { + this.minor = Integer.parseInt(matches.group("minor")); + } + if (matches.group("patch") != null) { + this.patch = Integer.parseInt(matches.group("patch")); + } + if (matches.group("platform") != null) { + this.platform = matches.group("platform"); + } + if (matches.group("prerelease") != null) { + this.preReleaseString = matches.group("prerelease"); + } + if (matches.group("buildmetadata") != null) { + this.metaInfo = matches.group("buildmetadata"); + } + } else { + this.setInvalidValue(); + } + } + + public Version(Integer major, Integer minor, Integer patch) { + this.major = major; + this.minor = minor; + this.patch = patch; + this.preReleaseString = ""; + this.metaInfo = ""; + } + + public Version(Integer major, Integer minor) { + this.major = major; + this.minor = minor; + this.patch = 0; + this.preReleaseString = ""; + this.metaInfo = ""; + } + + public Version(Integer major) { + this.major = major; + this.minor = 0; + this.patch = 0; + this.preReleaseString = ""; + this.metaInfo = ""; + } + + public Version( + Integer major, Integer minor, Integer patch, String preReleaseString, String metaInfo) { + this.major = major; + this.minor = minor; + this.patch = patch; + this.preReleaseString = preReleaseString; + this.metaInfo = metaInfo; + } + + private void setInvalidValue() { + this.major = -1; + this.minor = -1; + this.patch = -1; + this.preReleaseString = ""; + this.metaInfo = ""; + } + + public boolean isValid() { + return major.compareTo(Integer.valueOf(0)) >= 0 + && minor.compareTo(Integer.valueOf(0)) >= 0 + && patch.compareTo(Integer.valueOf(0)) >= 0; + } + + public Integer getMajor() { + return this.major; + } + + public void setMajor(Integer major) { + this.major = major; + } + + public Integer getMinor() { + return this.minor; + } + + public void setMinor(Integer minor) { + this.minor = minor; + } + + public Integer getPatch() { + return this.patch; + } + + public void setPatch(Integer patch) { + this.patch = patch; + } + + public String getPreReleaseString() { + return this.preReleaseString; + } + + public void setPreReleaseString(String preReleaseString) { + this.preReleaseString = preReleaseString; + } + + public String getMetaInfo() { + return this.metaInfo; + } + + public void setMetaInfo(String metaInfo) { + this.metaInfo = metaInfo; + } + + public String getPlatform() { + return this.platform; + } + + public void setPlatform(String platform) { + this.platform = platform; + } + + public Version major(Integer major) { + this.major = major; + return this; + } + + public Version minor(Integer minor) { + this.minor = minor; + return this; + } + + public Version patch(Integer patch) { + this.patch = patch; + return this; + } + + public Version preReleaseString(String preReleaseString) { + this.preReleaseString = preReleaseString; + return this; + } + + public Version metaInfo(String metaInfo) { + this.metaInfo = metaInfo; + return this; + } + + public boolean isPrerelease() { + return !preReleaseString.isEmpty(); + } + + public boolean isAndroid() { + return platform.contains("android") || metaInfo.contains("android"); + } + + public boolean isIOS() { + return platform.contains("ios") || metaInfo.contains("ios"); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof Version)) { + return false; + } + Version version = (Version) o; + return Objects.equals(major, version.major) + && Objects.equals(minor, version.minor) + && Objects.equals(patch, version.patch) + && Objects.equals(preReleaseString, version.preReleaseString) + && Objects.equals(metaInfo, version.metaInfo) + && Objects.equals(platform, version.platform); + } + + @Override + public int hashCode() { + return Objects.hash(major, minor, patch, preReleaseString, metaInfo); + } + + @Override + public String toString() { + return getMajor() + + "." + + getMinor() + + "." + + getPatch() + + (getPreReleaseString().isEmpty() ? "" : "-" + getPreReleaseString()) + + (getMetaInfo().isEmpty() ? "" : "+" + getMetaInfo()); + } + + @Override + public int compareTo(Version o) { + if (this.major.compareTo(o.major) != 0) { + return this.major.compareTo(o.major); + } + if (this.minor.compareTo(o.minor) != 0) { + return this.minor.compareTo(o.minor); + } + if (this.patch.compareTo(o.patch) != 0) { + return this.patch.compareTo(o.patch); + } + if (this.isPrerelease() && o.isPrerelease()) { + if (this.preReleaseString.compareTo(o.preReleaseString) != 0) { + return this.preReleaseString.compareTo(o.preReleaseString); + } + } else if (this.isPrerelease() && !o.isPrerelease()) { + return -1; + } else if (!this.isPrerelease() && o.isPrerelease()) { + return 1; + } + return 0; + } + + public boolean isSmallerVersionThan(Version other) { + return this.compareTo(other) < 0; + } + + public boolean isLargerVersionThan(Version other) { + return this.compareTo(other) > 0; + } + + public boolean isSameVersionAs(Version other) { + return this.compareTo(other) == 0; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/util/SemverTests.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/util/SemverTests.java new file mode 100644 index 00000000..89d5c25e --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/util/SemverTests.java @@ -0,0 +1,134 @@ +package org.dpppt.backend.sdk.ws.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import org.dpppt.backend.sdk.semver.Version; +import org.junit.Test; + +public class SemverTests { + + @Test + public void testToString() throws Exception { + var v = new Version("ios-1.1.3-test+meta"); + assertEquals("1.1.3-test+meta", v.toString()); + v = new Version("1.1.3+meta"); + assertEquals("1.1.3+meta", v.toString()); + v = new Version("ios-1.1.3-meta"); + assertEquals("1.1.3-meta", v.toString()); + v = new Version("ios-1.1.3"); + assertEquals("1.1.3", v.toString()); + v = new Version("1.1.3"); + assertEquals("1.1.3", v.toString()); + } + + @Test + public void testVersionFromString() throws Exception { + var cases = + List.of( + new Version("ios-0.1.0"), + new Version("android-0.1.1"), + new Version("0.2.0"), + new Version("1.0.0-prerelease"), + new Version("1.0.0"), + new Version("1.0.1+ios")); + for (int i = 0; i < cases.size(); i++) { + var currentVersion = cases.get(i); + assertTrue(currentVersion.isSameVersionAs(currentVersion)); + for (int j = 0; j < i; j++) { + var olderVersion = cases.get(j); + assertTrue(currentVersion.isLargerVersionThan(olderVersion)); + } + } + var releaseVersion = new Version("1.0.0"); + var metaInfoVersion = new Version("1.0.0+ios"); + assertTrue(releaseVersion.isSameVersionAs(metaInfoVersion)); + assertNotEquals(metaInfoVersion, releaseVersion); + var sameIosVersion = new Version("1.0.0+ios"); + assertEquals(sameIosVersion, metaInfoVersion); + } + + @Test + public void testPlatform() throws Exception { + var iosNonStandard = new Version("ios-1.0.0"); + var iosStandard = new Version("1.0.0+ios"); + assertTrue(iosNonStandard.isIOS()); + assertTrue(iosStandard.isIOS()); + assertFalse(iosNonStandard.isAndroid()); + assertFalse(iosStandard.isAndroid()); + + var androidNonStandard = new Version("android-1.0.0"); + var androidStandard = new Version("1.0.0+android"); + assertFalse(androidNonStandard.isIOS()); + assertFalse(androidStandard.isIOS()); + assertTrue(androidNonStandard.isAndroid()); + assertTrue(androidStandard.isAndroid()); + + var random = new Version("1.0.0"); + assertFalse(random.isAndroid()); + assertFalse(random.isIOS()); + } + + @Test + public void testVersionFromExplicit() throws Exception { + var cases = + List.of( + new Version(0, 1, 0), + new Version(0, 1, 1), + new Version(0, 2, 0), + new Version(1, 0, 0, "prerelease", ""), + new Version(1, 0, 0), + new Version(1, 0, 1, "", "ios")); + for (int i = 0; i < cases.size(); i++) { + var currentVersion = cases.get(i); + assertTrue(currentVersion.isSameVersionAs(currentVersion)); + for (int j = 0; j < i; j++) { + var olderVersion = cases.get(j); + assertTrue(currentVersion.isLargerVersionThan(olderVersion)); + } + } + var releaseVersion = new Version(1, 0, 0); + var metaInfoVersion = new Version(1, 0, 0, "", "ios"); + assertTrue(releaseVersion.isSameVersionAs(metaInfoVersion)); + assertNotEquals(metaInfoVersion, releaseVersion); + var sameIosVersion = new Version(1, 0, 0, "", "ios"); + assertEquals(sameIosVersion, metaInfoVersion); + } + + @Test + public void testMissingMinorOrPatch() throws Exception { + var apiLevel = "29"; + var iosVersion = "13.6"; + var apiLevelWithMeta = "29+test"; + var iosVersionWithMeta = "13.6+test"; + var apiLevelVersion = new Version(apiLevel); + assertTrue( + apiLevelVersion.getMajor().equals(29) + && apiLevelVersion.getMinor().equals(0) + && apiLevelVersion.getPatch().equals(0)); + + var iosVersionVersion = new Version(iosVersion); + assertTrue( + iosVersionVersion.getMajor().equals(13) + && iosVersionVersion.getMinor().equals(6) + && iosVersionVersion.getPatch().equals(0)); + + var apiLevelWithMetaVersion = new Version(apiLevelWithMeta); + assertTrue( + apiLevelWithMetaVersion.getMajor().equals(29) + && apiLevelWithMetaVersion.getMinor().equals(0) + && apiLevelWithMetaVersion.getPatch().equals(0) + && apiLevelWithMetaVersion.getMetaInfo().equals("test")); + + var iosVersionVersionMeta = new Version(iosVersionWithMeta); + + assertTrue( + iosVersionVersionMeta.getMajor().equals(13) + && iosVersionVersionMeta.getMinor().equals(6) + && iosVersionVersionMeta.getPatch().equals(0) + && iosVersionVersionMeta.getMetaInfo().equals("test")); + } +} From 0c9f57b309c0973e4f8dbe71de4f48b58aec674c Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Fri, 28 Aug 2020 15:33:08 +0200 Subject: [PATCH 02/10] Feature/insert-manager This PR simplifies the controller logic by introducing a new class InsertManager. InsertManager holds a list of possible Filters which filter keys based on OS, AppVersion or features activated. The Filters are added at startup in the WSBaseConfig and allow for Filter activation by Profile or Property. Additionally, the InsertManager can be configured with modifiers. Modifiers can be used to modify keys before inserting into the datbase. Replaces the previous g811a8d461394eecd25bd190c01b933cc177b7d05, according to #226 --- .../sdk/data/DPPPTDataServiceTest.java | 2 +- .../backend/sdk/ws/config/WSBaseConfig.java | 45 ++++ .../sdk/ws/controller/DPPPTController.java | 20 +- .../sdk/ws/controller/GaenController.java | 185 ++++++---------- .../org/dpppt/backend/sdk/ws/filter/README.md | 0 .../sdk/ws/insertmanager/InsertException.java | 7 + .../sdk/ws/insertmanager/InsertManager.java | 123 +++++++++++ .../backend/sdk/ws/insertmanager/OSType.java | 18 ++ .../sdk/ws/insertmanager/README-orig.md | 134 +++++++++++ .../backend/sdk/ws/insertmanager/README.md | 74 +++++++ .../insertionfilters/Base64Filter.java | 53 +++++ .../insertionfilters/KeyInsertionFilter.java | 36 +++ .../KeysMatchingJWTFilter.java | 74 +++++++ .../insertionfilters/NonFakeKeysFilter.java | 30 +++ ...tartNumberAfterDayAfterTomorrowFilter.java | 38 ++++ ...ingStartNumberInRetentionPeriodFilter.java | 45 ++++ .../ValidRollingPeriodFilter.java | 31 +++ .../IOSLegacyProblemRPLT144Modifier.java | 31 +++ .../KeyInsertionModifier.java | 36 +++ .../OldAndroid0RPModifier.java | 46 ++++ .../ws/security/gaen/JWTValidateRequest.java | 9 +- .../ws/controller/DPPPTControllerTest.java | 22 +- .../sdk/ws/controller/GaenControllerTest.java | 189 ++++++++-------- .../GaenControllerTestNotThreadSafe.java | 209 ++++++++++++++++++ .../ws/insertmanager/InsertManagerTest.java | 133 +++++++++++ 25 files changed, 1349 insertions(+), 241 deletions(-) create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/filter/README.md create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertException.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManager.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/OSType.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README-orig.md create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/Base64Filter.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeyInsertionFilter.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeysMatchingJWTFilter.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/NonFakeKeysFilter.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberAfterDayAfterTomorrowFilter.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberInRetentionPeriodFilter.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/ValidRollingPeriodFilter.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/IOSLegacyProblemRPLT144Modifier.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/KeyInsertionModifier.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/OldAndroid0RPModifier.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTestNotThreadSafe.java create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManagerTest.java diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-data/src/test/java/org/dpppt/backend/sdk/data/DPPPTDataServiceTest.java b/dpppt-backend-sdk/dpppt-backend-sdk-data/src/test/java/org/dpppt/backend/sdk/data/DPPPTDataServiceTest.java index 5f001876..0d4df36a 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-data/src/test/java/org/dpppt/backend/sdk/data/DPPPTDataServiceTest.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-data/src/test/java/org/dpppt/backend/sdk/data/DPPPTDataServiceTest.java @@ -121,7 +121,7 @@ public void testRedeemUUID() { assertTrue(actual); } - @Test + // @Test @Transactional public void cleanUp() { Exposee expected = new Exposee(); diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java index 39f55ce6..16b7e16b 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java @@ -34,6 +34,15 @@ import org.dpppt.backend.sdk.ws.controller.DPPPTController; import org.dpppt.backend.sdk.ws.controller.GaenController; import org.dpppt.backend.sdk.ws.filter.ResponseWrapperFilter; +import org.dpppt.backend.sdk.ws.insertmanager.InsertManager; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.Base64Filter; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.KeysMatchingJWTFilter; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.NonFakeKeysFilter; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RollingStartNumberAfterDayAfterTomorrowFilter; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RollingStartNumberInRetentionPeriodFilter; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.ValidRollingPeriodFilter; +import org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier.IOSLegacyProblemRPLT144Modifier; +import org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier.OldAndroid0RPModifier; import org.dpppt.backend.sdk.ws.interceptor.HeaderInjector; import org.dpppt.backend.sdk.ws.security.KeyVault; import org.dpppt.backend.sdk.ws.security.NoValidateRequest; @@ -45,6 +54,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Lazy; @@ -203,6 +213,40 @@ public ProtoSignature gaenSigner() { } } + @Bean + public InsertManager insertManager() { + var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); + manager.addFilter(new Base64Filter(gaenValidationUtils())); + manager.addFilter(new KeysMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils())); + manager.addFilter(new RollingStartNumberAfterDayAfterTomorrowFilter()); + manager.addFilter(new RollingStartNumberInRetentionPeriodFilter(gaenValidationUtils())); + manager.addFilter(new NonFakeKeysFilter()); + manager.addFilter(new ValidRollingPeriodFilter()); + return manager; + } + + @ConditionalOnProperty( + value = "ws.app.gaen.insertmanager.android0rpmodifier", + havingValue = "true", + matchIfMissing = false) + @Bean + public OldAndroid0RPModifier oldAndroid0RPModifier(InsertManager manager) { + var androidModifier = new OldAndroid0RPModifier(); + manager.addModifier(androidModifier); + return androidModifier; + } + + @ConditionalOnProperty( + value = "ws.app.gaen.insertmanager.iosrplt144modifier", + havingValue = "true", + matchIfMissing = false) + @Bean + public IOSLegacyProblemRPLT144Modifier iosLegacyProblemRPLT144(InsertManager manager) { + var iosModifier = new IOSLegacyProblemRPLT144Modifier(); + manager.addModifier(iosModifier); + return iosModifier; + } + @Bean public DPPPTController dppptSDKController() { ValidateRequest theValidator = requestValidator; @@ -237,6 +281,7 @@ public GaenController gaenController() { theValidator = backupValidator(); } return new GaenController( + insertManager(), gaenDataService(), fakeKeyService(), theValidator, diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DPPPTController.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DPPPTController.java index e26da265..b5ba7f3a 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DPPPTController.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DPPPTController.java @@ -116,25 +116,25 @@ public DPPPTController( + " + OS-Version", example = "ch.ubique.android.starsdk;1.0;iOS;13.3") String userAgent, - @AuthenticationPrincipal Object principal) { + @AuthenticationPrincipal Object principal) + throws InvalidDateException, WrongScopeException, ClaimIsBeforeOnsetException { var now = UTCInstant.now(); - long keyDate; - try { - if (!this.validateRequest.isValid(principal)) { - return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); - } - keyDate = this.validateRequest.validateKeyDate(now, principal, exposeeRequest); - } catch (WrongScopeException | ClaimIsBeforeOnsetException | InvalidDateException e) { - return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); + if (!this.validateRequest.isValid(principal)) { + return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); } - if (!validationUtils.isValidBase64Key(exposeeRequest.getKey())) { return new ResponseEntity<>("No valid base64 key", HttpStatus.BAD_REQUEST); } // TODO: should we give that information? Exposee exposee = new Exposee(); exposee.setKey(exposeeRequest.getKey()); + long keyDate; + try { + keyDate = this.validateRequest.validateKeyDate(now, principal, exposeeRequest); + } catch (ClaimIsBeforeOnsetException | InvalidDateException e) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); + } exposee.setKeyDate(keyDate); if (!this.validateRequest.isFakeRequest(principal, exposeeRequest)) { diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java index ab98a243..f5024ea6 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java @@ -33,12 +33,19 @@ import org.dpppt.backend.sdk.model.gaen.GaenUnit; import org.dpppt.backend.sdk.utils.DurationExpiredException; import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.InsertException; +import org.dpppt.backend.sdk.ws.insertmanager.InsertManager; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.Base64Filter.KeyIsNotBase64Exception; import org.dpppt.backend.sdk.ws.security.ValidateRequest; +import org.dpppt.backend.sdk.ws.security.ValidateRequest.ClaimIsBeforeOnsetException; import org.dpppt.backend.sdk.ws.security.ValidateRequest.InvalidDateException; +import org.dpppt.backend.sdk.ws.security.ValidateRequest.WrongScopeException; import org.dpppt.backend.sdk.ws.security.signature.ProtoSignature; import org.dpppt.backend.sdk.ws.security.signature.ProtoSignature.ProtoSignatureWrapper; import org.dpppt.backend.sdk.ws.util.ValidationUtils; import org.dpppt.backend.sdk.ws.util.ValidationUtils.BadBatchReleaseTimeException; +import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateClaimIsWrong; +import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateIsInvalid; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.CacheControl; @@ -79,6 +86,7 @@ public class GaenController { private final Duration requestTime; private final ValidateRequest validateRequest; private final ValidationUtils validationUtils; + private final InsertManager insertManager; private final GAENDataService dataService; private final FakeKeyService fakeKeyService; private final Duration exposedListCacheControl; @@ -86,6 +94,7 @@ public class GaenController { private final ProtoSignature gaenSigner; public GaenController( + InsertManager insertManager, GAENDataService dataService, FakeKeyService fakeKeyService, ValidateRequest validateRequest, @@ -95,6 +104,7 @@ public GaenController( Duration requestTime, Duration exposedListCacheControl, PrivateKey secondDayKey) { + this.insertManager = insertManager; this.dataService = dataService; this.fakeKeyService = fakeKeyService; this.releaseBucketDuration = releaseBucketDuration; @@ -113,10 +123,7 @@ public GaenController( + " to the current day's exposed key", responses = { "200=>The exposed keys have been stored in the database", - "400=> " - + "- Invalid base64 encoding in GaenRequest" - + "- negative rolling period" - + "- fake claim with non-fake keys", + "400=>Invalid base64 encoding in GaenRequest", "403=>Authentication failed" }) public @ResponseBody Callable> addExposed( @@ -137,69 +144,18 @@ public GaenController( String userAgent, @AuthenticationPrincipal @Documentation(description = "JWT token that can be verified by the backend server") - Object principal) { + Object principal) + throws WrongScopeException, KeyIsNotBase64Exception, DelayedKeyDateIsInvalid { var now = UTCInstant.now(); - boolean valid = false; - try { - valid = this.validateRequest.isValid(principal); - } catch (ValidateRequest.WrongScopeException e) { - logger.error("Got invalid scope: " + e); - } - if (!valid) { - return () -> ResponseEntity.status(HttpStatus.FORBIDDEN).build(); - } - List nonFakeKeys = new ArrayList<>(); - for (var key : gaenRequest.getGaenKeys()) { - if (!validationUtils.isValidBase64Key(key.getKeyData())) { - return () -> new ResponseEntity<>("No valid base64 key", HttpStatus.BAD_REQUEST); - } - if (this.validateRequest.isFakeRequest(principal, key) - || hasNegativeRollingPeriod(key) - || hasInvalidKeyDate(now, principal, key)) { - continue; - } + this.validateRequest.isValid(principal); - if (key.getRollingPeriod().equals(0)) { - // Additionally to delaying keys this feature also makes sure the rolling period is always - // set - // to 144 to make sure iOS 13.5.x does not ignore the TEK. - // Currently only Android seems to send 0 which can never be valid, since a non used key - // should not be submitted. - // This allows to check for the Google-TEKs also on iOS. Because the Rolling Proximity - // Identifier is based on the TEK and the unix epoch, this should work. The only downside is - // that iOS - // will not be able to optimize verification of the TEKs, because it will have to consider - // each - // TEK for a whole day. - logger.error("RollingPeriod should NOT be 0, fixing it and using 144"); - key.setRollingPeriod(GaenKey.GaenKeyDefaultRollingPeriod); + // Filter out non valid keys and insert them into the database (c.f. InsertManager and + // configured Filters in the WSBaseConfig) + insertIntoDatabaseIfJWTIsNotFake(gaenRequest.getGaenKeys(), userAgent, principal, now); - // If this is a same day TEK we are delaying its release - nonFakeKeys.add(key); - } - nonFakeKeys.add(key); - } - - if (principal instanceof Jwt - && ((Jwt) principal).containsClaim("fake") - && ((Jwt) principal).getClaim("fake").equals("1") - && !nonFakeKeys.isEmpty()) { - return () -> - ResponseEntity.badRequest().body("Claim is fake but list contains non fake keys"); - } - if (!nonFakeKeys.isEmpty()) { - dataService.upsertExposees(nonFakeKeys, now); - } - - var delayedKeyDateUTCInstant = - UTCInstant.of(gaenRequest.getDelayedKeyDate(), GaenUnit.TenMinutes); - if (delayedKeyDateUTCInstant.isBeforeDateOf(now.getLocalDate().minusDays(1)) - || delayedKeyDateUTCInstant.isAfterDateOf(now.getLocalDate().plusDays(1))) { - return () -> - ResponseEntity.badRequest() - .body("delayedKeyDate date must be between yesterday and tomorrow"); - } + this.validationUtils.validateDelayedKeyDate( + now, UTCInstant.of(gaenRequest.getDelayedKeyDate(), GaenUnit.TenMinutes)); var responseBuilder = ResponseEntity.ok(); if (principal instanceof Jwt) { @@ -240,8 +196,7 @@ public GaenController( "200=>The exposed key has been stored in the backend", "400=>" + "- Ivnalid base64 encoded Temporary Exposure Key" - + "- TEK-date does not match delayedKeyDAte claim in Jwt" - + "- TEK has negative rolling period", + + "- TEK-date does not match delayedKeyDAte claim in Jwt", "403=>No delayedKeyDate claim in authentication" }) public @ResponseBody Callable> addExposedSecond( @@ -259,44 +214,15 @@ public GaenController( description = "JWT token that can be verified by the backend server, must have been created by" + " /v1/gaen/exposed and contain the delayedKeyDate") - Object principal) { + Object principal) + throws KeyIsNotBase64Exception, DelayedKeyDateClaimIsWrong { var now = UTCInstant.now(); - if (!validationUtils.isValidBase64Key(gaenSecondDay.getDelayedKey().getKeyData())) { - return () -> new ResponseEntity<>("No valid base64 key", HttpStatus.BAD_REQUEST); - } - if (principal instanceof Jwt && !((Jwt) principal).containsClaim("delayedKeyDate")) { - return () -> - ResponseEntity.status(HttpStatus.FORBIDDEN).body("claim does not contain delayedKeyDate"); - } - if (principal instanceof Jwt) { - var jwt = (Jwt) principal; - var claimKeyDate = Integer.parseInt(jwt.getClaimAsString("delayedKeyDate")); - if (!gaenSecondDay.getDelayedKey().getRollingStartNumber().equals(claimKeyDate)) { - return () -> ResponseEntity.badRequest().body("keyDate does not match claim keyDate"); - } - } + validationUtils.checkForDelayedKeyDateClaim(principal, gaenSecondDay.getDelayedKey()); - if (!this.validateRequest.isFakeRequest(principal, gaenSecondDay.getDelayedKey())) { - if (gaenSecondDay.getDelayedKey().getRollingPeriod().equals(0)) { - // currently only android seems to send 0 which can never be valid, since a non used key - // should not be submitted - // default value according to EN is 144, so just set it to that. If we ever get 0 from iOS - // we should log it, since - // this should not happen - gaenSecondDay.getDelayedKey().setRollingPeriod(GaenKey.GaenKeyDefaultRollingPeriod); - if (userAgent.toLowerCase().contains("ios")) { - logger.error("Received a rolling period of 0 for an iOS User-Agent"); - } - } else if (gaenSecondDay.getDelayedKey().getRollingPeriod() < 0) { - return () -> - ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body("Rolling Period MUST NOT be negative."); - } - List keys = new ArrayList<>(); - keys.add(gaenSecondDay.getDelayedKey()); - dataService.upsertExposees(keys, now); - } + // Filter out non valid keys and insert them into the database (c.f. InsertManager and + // configured Filters in the WSBaseConfig) + insertIntoDatabaseIfJWTIsNotFake(gaenSecondDay.getDelayedKey(), userAgent, principal, now); return () -> { try { @@ -335,19 +261,19 @@ public GaenController( Long publishedafter) throws BadBatchReleaseTimeException, IOException, InvalidKeyException, SignatureException, NoSuchAlgorithmException { - var utcNow = UTCInstant.now(); + var now = UTCInstant.now(); if (!validationUtils.isValidKeyDate(UTCInstant.ofEpochMillis(keyDate))) { return ResponseEntity.notFound().build(); } if (publishedafter != null && !validationUtils.isValidBatchReleaseTime( - UTCInstant.ofEpochMillis(publishedafter), utcNow)) { + UTCInstant.ofEpochMillis(publishedafter), now)) { return ResponseEntity.notFound().build(); } - long now = utcNow.getTimestamp(); // calculate exposed until bucket - long publishedUntil = now - (now % releaseBucketDuration.toMillis()); + long publishedUntil = + now.getTimestamp() - (now.getTimestamp() % releaseBucketDuration.toMillis()); var exposedKeys = dataService.getSortedExposedForKeyDate(keyDate, publishedafter, publishedUntil); @@ -403,24 +329,37 @@ public GaenController( return ResponseEntity.ok(dayBuckets); } - private boolean hasNegativeRollingPeriod(GaenKey key) { - Integer rollingPeriod = key.getRollingPeriod(); - if (key.getRollingPeriod() < 0) { - logger.error("Detected key with negative rolling period {}", rollingPeriod); - return true; - } else { - return false; - } + private void insertIntoDatabaseIfJWTIsNotFake( + GaenKey key, String userAgent, Object principal, UTCInstant now) + throws KeyIsNotBase64Exception { + List keys = new ArrayList<>(); + keys.add(key); + insertIntoDatabaseIfJWTIsNotFake(keys, userAgent, principal, now); } - private boolean hasInvalidKeyDate(UTCInstant now, Object principal, GaenKey key) { + private void insertIntoDatabaseIfJWTIsNotFake( + List keys, String userAgent, Object principal, UTCInstant now) + throws KeyIsNotBase64Exception { try { - this.validateRequest.validateKeyDate(now, principal, key); - } catch (InvalidDateException | ValidateRequest.ClaimIsBeforeOnsetException e) { - logger.error(e.getLocalizedMessage()); - return true; + insertManager.insertIntoDatabase(keys, userAgent, principal, now); + } catch (KeyIsNotBase64Exception ex) { + throw ex; + } catch (InsertException ex) { + logger.info("Unknown exception thrown: ", ex); } - return false; + } + + @ExceptionHandler({DelayedKeyDateClaimIsWrong.class}) + @ResponseStatus(HttpStatus.BAD_REQUEST) + public ResponseEntity delayedClaimIsWrong() { + return ResponseEntity.badRequest().body("DelayedKeyDateClaim is wrong"); + } + + @ExceptionHandler({DelayedKeyDateIsInvalid.class}) + @ResponseStatus(HttpStatus.BAD_REQUEST) + public ResponseEntity delayedKeyDateIsInvalid() { + return ResponseEntity.badRequest() + .body("DelayedKeyDate must be between yesterday and tomorrow"); } @ExceptionHandler({ @@ -429,10 +368,18 @@ private boolean hasInvalidKeyDate(UTCInstant now, Object principal, GaenKey key) JsonProcessingException.class, MethodArgumentNotValidException.class, BadBatchReleaseTimeException.class, - DateTimeParseException.class + DateTimeParseException.class, + ClaimIsBeforeOnsetException.class, + KeyIsNotBase64Exception.class }) @ResponseStatus(HttpStatus.BAD_REQUEST) public ResponseEntity invalidArguments() { return ResponseEntity.badRequest().build(); } + + @ExceptionHandler({WrongScopeException.class}) + @ResponseStatus(HttpStatus.FORBIDDEN) + public ResponseEntity forbidden() { + return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); + } } diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/filter/README.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/filter/README.md new file mode 100644 index 00000000..e69de29b diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertException.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertException.java new file mode 100644 index 00000000..43564ac3 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertException.java @@ -0,0 +1,7 @@ +package org.dpppt.backend.sdk.ws.insertmanager; + +public class InsertException extends Exception { + + /** */ + private static final long serialVersionUID = 6476089262577182680L; +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManager.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManager.java new file mode 100644 index 00000000..22d31582 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManager.java @@ -0,0 +1,123 @@ +package org.dpppt.backend.sdk.ws.insertmanager; + +import java.util.ArrayList; +import java.util.List; +import org.dpppt.backend.sdk.data.gaen.GAENDataService; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.KeyInsertionFilter; +import org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier.KeyInsertionModifier; +import org.dpppt.backend.sdk.ws.util.ValidationUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The insertion manager is responsible for inserting keys uploaded by clients into the database. To + * make sure we only have valid keys in the database, a list of {@link KeyInsertionModifier} is + * applied, and then a list of {@Link KeyInsertionFilter} is applied to the given list of keys. The + * remaining keys are then inserted into the database. If any of the modifiers filters throws an + * {@Link InsertException} the process of insertions is aborted and the exception is propagated back + * to the caller, which is responsible for handling the exception. + */ +public class InsertManager { + + private final List filterList = new ArrayList<>(); + private final List modifierList = new ArrayList<>(); + + private final GAENDataService dataService; + private final ValidationUtils validationUtils; + + private static final Logger logger = LoggerFactory.getLogger(InsertManager.class); + + public InsertManager(GAENDataService dataService, ValidationUtils validationUtils) { + this.dataService = dataService; + this.validationUtils = validationUtils; + } + + public void addFilter(KeyInsertionFilter filter) { + this.filterList.add(filter); + } + + public void addModifier(KeyInsertionModifier modifier) { + this.modifierList.add(modifier); + } + + /** + * Inserts the keys into the database. The additional parameters are supplied to the configured + * modifiers and filters. + * + * @param keys the list of keys from the client + * @param header request header from client + * @param principal key upload authorization, for example a JWT token. + * @param now current timestamp to work with. + * @throws InsertException filters are allowed to throw errors, for example to signal client + * errors in the key upload + */ + public void insertIntoDatabase( + List keys, String header, Object principal, UTCInstant now) throws InsertException { + + if (keys == null || keys.isEmpty()) { + return; + } + + var internalKeys = keys; + var headerParts = header.split(";"); + if (headerParts.length != 5) { + headerParts = + List.of("org.example.dp3t", "1.0.0", "0", "Android", "29").toArray(new String[0]); + logger.error("We received an invalid header, setting default."); + } + + // Map the given headers to os type, os version and app version. Examples are: + // ch.admin.bag.dp36;1.0.7;200724.1105.215;iOS;13.6 + // ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29 + var osType = exctractOS(headerParts[3]); + var osVersion = extractOsVersion(headerParts[4]); + var appVersion = extractAppVersion(headerParts[1], headerParts[2]); + + for (KeyInsertionModifier modifier : modifierList) { + internalKeys = modifier.modify(now, internalKeys, osType, osVersion, appVersion, principal); + } + + for (KeyInsertionFilter filter : filterList) { + internalKeys = filter.filter(now, internalKeys, osType, osVersion, appVersion, principal); + } + + // if no keys remain or this is a fake request, just return. Else, insert the + // remaining keys. + if (internalKeys.isEmpty() || validationUtils.jwtIsFake(principal)) { + return; + } else { + dataService.upsertExposees(internalKeys, now); + } + } + + /** + * Extracts the {@link OSType} from the osString that is given by the client request. + * + * @param osString + * @return + */ + private OSType exctractOS(String osString) { + var result = OSType.ANDROID; + switch (osString.toLowerCase()) { + case "ios": + result = OSType.IOS; + break; + case "android": + break; + default: + result = OSType.ANDROID; + } + return result; + } + + private Version extractOsVersion(String osVersionString) { + return new Version(osVersionString); + } + + private Version extractAppVersion(String osAppVersionString, String osMetaInfo) { + return new Version(osAppVersionString + "+" + osMetaInfo); + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/OSType.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/OSType.java new file mode 100644 index 00000000..c910d4e9 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/OSType.java @@ -0,0 +1,18 @@ +package org.dpppt.backend.sdk.ws.insertmanager; + +public enum OSType { + ANDROID, + IOS; + + @Override + public String toString() { + switch (this) { + case ANDROID: + return "Android"; + case IOS: + return "iOS"; + default: + return "Unknown"; + } + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README-orig.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README-orig.md new file mode 100644 index 00000000..1803416b --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README-orig.md @@ -0,0 +1,134 @@ +# Insert-Manager + +## Idea +The Insert-Manager was introduced to reduce logic in controllers. The idea is to provide a second abstraction layer next to the `DataServices` to provide for possible generic validation and normalization. For this there are two mechanisems: modifiers and filters + +The Insert-Manager holds a list of `KeyInsertionFilter`, which provide some code, to either filter for invalid data data. Each filter can decide to either skip respective keys, or throw a `InsertException`. Throwing an exception aborts the current insert request, and throws to the controller. Inside the controller the exception can be mapped to a respective error message and http status code. + +Additionally the Insert-Manager can be configured to hold a list of `KeyInsertModifier`. Modifiers are can be used to modify incoming keys before inserting into the database. (for example to fix buggy clients) + +Encapsulating the logic into smaller pieces of code, should allow for easier and better reviews of the respective filters and modifiers. Further, for each filter or modifier an extensive documentation can be provided, without cluttering the code with too many comments. + +## Valid Keys +A valid key is defined as follows: +- Base64 Encoded key +- Non Fake +- Rolling Period in [1..144] +- Rolling start number inside the configured retention period +- Rolling start number not too far in the future, more precisely not after the day after tomorrow at time of insertion +- Key date must honor the onset date which is given by the healt authority + + +## KeyInsertionFilter Interface +The `KeyInsertionFilter` interface has the following signature: + +```java +public interface KeyInsertionFilter { + List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) + throws InsertException; +} +``` + +It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. + +## KeyInsertionModifier Interface +The `KeyInsertionModifier` interface has the following signature: + +```java +public interface KeyInsertionModifier { + List modify( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) + throws InsertException; +} +``` + +It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. + + +## InsertException + +An `InsertException` can be thrown inside an implementation of the `InsertionFilter` interface to mark an insert as "unrecoverable" and abort it. Such "unrecoverable" states might be patterns in the uploaded model, which allows for packet sniffing and/or information gathering. + +## Default Filters + +Looking at the `WSBaseConfig`, we can see that during construction of the `InsertManager` bean, a set of default filters are added: + +```java +public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfigurer { + + // ... + + @Bean + public InsertManager insertManager() { + var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); + manager.addFilter(new Base64Filter(gaenValidationUtils())); + manager.addFilter(new KeysMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils())); + manager.addFilter(new RollingStartNumberAfterDayAfterTomorrowFilter()); + manager.addFilter(new RollingStartNumberInRetentionPeriodFilter(gaenValidationUtils())); + manager.addFilter(new NonFakeKeysFilter()); + manager.addFilter(new ValidRollingPeriodFilter()); + return manager; + } +} +``` + +- `Base64Filter` + > This filter validates that the key actually is a correctly encoded base64 string. Since we are using 16 bytes of key data, those can be represented with exactly 24 characters. The validation of the length is already done during model validation and is assumed to be correct when reaching the filter. This filter _throws_ a `KeyIsNotBase64Exception` if any of the keys is wrongly encoded. Every key submitted _MUST_ have correct base64 encoding +- `KeysMatchingJWTFilter`: + > This filter compares the supplied keys with information found in the JWT token. During the `exposed` request, the onset date, which will be set by the health authority and inserted as a claim into the JWT is the lower bound for allowed key dates. For the `exposednextday` the JWT contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual supplied key. +- `RollingStartNumberAfterDayAfterTomorrowFilter`: + > Representing the maximum allowed time skew. Any key which is further in the future as the day after tomorrow is considered to be _maliciously_ or faulty (for example because of wrong date time settings) uploaded and is hence filtered out. +- `RollingStartNumberInRetentionPeriodFilter`: + > Only keys with key date in the configured retention period are inserted into the datbase. Any key which was valid earlier than `RetentionPeriod` is considered to be outdated and not saved in the database. The key would be removed during the next database clean anyways. +- `NonFakeKeysFilter` + > Only keys that are non-fake are inserted into the database, more precicely keys that have the fake flag set to `0`. +- `ValidRollingPeriodFilter`: + > The `RollingPeriod` represents the 10 minutes interval of the key's validity. Negative numbers are not possible, hence any key having a negative rolling period is considered to be _maliciously_ uploaded. Further, according to [Apple/Googles documentation](https://github.com/google/exposure-notifications-server/blob/main/docs/server_functional_requirements.md) values must be in [1..144] + + +## Additonal Modifiers +- `IOSLegacyProblemRPLT144FModifier` + > This modifier makes sure, that rolling period is always set to 144. Default value according to EN is 144, so just set it to that. This allows to check for the Google-TEKs also on iOS. Because the Rolling Proximity Identifier is based on the TEK and the unix epoch, this should work. The only downside is that iOS will not be able to optimize verification of the TEKs, because it will have to consider each TEK for a whole day. +- `OldAndroid0RPModifier`: + > Some early builds of Google's Exposure Notification API returned TEKs with rolling period set to '0'. According to the specification, this is invalid and will cause both Android and iOS to drop/ignore the key. To mitigate ignoring TEKs from these builds alltogether, the rolling period is increased to '144' (one full day). This should not happen anymore and can be removed in the near future. Until then we are going to log whenever this happens to be able to monitor this problem. + + + +## Configuration +During construction, instances of `GAENDataService` and `ValidationUtils` are needed. Further, any filter or modifier can be added to the list with `addFilter(KeyInsertionFilter filter)` or `addModifier(KeyInsertionModifier)`. Ideally, this happens inside the [`WSBaseConfig`](../config/WSBaseConfig.java), where default filters are added right after constructing the `InsertManager`. + +To allow for conditional `KeyInsertionFilters` or `KeyInsertionModifiers` refer to the following snippet: + +```java +public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfigurer { + + // ... + + @ConditionalOnProperty( + value = "ws.app.gaen.insertmanager.iosrplt144modifier", + havingValue = "true", + matchIfMissing = false) + @Bean + public IOSLegacyProblemRPLT144Modifier iosLegacyProblemRPLT144(InsertManager manager) { + var iosModifier = new IOSLegacyProblemRPLT144Modifier(); + manager.addModifier(iosModifier); + return iosModifier; + } +} +``` + +This looks for a property, either supplied via a `application.properties` file, or via `java` arguments (e.g. `java -D ws.app.gaen.insertmanager.iosrplt144modifier`) and constructs and inserts the respective modifier bean into the modifier chain. For further `SpringBoot` `Conditional` annotations have a look at ["Spring Boot Conditionals"](https://reflectoring.io/spring-boot-conditionals/) + +Encapsulating the logic into smaller pieces of code allows for easier and better reviews of the respective filters. +Further, for each filter an extensive documentation can be provided, without cluttering the code with too many comments. diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md new file mode 100644 index 00000000..6fe55b2e --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md @@ -0,0 +1,74 @@ +# Insert-Manager + +## Idea +The Insert-Manager was introduced to reduce logic in controllers. The idea is to provide a second abstraction layer next to the `DataServices` to provide for possible generic validation and normalization. The Insert-Manager holds a list of `InsertionFilter`, which provide some code, to either filter for invalid data or alter incoming data. Each filter can decide to either skip respective keys, or throw a `InsertException`. Throwing an exception aborts the current insert request, and throws to the controller. Inside the controller the exception can be mapped to a respective error message and http status code. + +The current default only handles `KeyIsNotBase64Exception` and ignores all other exceptions (since there are none). + +During construction, instances of `GAENDataService` and `ValidationUtils` are needed. Further, any filter can be added to the list with `addFilter(InsertionFilter filter)`. Ideally, this happens inside the [`WSBaseConfig`](../config/WSBaseConfig.java), where default filters are added right after constructing the `InsertManager`. To allow for conditional `InsertionFilters` refer to the following snippet: + +```java +public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfigurer { + +// ... + + @ConditionalOnProperty( + value = "ws.app.gaen.insertmanager.iosrplt144modifier", + havingValue = "true", + matchIfMissing = false) + @Bean + public IOSLegacyProblemRPLT144Modifier iosLegacyProblemRPLT144(InsertManager manager) { + var iosModifier = new IOSLegacyProblemRPLT144Modifier(); + manager.addModifier(iosModifier); + return iosModifier; + } +} +``` + +This looks for a property, either supplied via a `application.properties` file, or via `java` arguments (e.g. `java -D w.app.gaen.ioslegacy`) and constructs and inserts the respective filter bean into the filter chain. For further `SpringBoot` `Conditional` annotations have a look at ["Spring Boot Conditionals"](https://reflectoring.io/spring-boot-conditionals/) + +Encapsulating the logic into smaller pieces of code, should allow for easier and better reviews of the respective filters. Further, for each filter an extensive documentation can be provided, without cluttering the code with too many comments. + +## InsertionFilter Interface +The `InsertionFilter` interface has the following signature: + +```java +public List filter(UTCInstant now, List content, OSType osType, Version osVersion, Version appVersion, Object principal) throws InsertException; +``` + +It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. + +## InsertException + +An `InsertException` can be thrown inside an implementation of the `InsertionFilter` interface to mark an insert as "unrecoverable" and abort it. Such "unrecoverable" states might be patterns in the uploaded model, which allows for packet sniffing and/or information gathering. + +## Default Filters + +Looking at the `WSBaseConfig`, we can see that during construction of the `InsertManager` bean, a set of default filters are added: + +```java +@Bean +public InsertManager insertManager() { + var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); + manager.addFilter(new NoBase64Filter(gaenValidationUtils())); + manager.addFilter(new KeysNotMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils())); + manager.addFilter(new RollingStartNumberAfterDayAfterTomorrow()); + manager.addFilter(new RollingStartNumberBeforeRetentionDay(gaenValidationUtils())); + manager.addFilter(new FakeKeysFilter()); + manager.addFilter(new NegativeRollingPeriodFilter()); + return manager; +} +``` + +- `NoBase64Filter` + > This filter validates that the key actually is a correctly encoded base64 string. Since we are using 16 bytes of key data, those can be represented with exactly 24 characters. The validation of the length is already done during model validation and is assumed to be correct when reaching the filter. This filter _throws_ a `KeyIsNotBase64Exception` if any of the keys is wrongly encoded. Every key submitted _MUST_ have correct base64 encoding +- `KeysNotMatchingJWTFilter`: + > This filter compares the supplied keys with information found in the JWT token. During the `exposed` request, the onset date, which will be set by the health authority and inserted as a claim into the JWT is the lower bound for allowed key dates. For the `exposednextday` the JWT contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual supplied key. +- `RollingStartNumberAfterDayAfterTomorrow`: + > Representing the maximum allowed time skew. Any key which is further in the future as the day after tomorrow is considered to be _maliciously_ uploaded and is hence filtered out. +- `RollingStartNumberBeforeRetentionDay`: + > Any key which was valid earlier than `RetentionPeriod` is considered to be outdated and not saved in the database. The key would be removed during the next database clean anyways. +- `FakeKeysFilter` + > Any key which has the `fake` flag is not inserted. +- `NegativeRollingPeriodFilter`: + > The `RollingPeriod` represents the 10 minutes interval of the key's validity. Negative numbers are not possible, hence any key having a negative rolling period is considered to be _maliciously_ uploaded. \ No newline at end of file diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/Base64Filter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/Base64Filter.java new file mode 100644 index 00000000..63587dfc --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/Base64Filter.java @@ -0,0 +1,53 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionfilters; + +import java.util.List; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.InsertException; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; +import org.dpppt.backend.sdk.ws.util.ValidationUtils; + +/** + * All keys must be valid Base64 encoded. Non valid Base64 keys are not allowed and are filtered + * out. This filter rejects the whole submitted batch of keys, if any of the keys is not valid + * Base64, as this is a client error. + */ +public class Base64Filter implements KeyInsertionFilter { + + private final ValidationUtils validationUtils; + + public Base64Filter(ValidationUtils validationUtils) { + this.validationUtils = validationUtils; + } + + /** + * Loop through all keys and check for Base64 validity using {@link + * ValidationUtils#isValidBase64Key(String)} and count the number of invalid keys. If the count is + * > 0, a {@link KeyIsNotBase64Exception} is thrown which results in a client error. + */ + @Override + public List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) + throws InsertException { + + var numberOfInvalidKeys = + content.stream().filter(key -> !validationUtils.isValidBase64Key(key.getKeyData())).count(); + + if (numberOfInvalidKeys > 0) { + throw new KeyIsNotBase64Exception(); + } + return content; + } + + public class KeyIsNotBase64Exception extends InsertException { + + /** */ + private static final long serialVersionUID = -918099046973553472L; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeyInsertionFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeyInsertionFilter.java new file mode 100644 index 00000000..7e385884 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeyInsertionFilter.java @@ -0,0 +1,36 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionfilters; + +import java.util.List; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.InsertException; +import org.dpppt.backend.sdk.ws.insertmanager.InsertManager; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; + +/** Interface for filters than can be configured in the {@link InsertManager} */ +public interface KeyInsertionFilter { + + /** + * The {@link InsertManager} goes through all configured filters and calls them with a list of + * {@link GaenKey} where the filters are applied before inserting into the database. + * + * @param now current timestamp + * @param content the list of new gaen keys for insertion + * @param osType the os type of the client which uploaded the keys + * @param osVersion the os version of the client which uploaded the keys + * @param appVersion the app version of the client which uploaded the keys + * @param principal the authorization context which belongs to the uploaded keys. Depending on the + * configured system, this could be a JWT token for example. + * @return + * @throws InsertException + */ + public List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) + throws InsertException; +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeysMatchingJWTFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeysMatchingJWTFilter.java new file mode 100644 index 00000000..a831f6f2 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeysMatchingJWTFilter.java @@ -0,0 +1,74 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionfilters; + +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.model.gaen.GaenUnit; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; +import org.dpppt.backend.sdk.ws.security.ValidateRequest; +import org.dpppt.backend.sdk.ws.security.ValidateRequest.ClaimIsBeforeOnsetException; +import org.dpppt.backend.sdk.ws.security.ValidateRequest.InvalidDateException; +import org.dpppt.backend.sdk.ws.util.ValidationUtils; +import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateClaimIsWrong; +import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateIsInvalid; + +/** + * This filter compares the supplied keys with information found in the JWT token. During the + * `exposed` request, the onset date, which will be set by the health authority and inserted as a + * claim into the JWT is the lower bound for allowed key dates. For the `exposednextday` the JWT + * contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual + * supplied key. + */ +public class KeysMatchingJWTFilter implements KeyInsertionFilter { + + private final ValidateRequest validateRequest; + private final ValidationUtils validationUtils; + + public KeysMatchingJWTFilter(ValidateRequest validateRequest, ValidationUtils utils) { + this.validateRequest = validateRequest; + this.validationUtils = utils; + } + + @Override + public List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) { + return content.stream() + .filter( + key -> { + try { + validationUtils.checkForDelayedKeyDateClaim(principal, key); + var delayedKeyDate = + UTCInstant.of(key.getRollingStartNumber(), GaenUnit.TenMinutes); + return isValidDelayedKeyDate(now, delayedKeyDate); + } catch (DelayedKeyDateClaimIsWrong ex) { + return isValidKeyDate(key, principal, now); + } + }) + .collect(Collectors.toList()); + } + + private boolean isValidKeyDate(GaenKey key, Object principal, UTCInstant now) { + try { + validateRequest.validateKeyDate(now, principal, key); + return true; + } catch (InvalidDateException | ClaimIsBeforeOnsetException es) { + return false; + } + } + + private boolean isValidDelayedKeyDate(UTCInstant now, UTCInstant delayedKeyDate) { + try { + validationUtils.validateDelayedKeyDate(now, delayedKeyDate); + return true; + } catch (DelayedKeyDateIsInvalid ex) { + return false; + } + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/NonFakeKeysFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/NonFakeKeysFilter.java new file mode 100644 index 00000000..063f3977 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/NonFakeKeysFilter.java @@ -0,0 +1,30 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionfilters; + +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; + +/** + * Filters out fake keys from fake upload requests. Only Non-Fake keys are inserted into the + * database. + */ +public class NonFakeKeysFilter implements KeyInsertionFilter { + + /** + * Loops through the list of given keys and checks the fake flag. Only return keys that have fake + * flag set to 0 + */ + @Override + public List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) { + return content.stream().filter(key -> key.getFake().equals(0)).collect(Collectors.toList()); + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberAfterDayAfterTomorrowFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberAfterDayAfterTomorrowFilter.java new file mode 100644 index 00000000..c45af883 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberAfterDayAfterTomorrowFilter.java @@ -0,0 +1,38 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionfilters; + +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.model.gaen.GaenUnit; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; + +/** + * Checks if a key has rolling start number after the day after tomorrow. If so, the key is filtered + * out, as this is not allowed by the system to insert keys too far in the future. + */ +public class RollingStartNumberAfterDayAfterTomorrowFilter implements KeyInsertionFilter { + + /** + * Loops through all the keys and converts the rolling start number to a timstamp. The it is + * checked if the timestamp is before now + 2 days. + */ + @Override + public List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) { + return content.stream() + .filter( + key -> { + var rollingStartNumberInstant = + UTCInstant.of(key.getRollingStartNumber(), GaenUnit.TenMinutes); + return rollingStartNumberInstant.isBeforeDateOf(now.plusDays(2)); + }) + .collect(Collectors.toList()); + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberInRetentionPeriodFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberInRetentionPeriodFilter.java new file mode 100644 index 00000000..2d5f52bc --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberInRetentionPeriodFilter.java @@ -0,0 +1,45 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionfilters; + +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.model.gaen.GaenUnit; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; +import org.dpppt.backend.sdk.ws.util.ValidationUtils; + +/** + * Checks if a key is in the configured retention period. If a key is before the retention period it + * is filtered out, as it will not be relevant for the system anymore. + */ +public class RollingStartNumberInRetentionPeriodFilter implements KeyInsertionFilter { + + private final ValidationUtils validationUtils; + + public RollingStartNumberInRetentionPeriodFilter(ValidationUtils validationUtils) { + this.validationUtils = validationUtils; + } + + /** + * Loops through all the keys and converts the rolling start number to a timestamp. Using {@link + * ValidationUtils#isBeforeRetention(UTCInstant, UTCInstant)} only keys are accepted that are not + * before the retention period. Keys before the retention period are filtered out. + */ + @Override + public List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) { + return content.stream() + .filter( + key -> { + var timestamp = UTCInstant.of(key.getRollingStartNumber(), GaenUnit.TenMinutes); + return !validationUtils.isBeforeRetention(timestamp, now); + }) + .collect(Collectors.toList()); + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/ValidRollingPeriodFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/ValidRollingPeriodFilter.java new file mode 100644 index 00000000..5b4610cc --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/ValidRollingPeriodFilter.java @@ -0,0 +1,31 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionfilters; + +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; + +/** + * This filter checks for valid rolling period. The rolling period must always be in [1..144], + * otherwise the key is not valid and is filtered out. See EN documentation + */ +public class ValidRollingPeriodFilter implements KeyInsertionFilter { + + /** Loop through given keys and filter out keys which have rolling period < 1 or > 144. */ + @Override + public List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) { + return content.stream() + .filter(key -> key.getRollingPeriod() >= 1 && key.getRollingPeriod() <= 144) + .collect(Collectors.toList()); + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/IOSLegacyProblemRPLT144Modifier.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/IOSLegacyProblemRPLT144Modifier.java new file mode 100644 index 00000000..9d255562 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/IOSLegacyProblemRPLT144Modifier.java @@ -0,0 +1,31 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier; + +import java.util.List; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; + +/** + * This key modifier makes sure, that rolling period is always set to 144. Default value according + * to EN is 144, so just set it to that. This allows to check for the Google-TEKs also on iOS. + * Because the Rolling Proximity Identifier is based on the TEK and the unix epoch, this should + * work. The only downside is that iOS will not be able to optimize verification of the TEKs, + * because it will have to consider each TEK for a whole day. + */ +public class IOSLegacyProblemRPLT144Modifier implements KeyInsertionModifier { + + @Override + public List modify( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) { + for (GaenKey key : content) { + key.setRollingPeriod(144); + } + return content; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/KeyInsertionModifier.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/KeyInsertionModifier.java new file mode 100644 index 00000000..e83c1eb7 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/KeyInsertionModifier.java @@ -0,0 +1,36 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier; + +import java.util.List; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.InsertException; +import org.dpppt.backend.sdk.ws.insertmanager.InsertManager; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; + +/** Interface for key modifiers than can be configured in the {@link InsertManager} */ +public interface KeyInsertionModifier { + + /** + * The {@link InsertManager} goes through all configured key modifiers and calls them with a list + * of {@link GaenKey} where the modifieres are applied before inserting into the database. + * + * @param now current timestamp + * @param content the list of new gaen keys for modification + * @param osType the os type of the client which uploaded the keys + * @param osVersion the os version of the client which uploaded the keys + * @param appVersion the app version of the client which uploaded the keys + * @param principal the authorization context which belongs to the uploaded keys. Depending on the + * configured system, this could be a JWT token for example. + * @return + * @throws InsertException + */ + public List modify( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) + throws InsertException; +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/OldAndroid0RPModifier.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/OldAndroid0RPModifier.java new file mode 100644 index 00000000..731711b9 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/OldAndroid0RPModifier.java @@ -0,0 +1,46 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier; + +import java.util.List; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Some early builds of Google's Exposure Notification API returned TEKs with rolling period set to + * '0'. According to the specification, this is invalid and will cause both Android and iOS to + * drop/ignore the key. To mitigate ignoring TEKs from these builds altogether, the rolling period + * is increased to '144' (one full day). This should not happen anymore and can be removed in the + * near future. Until then we are going to log whenever this happens to be able to monitor this + * problem. + */ +public class OldAndroid0RPModifier implements KeyInsertionModifier { + + private static final Logger logger = LoggerFactory.getLogger(OldAndroid0RPModifier.class); + + /** + * Loop through all the given keys and check if the rolling period is equal to 0. If so, set to + * 144. In case a key with rolling period 0 is received from an iOS client, an error log is + * printed. + */ + @Override + public List modify( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) { + for (GaenKey gaenKey : content) { + if (gaenKey.getRollingPeriod().equals(0)) { + if (osType.equals(OSType.IOS)) { + logger.error("We got a rollingPeriod of 0 ({},{},{})", osType, osVersion, appVersion); + } + gaenKey.setRollingPeriod(144); + } + } + return content; + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/security/gaen/JWTValidateRequest.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/security/gaen/JWTValidateRequest.java index 68d68217..17aadccd 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/security/gaen/JWTValidateRequest.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/security/gaen/JWTValidateRequest.java @@ -18,11 +18,8 @@ import org.springframework.security.oauth2.jwt.Jwt; public class JWTValidateRequest implements ValidateRequest { - private final ValidationUtils validationUtils; - public JWTValidateRequest(ValidationUtils validationUtils) { - this.validationUtils = validationUtils; - } + public JWTValidateRequest(ValidationUtils validationUtils) {} @Override public boolean isValid(Object authObject) throws WrongScopeException { @@ -39,7 +36,7 @@ public boolean isValid(Object authObject) throws WrongScopeException { @Override public long validateKeyDate(UTCInstant now, Object authObject, Object others) - throws ClaimIsBeforeOnsetException, InvalidDateException { + throws ClaimIsBeforeOnsetException { if (authObject instanceof Jwt) { Jwt token = (Jwt) authObject; var jwtKeyDate = UTCInstant.parseDate(token.getClaim("onset")); @@ -48,8 +45,6 @@ public long validateKeyDate(UTCInstant now, Object authObject, Object others) var keyDate = UTCInstant.of(request.getRollingStartNumber(), GaenUnit.TenMinutes); if (keyDate.isBeforeEpochMillisOf(jwtKeyDate)) { throw new ClaimIsBeforeOnsetException(); - } else if (!validationUtils.isDateInRange(keyDate, now)) { - throw new InvalidDateException(); } jwtKeyDate = keyDate; } diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/DPPPTControllerTest.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/DPPPTControllerTest.java index d94cacc7..7244219c 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/DPPPTControllerTest.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/DPPPTControllerTest.java @@ -306,17 +306,17 @@ public void cannotUseKeyDateBeforeOnset() throws Exception { createToken( UTCInstant.now().plusMinutes(5), UTCInstant.now().getLocalDate().format(DateTimeFormatter.ISO_DATE)); - - mockMvc - .perform( - post("/v1/exposed") - .contentType(MediaType.APPLICATION_JSON) - .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") - .content(json(exposeeRequest))) - .andExpect(status().is(400)) - .andReturn() - .getResponse(); + MockHttpServletResponse response = + mockMvc + .perform( + post("/v1/exposed") + .contentType(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + token) + .header("User-Agent", "MockMVC") + .content(json(exposeeRequest))) + .andExpect(status().is(400)) + .andReturn() + .getResponse(); } @Test diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTest.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTest.java index fd3f299e..c60f853a 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTest.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTest.java @@ -133,13 +133,13 @@ private void testNKeys(UTCInstant now, int n, boolean shouldSucceed) throws Exce var gaenKey1 = new GaenKey(); gaenKey1.setRollingStartNumber((int) now.atStartOfDay().minusDays(1).get10MinutesSince1970()); gaenKey1.setKeyData(Base64.getEncoder().encodeToString("testKey32Bytes01".getBytes("UTF-8"))); - gaenKey1.setRollingPeriod(0); + gaenKey1.setRollingPeriod(144); gaenKey1.setFake(0); gaenKey1.setTransmissionRiskLevel(0); var gaenKey2 = new GaenKey(); gaenKey2.setRollingStartNumber((int) now.atStartOfDay().minusDays(1).get10MinutesSince1970()); gaenKey2.setKeyData(Base64.getEncoder().encodeToString("testKey32Bytes02".getBytes("UTF-8"))); - gaenKey2.setRollingPeriod(0); + gaenKey2.setRollingPeriod(144); gaenKey2.setFake(0); gaenKey2.setTransmissionRiskLevel(0); List exposedKeys = new ArrayList<>(); @@ -164,7 +164,7 @@ private void testNKeys(UTCInstant now, int n, boolean shouldSucceed) throws Exce post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))); MvcResult response; @@ -181,7 +181,7 @@ private void testNKeys(UTCInstant now, int n, boolean shouldSucceed) throws Exce post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(status().is(401)) .andExpect(request().asyncNotStarted()) @@ -247,7 +247,7 @@ public void testAllKeysWrongButStill200() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -259,7 +259,7 @@ public void testAllKeysWrongButStill200() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(status().is(401)) .andExpect(request().asyncNotStarted()) @@ -317,7 +317,7 @@ public void testSecurityHeaders() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "MockMVC")) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); @@ -368,7 +368,7 @@ public void testUploadWithNegativeRollingPeriodFails() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -439,7 +439,7 @@ public void testMultipleKeyUploadFake() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -450,7 +450,7 @@ public void testMultipleKeyUploadFake() throws Exception { post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -506,7 +506,7 @@ public void testMultipleKeyUploadFakeAllKeysNeedToBeFake() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -517,7 +517,7 @@ public void testMultipleKeyUploadFakeAllKeysNeedToBeFake() throws Exception { post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -573,7 +573,7 @@ public void testMultipleKeyUploadFakeIfJWTNotFakeAllKeysCanBeFake() throws Excep post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -584,7 +584,7 @@ public void testMultipleKeyUploadFakeIfJWTNotFakeAllKeysCanBeFake() throws Excep post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -607,7 +607,7 @@ public void testMultipleKeyNonEmptyUpload() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(request().asyncNotStarted()) .andExpect(status().is(400)) @@ -619,7 +619,7 @@ public void testMultipleKeyNonEmptyUpload() throws Exception { post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -643,7 +643,7 @@ public void testMultipleKeyNonNullUpload() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(request().asyncNotStarted()) .andExpect(status().is(400)) @@ -655,7 +655,7 @@ public void testMultipleKeyNonNullUpload() throws Exception { post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -676,7 +676,7 @@ public void keyNeedsToBeBase64() throws Exception { key.setRollingPeriod(144); key.setRollingStartNumber((int) now.get10MinutesSince1970()); key.setTransmissionRiskLevel(1); - key.setFake(1); + key.setFake(0); List keys = new ArrayList<>(); keys.add(key); for (int i = 0; i < 13; i++) { @@ -691,18 +691,16 @@ public void keyNeedsToBeBase64() throws Exception { } exposeeRequest.setGaenKeys(keys); - String token = createToken(true, now.plusMinutes(5)); - MvcResult response = - mockMvc - .perform( - post("/v1/gaen/exposed") - .contentType(MediaType.APPLICATION_JSON) - .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") - .content(json(exposeeRequest))) - .andExpect(request().asyncStarted()) - .andReturn(); - mockMvc.perform(asyncDispatch(response)).andExpect(status().is(400)); + String token = createToken(false, now.plusMinutes(5)); + mockMvc + .perform( + post("/v1/gaen/exposed") + .contentType(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + token) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .content(json(exposeeRequest))) + .andExpect(status().is(400)) + .andReturn(); } @Test @@ -743,7 +741,7 @@ public void testKeyDateBeforeOnsetIsNotInserted() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(exposeeRequest))) .andExpect(request().asyncStarted()) .andExpect(status().is(200)) @@ -793,7 +791,7 @@ public void cannotUseExpiredToken() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(exposeeRequest))) .andExpect(request().asyncNotStarted()) .andExpect(status().is4xxClientError()) @@ -806,10 +804,9 @@ public void cannotUseKeyDateInFuture() throws Exception { var midnight = now.atStartOfDay(); GaenRequest exposeeRequest = new GaenRequest(); - var duration = midnight.plusDays(1).get10MinutesSince1970(); - exposeeRequest.setDelayedKeyDate((int) duration); + exposeeRequest.setDelayedKeyDate((int) midnight.get10MinutesSince1970()); GaenKey key = new GaenKey(); - key.setKeyData(Base64.getEncoder().encodeToString("testKey32Bytes--".getBytes("UTF-8"))); + key.setKeyData(Base64.getEncoder().encodeToString("testKey32Bytes++".getBytes("UTF-8"))); key.setRollingPeriod(144); key.setRollingStartNumber((int) midnight.plusDays(2).get10MinutesSince1970()); key.setTransmissionRiskLevel(1); @@ -820,7 +817,7 @@ public void cannotUseKeyDateInFuture() throws Exception { var tmpKey = new GaenKey(); tmpKey.setRollingStartNumber( (int) Duration.ofMillis(Instant.now().toEpochMilli()).dividedBy(Duration.ofMinutes(10))); - tmpKey.setKeyData(Base64.getEncoder().encodeToString("testKey32Bytes--".getBytes("UTF-8"))); + tmpKey.setKeyData(Base64.getEncoder().encodeToString("testKey32Bytes++".getBytes("UTF-8"))); tmpKey.setRollingPeriod(144); tmpKey.setFake(1); tmpKey.setTransmissionRiskLevel(0); @@ -830,17 +827,16 @@ public void cannotUseKeyDateInFuture() throws Exception { String token = createToken(now.plusMinutes(5)); - mockMvc - .perform( - post("/v1/gaen/exposed") - .contentType(MediaType.APPLICATION_JSON) - .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") - .content(json(exposeeRequest))) - .andExpect(request().asyncStarted()) - .andExpect(status().is(200)) - .andReturn(); - + MvcResult response = + mockMvc + .perform( + post("/v1/gaen/exposed") + .contentType(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + token) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .content(json(exposeeRequest))) + .andExpect(status().is(200)) + .andReturn(); var result = gaenDataService.getSortedExposedForKeyDate( midnight.plusDays(2).getTimestamp(), @@ -867,7 +863,8 @@ public void keyDateNotOlderThan21Days() throws Exception { keys.add(key); for (int i = 0; i < 13; i++) { var tmpKey = new GaenKey(); - tmpKey.setRollingStartNumber((int) now.get10MinutesSince1970()); + tmpKey.setRollingStartNumber( + (int) Duration.ofMillis(Instant.now().toEpochMilli()).dividedBy(Duration.ofMinutes(10))); tmpKey.setKeyData(Base64.getEncoder().encodeToString("testKey32Bytes--".getBytes("UTF-8"))); tmpKey.setRollingPeriod(144); tmpKey.setFake(1); @@ -878,17 +875,17 @@ public void keyDateNotOlderThan21Days() throws Exception { String token = createToken(now.plusMinutes(5), "2020-01-01"); - mockMvc - .perform( - post("/v1/gaen/exposed") - .contentType(MediaType.APPLICATION_JSON) - .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") - .content(json(exposeeRequest))) - .andExpect(request().asyncStarted()) - .andExpect(status().is(200)) - .andReturn(); - + MvcResult response = + mockMvc + .perform( + post("/v1/gaen/exposed") + .contentType(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + token) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .content(json(exposeeRequest))) + .andExpect(request().asyncStarted()) + .andExpect(status().is(200)) + .andReturn(); var result = gaenDataService.getSortedExposedForKeyDate( midnight.minusDays(22).getTimestamp(), @@ -932,14 +929,11 @@ public void cannotUseTokenWithWrongScope() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(exposeeRequest))) - .andExpect(request().asyncStarted()) + .andExpect(status().is(403)) .andReturn(); - mockMvc - .perform(asyncDispatch(response)) - .andExpect(status().is(403)) - .andExpect(content().string("")); + // Also for a 403 response, the token cannot be used a 2nd time response = mockMvc @@ -947,7 +941,7 @@ public void cannotUseTokenWithWrongScope() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(exposeeRequest))) .andExpect(request().asyncNotStarted()) .andExpect(status().is(401)) @@ -984,7 +978,7 @@ public void uploadKeysAndUploadKeyNextDay() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(exposeeRequest))) .andExpect(request().asyncStarted()) .andReturn(); @@ -1016,14 +1010,15 @@ public void uploadKeysAndUploadKeyNextDay() throws Exception { post("/v1/gaen/exposednextday") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtString) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(secondDay))) .andExpect(request().asyncStarted()) .andReturn(); mockMvc.perform(asyncDispatch(responseAsync)).andExpect(status().is(200)); } - @Test + // @Test + // TODO: Is this still a requirement? Currently the key just gets filtered out public void uploadKeysAndUploadKeyNextDayWithNegativeRollingPeriodFails() throws Exception { var now = UTCInstant.now(); var midnight = now.atStartOfDay(); @@ -1052,7 +1047,7 @@ public void uploadKeysAndUploadKeyNextDayWithNegativeRollingPeriodFails() throws post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(exposeeRequest))) .andExpect(request().asyncStarted()) .andReturn(); @@ -1084,7 +1079,7 @@ public void uploadKeysAndUploadKeyNextDayWithNegativeRollingPeriodFails() throws post("/v1/gaen/exposednextday") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtString) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(secondDay))) .andExpect(request().asyncStarted()) .andReturn(); @@ -1124,28 +1119,35 @@ public void delayedKeyDateBoundaryCheck() throws Exception { exposeeRequest.setDelayedKeyDate(delayedKeyDateSent); exposeeRequest.setGaenKeys(keys); String token = createToken(now.plusMinutes(5)); - MvcResult responseAsync = - mockMvc - .perform( - post("/v1/gaen/exposed") - .contentType(MediaType.APPLICATION_JSON) - .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") - .content(json(exposeeRequest))) - .andExpect(request().asyncStarted()) - .andReturn(); if (pass) { + MvcResult responseAsync = + mockMvc + .perform( + post("/v1/gaen/exposed") + .contentType(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + token) + .header( + "User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .content(json(exposeeRequest))) + .andExpect(request().asyncStarted()) + .andReturn(); mockMvc .perform(asyncDispatch(responseAsync)) .andExpect(status().is(200)) .andReturn() .getResponse(); } else { - mockMvc - .perform(asyncDispatch(responseAsync)) - .andExpect(status().is(400)) - .andReturn() - .getResponse(); + MvcResult responseAsync = + mockMvc + .perform( + post("/v1/gaen/exposed") + .contentType(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + token) + .header( + "User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .content(json(exposeeRequest))) + .andExpect(status().is(400)) + .andReturn(); } } } @@ -1180,7 +1182,7 @@ public void testTokenValiditySurpassesMaxJwtValidity() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "MockMVC") + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") .content(json(exposeeRequest))) .andExpect(status().is(401)); } @@ -1203,7 +1205,8 @@ public void testDebugController() throws Exception { MockHttpServletResponse response = mockMvc .perform( - get("/v1/debug/exposed/" + midnight.getTimestamp()).header("User-Agent", "MockMVC")) + get("/v1/debug/exposed/" + midnight.getTimestamp()) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); @@ -1271,7 +1274,7 @@ public void testNonEmptyResponseAnd304() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "MockMVC")) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) .andExpect(status().isOk()) .andReturn() .getResponse(); @@ -1305,7 +1308,7 @@ public void testEtag() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "MockMVC")) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); @@ -1318,7 +1321,7 @@ public void testEtag() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "MockMVC")) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); @@ -1333,7 +1336,7 @@ public void testEtag() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "MockMVC")) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTestNotThreadSafe.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTestNotThreadSafe.java new file mode 100644 index 00000000..4539495a --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTestNotThreadSafe.java @@ -0,0 +1,209 @@ +package org.dpppt.backend.sdk.ws.controller; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.io.ByteArrayInputStream; +import java.security.SecureRandom; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.model.gaen.proto.TemporaryExposureKeyFormat; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.security.signature.ProtoSignature; +import org.junit.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.transaction.annotation.Transactional; + +@ActiveProfiles({"actuator-security"}) +@SpringBootTest( + properties = { + "ws.app.jwt.publickey=classpath://generated_pub.pem", + "logging.level.org.springframework.security=DEBUG", + "ws.exposedlist.releaseBucketDuration=7200000", + "ws.gaen.randomkeysenabled=true", + "ws.app.gaen.delayTodaysKeys=true", + "ws.monitor.prometheus.user=prometheus", + "ws.monitor.prometheus.password=prometheus", + "management.endpoints.enabled-by-default=true", + "management.endpoints.web.exposure.include=*" + }) +@Transactional +@Execution(ExecutionMode.SAME_THREAD) +public class GaenControllerTestNotThreadSafe extends BaseControllerTest { + @Autowired ProtoSignature signer; + + private static final Logger logger = LoggerFactory.getLogger(GaenControllerTest.class); + + @Test + @Transactional + public void zipContainsFiles() throws Exception { + var clockStartingAtMidnight = + Clock.offset(Clock.systemUTC(), UTCInstant.now().getDuration(UTCInstant.today()).negated()); + UTCInstant.setClock(clockStartingAtMidnight); + var now = UTCInstant.now(); + var midnight = now.atStartOfDay(); + + // insert two times 5 keys per day for the last 14 days. the second batch has a + // different received at timestamp. (+6 hours) + insertNKeysPerDayInInterval(14, midnight.minusDays(4), now, now.minusDays(1)); + + insertNKeysPerDayInInterval(14, midnight.minusDays(4), now, now.minusDays(12)); + + // request the keys with date date 1 day ago. no publish until. + MockHttpServletResponse response = + mockMvc + .perform( + get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse(); + + Long publishedUntil = Long.parseLong(response.getHeader("X-PUBLISHED-UNTIL")); + assertTrue(publishedUntil < now.getTimestamp()); + + verifyZipResponse(response, 20); + + // request again the keys with date date 1 day ago. with publish until, so that + // we only get the second batch. + var bucketAfterSecondRelease = + Duration.ofMillis(midnight.getTimestamp()) + .minusDays(1) + .plusHours(12) + .dividedBy(Duration.ofHours(2)) + * 2 + * 60 + * 60 + * 1000; + MockHttpServletResponse responseWithPublishedAfter = + mockMvc + .perform( + get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) + .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .param("publishedafter", Long.toString(bucketAfterSecondRelease))) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse(); + + // we always have 10 + verifyZipResponse(responseWithPublishedAfter, 10); + UTCInstant.resetClock(); + } + + private void verifyZipResponse(MockHttpServletResponse response, int expectKeyCount) + throws Exception { + ByteArrayInputStream baisOuter = new ByteArrayInputStream(response.getContentAsByteArray()); + ZipInputStream zipOuter = new ZipInputStream(baisOuter); + ZipEntry entry = zipOuter.getNextEntry(); + boolean foundData = false; + boolean foundSignature = false; + + byte[] signatureProto = null; + byte[] exportBin = null; + byte[] keyProto = null; + + while (entry != null) { + if (entry.getName().equals("export.bin")) { + foundData = true; + exportBin = zipOuter.readAllBytes(); + keyProto = new byte[exportBin.length - 16]; + System.arraycopy(exportBin, 16, keyProto, 0, keyProto.length); + } + if (entry.getName().equals("export.sig")) { + foundSignature = true; + signatureProto = zipOuter.readAllBytes(); + } + entry = zipOuter.getNextEntry(); + } + + assertTrue(foundData); + assertTrue(foundSignature); + + var list = TemporaryExposureKeyFormat.TEKSignatureList.parseFrom(signatureProto); + var export = TemporaryExposureKeyFormat.TemporaryExposureKeyExport.parseFrom(keyProto); + for (var key : export.getKeysList()) { + assertNotEquals(0, key.getRollingPeriod()); + } + var sig = list.getSignatures(0); + java.security.Signature signatureVerifier = + java.security.Signature.getInstance(sig.getSignatureInfo().getSignatureAlgorithm().trim()); + signatureVerifier.initVerify(signer.getPublicKey()); + + signatureVerifier.update(exportBin); + assertTrue(signatureVerifier.verify(sig.getSignature().toByteArray())); + assertEquals(expectKeyCount, export.getKeysCount()); + } + + private void insertNKeysPerDayInIntervalWithDebugFlag( + int n, UTCInstant start, UTCInstant end, UTCInstant receivedAt, boolean debug) + throws Exception { + var current = start; + Map rollingToCount = new HashMap<>(); + while (current.isBeforeEpochMillisOf(end)) { + List keys = new ArrayList<>(); + SecureRandom random = new SecureRandom(); + int lastRolling = (int) start.get10MinutesSince1970(); + for (int i = 0; i < n; i++) { + GaenKey key = new GaenKey(); + byte[] keyBytes = new byte[16]; + random.nextBytes(keyBytes); + key.setKeyData(Base64.getEncoder().encodeToString(keyBytes)); + key.setRollingPeriod(144); + logger.info("Rolling Start number: " + lastRolling); + key.setRollingStartNumber(lastRolling); + key.setTransmissionRiskLevel(1); + key.setFake(0); + keys.add(key); + + Integer count = rollingToCount.get(lastRolling); + if (count == null) { + count = 0; + } + count = count + 1; + rollingToCount.put(lastRolling, count); + + lastRolling -= Duration.ofDays(1).dividedBy(Duration.ofMinutes(10)); + } + if (debug) { + testGaenDataService.upsertExposeesDebug(keys, receivedAt); + } else { + testGaenDataService.upsertExposees(keys, receivedAt); + } + current = current.plusDays(1); + } + for (Entry entry : rollingToCount.entrySet()) { + logger.info( + "Rolling start number: " + + entry.getKey() + + " -> count: " + + entry.getValue() + + " (received at: " + + receivedAt.toString() + + ")"); + } + } + + private void insertNKeysPerDayInInterval( + int n, UTCInstant start, UTCInstant end, UTCInstant receivedAt) throws Exception { + insertNKeysPerDayInIntervalWithDebugFlag(n, start, end, receivedAt, false); + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManagerTest.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManagerTest.java new file mode 100644 index 00000000..382312f9 --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/insertmanager/InsertManagerTest.java @@ -0,0 +1,133 @@ +package org.dpppt.backend.sdk.ws.insertmanager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.AppenderBase; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier.OldAndroid0RPModifier; +import org.dpppt.backend.sdk.ws.util.ValidationUtils; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +public class InsertManagerTest { + @Test + public void testOSEnumWorks() { + assertEquals("Android", OSType.ANDROID.toString()); + assertEquals("iOS", OSType.IOS.toString()); + } + + @Test + public void emptyListShouldNotFail() { + Object theException = null; + try { + InsertManager manager = + new InsertManager( + new MockDataSource(), + new ValidationUtils(16, Duration.ofDays(14), Duration.ofHours(2).toMillis())); + manager.insertIntoDatabase(new ArrayList<>(), null, null, null); + } catch (Exception ex) { + theException = ex; + } + assertNull(theException); + } + + @Test + public void nullListShouldNotFail() throws Exception { + Object theException = null; + try { + InsertManager manager = + new InsertManager( + new MockDataSource(), + new ValidationUtils(16, Duration.ofDays(14), Duration.ofHours(2).toMillis())); + manager.insertIntoDatabase(null, null, null, null); + } catch (Exception ex) { + theException = ex; + } + assertNull(theException); + } + + @Test + public void wrongHeaderShouldNotFail() throws Exception { + Logger logger = (Logger) LoggerFactory.getLogger(InsertManager.class); + var appender = new TestAppender(); + appender.setContext(logger.getLoggerContext()); + appender.start(); + logger.addAppender(appender); + InsertManager manager = + new InsertManager( + new MockDataSource(), + new ValidationUtils(16, Duration.ofDays(14), Duration.ofHours(2).toMillis())); + var key = + new GaenKey("POSTMAN+POSTMAN+", (int) UTCInstant.now().get10MinutesSince1970(), 144, 0); + try { + manager.insertIntoDatabase(List.of(key), "test", null, UTCInstant.now()); + } catch (RuntimeException ex) { + if (!ex.getMessage().equals("UPSERT_EXPOSEES")) { + throw ex; + } + } + appender.stop(); + for (var event : appender.getLog()) { + assertEquals(Level.ERROR, event.getLevel()); + assertEquals("We received an invalid header, setting default.", event.getMessage()); + } + } + + @Test + public void iosRP0ShouldLog() throws Exception { + Logger logger = (Logger) LoggerFactory.getLogger(OldAndroid0RPModifier.class); + var appender = new TestAppender(); + appender.setContext(logger.getLoggerContext()); + appender.start(); + logger.addAppender(appender); + InsertManager manager = + new InsertManager( + new MockDataSource(), + new ValidationUtils(16, Duration.ofDays(14), Duration.ofHours(2).toMillis())); + manager.addModifier(new OldAndroid0RPModifier()); + var key = new GaenKey("POSTMAN+POSTMAN+", (int) UTCInstant.now().get10MinutesSince1970(), 0, 0); + try { + manager.insertIntoDatabase( + List.of(key), "org.dpppt.testrunner;1.0.0;1;iOS;29", null, UTCInstant.now()); + } catch (RuntimeException ex) { + if (!ex.getMessage().equals("UPSERT_EXPOSEES")) { + throw ex; + } + } + appender.stop(); + assertEquals(1, appender.getLog().size()); + for (var event : appender.getLog()) { + assertEquals(Level.ERROR, event.getLevel()); + assertEquals("We got a rollingPeriod of 0 ({},{},{})", event.getMessage()); + // osType, osVersion, appVersion + var osType = (OSType) event.getArgumentArray()[0]; + var osVersion = (Version) event.getArgumentArray()[1]; + var appVersion = (Version) event.getArgumentArray()[2]; + assertEquals(OSType.IOS, osType); + assertEquals("29.0.0", osVersion.toString()); + assertEquals("1.0.0+1", appVersion.toString()); + } + } + + class TestAppender extends AppenderBase { + private final List log = new ArrayList(); + + @Override + protected void append(ILoggingEvent eventObject) { + log.add(eventObject); + } + + public List getLog() { + return log; + } + } +} From 7c0d208892f8ea88b6e50325da0d103abe307b44 Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Tue, 1 Sep 2020 12:47:49 +0200 Subject: [PATCH 03/10] Remove GaenControllerTestNotThreadSafe and put androidUserAgent string in class-variable --- .../sdk/data/DPPPTDataServiceTest.java | 2 +- .../sdk/ws/controller/GaenControllerTest.java | 74 +++---- .../GaenControllerTestNotThreadSafe.java | 209 ------------------ 3 files changed, 38 insertions(+), 247 deletions(-) delete mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTestNotThreadSafe.java diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-data/src/test/java/org/dpppt/backend/sdk/data/DPPPTDataServiceTest.java b/dpppt-backend-sdk/dpppt-backend-sdk-data/src/test/java/org/dpppt/backend/sdk/data/DPPPTDataServiceTest.java index 0d4df36a..5f001876 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-data/src/test/java/org/dpppt/backend/sdk/data/DPPPTDataServiceTest.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-data/src/test/java/org/dpppt/backend/sdk/data/DPPPTDataServiceTest.java @@ -121,7 +121,7 @@ public void testRedeemUUID() { assertTrue(actual); } - // @Test + @Test @Transactional public void cleanUp() { Exposee expected = new Exposee(); diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTest.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTest.java index c60f853a..e897c87c 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTest.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTest.java @@ -78,6 +78,8 @@ public class GaenControllerTest extends BaseControllerTest { @Autowired KeyVault keyVault; @Autowired GAENDataService gaenDataService; Long releaseBucketDuration = 7200000L; + private static final String androidUserAgent = + "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29"; private static final Logger logger = LoggerFactory.getLogger(GaenControllerTest.class); @@ -164,7 +166,7 @@ private void testNKeys(UTCInstant now, int n, boolean shouldSucceed) throws Exce post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))); MvcResult response; @@ -181,7 +183,7 @@ private void testNKeys(UTCInstant now, int n, boolean shouldSucceed) throws Exce post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(status().is(401)) .andExpect(request().asyncNotStarted()) @@ -247,7 +249,7 @@ public void testAllKeysWrongButStill200() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -259,7 +261,7 @@ public void testAllKeysWrongButStill200() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(status().is(401)) .andExpect(request().asyncNotStarted()) @@ -317,7 +319,7 @@ public void testSecurityHeaders() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) + .header("User-Agent", androidUserAgent)) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); @@ -368,7 +370,7 @@ public void testUploadWithNegativeRollingPeriodFails() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -439,7 +441,7 @@ public void testMultipleKeyUploadFake() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -450,7 +452,7 @@ public void testMultipleKeyUploadFake() throws Exception { post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -506,7 +508,7 @@ public void testMultipleKeyUploadFakeAllKeysNeedToBeFake() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -517,7 +519,7 @@ public void testMultipleKeyUploadFakeAllKeysNeedToBeFake() throws Exception { post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -573,7 +575,7 @@ public void testMultipleKeyUploadFakeIfJWTNotFakeAllKeysCanBeFake() throws Excep post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(request().asyncStarted()) .andReturn(); @@ -584,7 +586,7 @@ public void testMultipleKeyUploadFakeIfJWTNotFakeAllKeysCanBeFake() throws Excep post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -607,7 +609,7 @@ public void testMultipleKeyNonEmptyUpload() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(request().asyncNotStarted()) .andExpect(status().is(400)) @@ -619,7 +621,7 @@ public void testMultipleKeyNonEmptyUpload() throws Exception { post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -643,7 +645,7 @@ public void testMultipleKeyNonNullUpload() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(request().asyncNotStarted()) .andExpect(status().is(400)) @@ -655,7 +657,7 @@ public void testMultipleKeyNonNullUpload() throws Exception { post("/v1/gaen/exposedlist") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtToken) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(requestList))) .andExpect(status().is(401)) .andExpect(content().string("")) @@ -697,7 +699,7 @@ public void keyNeedsToBeBase64() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(status().is(400)) .andReturn(); @@ -741,7 +743,7 @@ public void testKeyDateBeforeOnsetIsNotInserted() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(request().asyncStarted()) .andExpect(status().is(200)) @@ -791,7 +793,7 @@ public void cannotUseExpiredToken() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(request().asyncNotStarted()) .andExpect(status().is4xxClientError()) @@ -833,7 +835,7 @@ public void cannotUseKeyDateInFuture() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(status().is(200)) .andReturn(); @@ -881,7 +883,7 @@ public void keyDateNotOlderThan21Days() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(request().asyncStarted()) .andExpect(status().is(200)) @@ -929,7 +931,7 @@ public void cannotUseTokenWithWrongScope() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(status().is(403)) .andReturn(); @@ -941,7 +943,7 @@ public void cannotUseTokenWithWrongScope() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(request().asyncNotStarted()) .andExpect(status().is(401)) @@ -978,7 +980,7 @@ public void uploadKeysAndUploadKeyNextDay() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(request().asyncStarted()) .andReturn(); @@ -1010,7 +1012,7 @@ public void uploadKeysAndUploadKeyNextDay() throws Exception { post("/v1/gaen/exposednextday") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtString) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(secondDay))) .andExpect(request().asyncStarted()) .andReturn(); @@ -1047,7 +1049,7 @@ public void uploadKeysAndUploadKeyNextDayWithNegativeRollingPeriodFails() throws post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(request().asyncStarted()) .andReturn(); @@ -1079,7 +1081,7 @@ public void uploadKeysAndUploadKeyNextDayWithNegativeRollingPeriodFails() throws post("/v1/gaen/exposednextday") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + jwtString) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(secondDay))) .andExpect(request().asyncStarted()) .andReturn(); @@ -1126,8 +1128,7 @@ public void delayedKeyDateBoundaryCheck() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header( - "User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(request().asyncStarted()) .andReturn(); @@ -1143,8 +1144,7 @@ public void delayedKeyDateBoundaryCheck() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header( - "User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(status().is(400)) .andReturn(); @@ -1182,7 +1182,7 @@ public void testTokenValiditySurpassesMaxJwtValidity() throws Exception { post("/v1/gaen/exposed") .contentType(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer " + token) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") + .header("User-Agent", androidUserAgent) .content(json(exposeeRequest))) .andExpect(status().is(401)); } @@ -1206,7 +1206,7 @@ public void testDebugController() throws Exception { mockMvc .perform( get("/v1/debug/exposed/" + midnight.getTimestamp()) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) + .header("User-Agent", androidUserAgent)) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); @@ -1274,7 +1274,7 @@ public void testNonEmptyResponseAnd304() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) + .header("User-Agent", androidUserAgent)) .andExpect(status().isOk()) .andReturn() .getResponse(); @@ -1308,7 +1308,7 @@ public void testEtag() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) + .header("User-Agent", androidUserAgent)) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); @@ -1321,7 +1321,7 @@ public void testEtag() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) + .header("User-Agent", androidUserAgent)) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); @@ -1336,7 +1336,7 @@ public void testEtag() throws Exception { mockMvc .perform( get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) + .header("User-Agent", androidUserAgent)) .andExpect(status().is2xxSuccessful()) .andReturn() .getResponse(); diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTestNotThreadSafe.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTestNotThreadSafe.java deleted file mode 100644 index 4539495a..00000000 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/test/java/org/dpppt/backend/sdk/ws/controller/GaenControllerTestNotThreadSafe.java +++ /dev/null @@ -1,209 +0,0 @@ -package org.dpppt.backend.sdk.ws.controller; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -import java.io.ByteArrayInputStream; -import java.security.SecureRandom; -import java.time.Clock; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Base64; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import org.dpppt.backend.sdk.model.gaen.GaenKey; -import org.dpppt.backend.sdk.model.gaen.proto.TemporaryExposureKeyFormat; -import org.dpppt.backend.sdk.utils.UTCInstant; -import org.dpppt.backend.sdk.ws.security.signature.ProtoSignature; -import org.junit.Test; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.transaction.annotation.Transactional; - -@ActiveProfiles({"actuator-security"}) -@SpringBootTest( - properties = { - "ws.app.jwt.publickey=classpath://generated_pub.pem", - "logging.level.org.springframework.security=DEBUG", - "ws.exposedlist.releaseBucketDuration=7200000", - "ws.gaen.randomkeysenabled=true", - "ws.app.gaen.delayTodaysKeys=true", - "ws.monitor.prometheus.user=prometheus", - "ws.monitor.prometheus.password=prometheus", - "management.endpoints.enabled-by-default=true", - "management.endpoints.web.exposure.include=*" - }) -@Transactional -@Execution(ExecutionMode.SAME_THREAD) -public class GaenControllerTestNotThreadSafe extends BaseControllerTest { - @Autowired ProtoSignature signer; - - private static final Logger logger = LoggerFactory.getLogger(GaenControllerTest.class); - - @Test - @Transactional - public void zipContainsFiles() throws Exception { - var clockStartingAtMidnight = - Clock.offset(Clock.systemUTC(), UTCInstant.now().getDuration(UTCInstant.today()).negated()); - UTCInstant.setClock(clockStartingAtMidnight); - var now = UTCInstant.now(); - var midnight = now.atStartOfDay(); - - // insert two times 5 keys per day for the last 14 days. the second batch has a - // different received at timestamp. (+6 hours) - insertNKeysPerDayInInterval(14, midnight.minusDays(4), now, now.minusDays(1)); - - insertNKeysPerDayInInterval(14, midnight.minusDays(4), now, now.minusDays(12)); - - // request the keys with date date 1 day ago. no publish until. - MockHttpServletResponse response = - mockMvc - .perform( - get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29")) - .andExpect(status().is2xxSuccessful()) - .andReturn() - .getResponse(); - - Long publishedUntil = Long.parseLong(response.getHeader("X-PUBLISHED-UNTIL")); - assertTrue(publishedUntil < now.getTimestamp()); - - verifyZipResponse(response, 20); - - // request again the keys with date date 1 day ago. with publish until, so that - // we only get the second batch. - var bucketAfterSecondRelease = - Duration.ofMillis(midnight.getTimestamp()) - .minusDays(1) - .plusHours(12) - .dividedBy(Duration.ofHours(2)) - * 2 - * 60 - * 60 - * 1000; - MockHttpServletResponse responseWithPublishedAfter = - mockMvc - .perform( - get("/v1/gaen/exposed/" + midnight.minusDays(8).getTimestamp()) - .header("User-Agent", "ch.admin.bag.dp3t.dev;1.0.7;1595591959493;Android;29") - .param("publishedafter", Long.toString(bucketAfterSecondRelease))) - .andExpect(status().is2xxSuccessful()) - .andReturn() - .getResponse(); - - // we always have 10 - verifyZipResponse(responseWithPublishedAfter, 10); - UTCInstant.resetClock(); - } - - private void verifyZipResponse(MockHttpServletResponse response, int expectKeyCount) - throws Exception { - ByteArrayInputStream baisOuter = new ByteArrayInputStream(response.getContentAsByteArray()); - ZipInputStream zipOuter = new ZipInputStream(baisOuter); - ZipEntry entry = zipOuter.getNextEntry(); - boolean foundData = false; - boolean foundSignature = false; - - byte[] signatureProto = null; - byte[] exportBin = null; - byte[] keyProto = null; - - while (entry != null) { - if (entry.getName().equals("export.bin")) { - foundData = true; - exportBin = zipOuter.readAllBytes(); - keyProto = new byte[exportBin.length - 16]; - System.arraycopy(exportBin, 16, keyProto, 0, keyProto.length); - } - if (entry.getName().equals("export.sig")) { - foundSignature = true; - signatureProto = zipOuter.readAllBytes(); - } - entry = zipOuter.getNextEntry(); - } - - assertTrue(foundData); - assertTrue(foundSignature); - - var list = TemporaryExposureKeyFormat.TEKSignatureList.parseFrom(signatureProto); - var export = TemporaryExposureKeyFormat.TemporaryExposureKeyExport.parseFrom(keyProto); - for (var key : export.getKeysList()) { - assertNotEquals(0, key.getRollingPeriod()); - } - var sig = list.getSignatures(0); - java.security.Signature signatureVerifier = - java.security.Signature.getInstance(sig.getSignatureInfo().getSignatureAlgorithm().trim()); - signatureVerifier.initVerify(signer.getPublicKey()); - - signatureVerifier.update(exportBin); - assertTrue(signatureVerifier.verify(sig.getSignature().toByteArray())); - assertEquals(expectKeyCount, export.getKeysCount()); - } - - private void insertNKeysPerDayInIntervalWithDebugFlag( - int n, UTCInstant start, UTCInstant end, UTCInstant receivedAt, boolean debug) - throws Exception { - var current = start; - Map rollingToCount = new HashMap<>(); - while (current.isBeforeEpochMillisOf(end)) { - List keys = new ArrayList<>(); - SecureRandom random = new SecureRandom(); - int lastRolling = (int) start.get10MinutesSince1970(); - for (int i = 0; i < n; i++) { - GaenKey key = new GaenKey(); - byte[] keyBytes = new byte[16]; - random.nextBytes(keyBytes); - key.setKeyData(Base64.getEncoder().encodeToString(keyBytes)); - key.setRollingPeriod(144); - logger.info("Rolling Start number: " + lastRolling); - key.setRollingStartNumber(lastRolling); - key.setTransmissionRiskLevel(1); - key.setFake(0); - keys.add(key); - - Integer count = rollingToCount.get(lastRolling); - if (count == null) { - count = 0; - } - count = count + 1; - rollingToCount.put(lastRolling, count); - - lastRolling -= Duration.ofDays(1).dividedBy(Duration.ofMinutes(10)); - } - if (debug) { - testGaenDataService.upsertExposeesDebug(keys, receivedAt); - } else { - testGaenDataService.upsertExposees(keys, receivedAt); - } - current = current.plusDays(1); - } - for (Entry entry : rollingToCount.entrySet()) { - logger.info( - "Rolling start number: " - + entry.getKey() - + " -> count: " - + entry.getValue() - + " (received at: " - + receivedAt.toString() - + ")"); - } - } - - private void insertNKeysPerDayInInterval( - int n, UTCInstant start, UTCInstant end, UTCInstant receivedAt) throws Exception { - insertNKeysPerDayInIntervalWithDebugFlag(n, start, end, receivedAt, false); - } -} From 47b888fe05def9dee2002dd50bd396758423603d Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Tue, 1 Sep 2020 12:52:07 +0200 Subject: [PATCH 04/10] remove README.md from documentation/filter-documentation --- .../sdk/ws/insertmanager/README-orig.md | 134 ----------------- .../backend/sdk/ws/insertmanager/README.md | 140 +++++++++++++----- 2 files changed, 100 insertions(+), 174 deletions(-) delete mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README-orig.md diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README-orig.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README-orig.md deleted file mode 100644 index 1803416b..00000000 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README-orig.md +++ /dev/null @@ -1,134 +0,0 @@ -# Insert-Manager - -## Idea -The Insert-Manager was introduced to reduce logic in controllers. The idea is to provide a second abstraction layer next to the `DataServices` to provide for possible generic validation and normalization. For this there are two mechanisems: modifiers and filters - -The Insert-Manager holds a list of `KeyInsertionFilter`, which provide some code, to either filter for invalid data data. Each filter can decide to either skip respective keys, or throw a `InsertException`. Throwing an exception aborts the current insert request, and throws to the controller. Inside the controller the exception can be mapped to a respective error message and http status code. - -Additionally the Insert-Manager can be configured to hold a list of `KeyInsertModifier`. Modifiers are can be used to modify incoming keys before inserting into the database. (for example to fix buggy clients) - -Encapsulating the logic into smaller pieces of code, should allow for easier and better reviews of the respective filters and modifiers. Further, for each filter or modifier an extensive documentation can be provided, without cluttering the code with too many comments. - -## Valid Keys -A valid key is defined as follows: -- Base64 Encoded key -- Non Fake -- Rolling Period in [1..144] -- Rolling start number inside the configured retention period -- Rolling start number not too far in the future, more precisely not after the day after tomorrow at time of insertion -- Key date must honor the onset date which is given by the healt authority - - -## KeyInsertionFilter Interface -The `KeyInsertionFilter` interface has the following signature: - -```java -public interface KeyInsertionFilter { - List filter( - UTCInstant now, - List content, - OSType osType, - Version osVersion, - Version appVersion, - Object principal) - throws InsertException; -} -``` - -It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. - -## KeyInsertionModifier Interface -The `KeyInsertionModifier` interface has the following signature: - -```java -public interface KeyInsertionModifier { - List modify( - UTCInstant now, - List content, - OSType osType, - Version osVersion, - Version appVersion, - Object principal) - throws InsertException; -} -``` - -It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. - - -## InsertException - -An `InsertException` can be thrown inside an implementation of the `InsertionFilter` interface to mark an insert as "unrecoverable" and abort it. Such "unrecoverable" states might be patterns in the uploaded model, which allows for packet sniffing and/or information gathering. - -## Default Filters - -Looking at the `WSBaseConfig`, we can see that during construction of the `InsertManager` bean, a set of default filters are added: - -```java -public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfigurer { - - // ... - - @Bean - public InsertManager insertManager() { - var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); - manager.addFilter(new Base64Filter(gaenValidationUtils())); - manager.addFilter(new KeysMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils())); - manager.addFilter(new RollingStartNumberAfterDayAfterTomorrowFilter()); - manager.addFilter(new RollingStartNumberInRetentionPeriodFilter(gaenValidationUtils())); - manager.addFilter(new NonFakeKeysFilter()); - manager.addFilter(new ValidRollingPeriodFilter()); - return manager; - } -} -``` - -- `Base64Filter` - > This filter validates that the key actually is a correctly encoded base64 string. Since we are using 16 bytes of key data, those can be represented with exactly 24 characters. The validation of the length is already done during model validation and is assumed to be correct when reaching the filter. This filter _throws_ a `KeyIsNotBase64Exception` if any of the keys is wrongly encoded. Every key submitted _MUST_ have correct base64 encoding -- `KeysMatchingJWTFilter`: - > This filter compares the supplied keys with information found in the JWT token. During the `exposed` request, the onset date, which will be set by the health authority and inserted as a claim into the JWT is the lower bound for allowed key dates. For the `exposednextday` the JWT contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual supplied key. -- `RollingStartNumberAfterDayAfterTomorrowFilter`: - > Representing the maximum allowed time skew. Any key which is further in the future as the day after tomorrow is considered to be _maliciously_ or faulty (for example because of wrong date time settings) uploaded and is hence filtered out. -- `RollingStartNumberInRetentionPeriodFilter`: - > Only keys with key date in the configured retention period are inserted into the datbase. Any key which was valid earlier than `RetentionPeriod` is considered to be outdated and not saved in the database. The key would be removed during the next database clean anyways. -- `NonFakeKeysFilter` - > Only keys that are non-fake are inserted into the database, more precicely keys that have the fake flag set to `0`. -- `ValidRollingPeriodFilter`: - > The `RollingPeriod` represents the 10 minutes interval of the key's validity. Negative numbers are not possible, hence any key having a negative rolling period is considered to be _maliciously_ uploaded. Further, according to [Apple/Googles documentation](https://github.com/google/exposure-notifications-server/blob/main/docs/server_functional_requirements.md) values must be in [1..144] - - -## Additonal Modifiers -- `IOSLegacyProblemRPLT144FModifier` - > This modifier makes sure, that rolling period is always set to 144. Default value according to EN is 144, so just set it to that. This allows to check for the Google-TEKs also on iOS. Because the Rolling Proximity Identifier is based on the TEK and the unix epoch, this should work. The only downside is that iOS will not be able to optimize verification of the TEKs, because it will have to consider each TEK for a whole day. -- `OldAndroid0RPModifier`: - > Some early builds of Google's Exposure Notification API returned TEKs with rolling period set to '0'. According to the specification, this is invalid and will cause both Android and iOS to drop/ignore the key. To mitigate ignoring TEKs from these builds alltogether, the rolling period is increased to '144' (one full day). This should not happen anymore and can be removed in the near future. Until then we are going to log whenever this happens to be able to monitor this problem. - - - -## Configuration -During construction, instances of `GAENDataService` and `ValidationUtils` are needed. Further, any filter or modifier can be added to the list with `addFilter(KeyInsertionFilter filter)` or `addModifier(KeyInsertionModifier)`. Ideally, this happens inside the [`WSBaseConfig`](../config/WSBaseConfig.java), where default filters are added right after constructing the `InsertManager`. - -To allow for conditional `KeyInsertionFilters` or `KeyInsertionModifiers` refer to the following snippet: - -```java -public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfigurer { - - // ... - - @ConditionalOnProperty( - value = "ws.app.gaen.insertmanager.iosrplt144modifier", - havingValue = "true", - matchIfMissing = false) - @Bean - public IOSLegacyProblemRPLT144Modifier iosLegacyProblemRPLT144(InsertManager manager) { - var iosModifier = new IOSLegacyProblemRPLT144Modifier(); - manager.addModifier(iosModifier); - return iosModifier; - } -} -``` - -This looks for a property, either supplied via a `application.properties` file, or via `java` arguments (e.g. `java -D ws.app.gaen.insertmanager.iosrplt144modifier`) and constructs and inserts the respective modifier bean into the modifier chain. For further `SpringBoot` `Conditional` annotations have a look at ["Spring Boot Conditionals"](https://reflectoring.io/spring-boot-conditionals/) - -Encapsulating the logic into smaller pieces of code allows for easier and better reviews of the respective filters. -Further, for each filter an extensive documentation can be provided, without cluttering the code with too many comments. diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md index 6fe55b2e..1803416b 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md @@ -1,43 +1,61 @@ # Insert-Manager ## Idea -The Insert-Manager was introduced to reduce logic in controllers. The idea is to provide a second abstraction layer next to the `DataServices` to provide for possible generic validation and normalization. The Insert-Manager holds a list of `InsertionFilter`, which provide some code, to either filter for invalid data or alter incoming data. Each filter can decide to either skip respective keys, or throw a `InsertException`. Throwing an exception aborts the current insert request, and throws to the controller. Inside the controller the exception can be mapped to a respective error message and http status code. +The Insert-Manager was introduced to reduce logic in controllers. The idea is to provide a second abstraction layer next to the `DataServices` to provide for possible generic validation and normalization. For this there are two mechanisems: modifiers and filters -The current default only handles `KeyIsNotBase64Exception` and ignores all other exceptions (since there are none). +The Insert-Manager holds a list of `KeyInsertionFilter`, which provide some code, to either filter for invalid data data. Each filter can decide to either skip respective keys, or throw a `InsertException`. Throwing an exception aborts the current insert request, and throws to the controller. Inside the controller the exception can be mapped to a respective error message and http status code. -During construction, instances of `GAENDataService` and `ValidationUtils` are needed. Further, any filter can be added to the list with `addFilter(InsertionFilter filter)`. Ideally, this happens inside the [`WSBaseConfig`](../config/WSBaseConfig.java), where default filters are added right after constructing the `InsertManager`. To allow for conditional `InsertionFilters` refer to the following snippet: +Additionally the Insert-Manager can be configured to hold a list of `KeyInsertModifier`. Modifiers are can be used to modify incoming keys before inserting into the database. (for example to fix buggy clients) -```java -public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfigurer { +Encapsulating the logic into smaller pieces of code, should allow for easier and better reviews of the respective filters and modifiers. Further, for each filter or modifier an extensive documentation can be provided, without cluttering the code with too many comments. -// ... +## Valid Keys +A valid key is defined as follows: +- Base64 Encoded key +- Non Fake +- Rolling Period in [1..144] +- Rolling start number inside the configured retention period +- Rolling start number not too far in the future, more precisely not after the day after tomorrow at time of insertion +- Key date must honor the onset date which is given by the healt authority - @ConditionalOnProperty( - value = "ws.app.gaen.insertmanager.iosrplt144modifier", - havingValue = "true", - matchIfMissing = false) - @Bean - public IOSLegacyProblemRPLT144Modifier iosLegacyProblemRPLT144(InsertManager manager) { - var iosModifier = new IOSLegacyProblemRPLT144Modifier(); - manager.addModifier(iosModifier); - return iosModifier; - } + +## KeyInsertionFilter Interface +The `KeyInsertionFilter` interface has the following signature: + +```java +public interface KeyInsertionFilter { + List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) + throws InsertException; } ``` -This looks for a property, either supplied via a `application.properties` file, or via `java` arguments (e.g. `java -D w.app.gaen.ioslegacy`) and constructs and inserts the respective filter bean into the filter chain. For further `SpringBoot` `Conditional` annotations have a look at ["Spring Boot Conditionals"](https://reflectoring.io/spring-boot-conditionals/) - -Encapsulating the logic into smaller pieces of code, should allow for easier and better reviews of the respective filters. Further, for each filter an extensive documentation can be provided, without cluttering the code with too many comments. +It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. -## InsertionFilter Interface -The `InsertionFilter` interface has the following signature: +## KeyInsertionModifier Interface +The `KeyInsertionModifier` interface has the following signature: ```java -public List filter(UTCInstant now, List content, OSType osType, Version osVersion, Version appVersion, Object principal) throws InsertException; +public interface KeyInsertionModifier { + List modify( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) + throws InsertException; +} ``` It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. + ## InsertException An `InsertException` can be thrown inside an implementation of the `InsertionFilter` interface to mark an insert as "unrecoverable" and abort it. Such "unrecoverable" states might be patterns in the uploaded model, which allows for packet sniffing and/or information gathering. @@ -47,28 +65,70 @@ An `InsertException` can be thrown inside an implementation of the `InsertionFil Looking at the `WSBaseConfig`, we can see that during construction of the `InsertManager` bean, a set of default filters are added: ```java -@Bean -public InsertManager insertManager() { +public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfigurer { + + // ... + + @Bean + public InsertManager insertManager() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); - manager.addFilter(new NoBase64Filter(gaenValidationUtils())); - manager.addFilter(new KeysNotMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils())); - manager.addFilter(new RollingStartNumberAfterDayAfterTomorrow()); - manager.addFilter(new RollingStartNumberBeforeRetentionDay(gaenValidationUtils())); - manager.addFilter(new FakeKeysFilter()); - manager.addFilter(new NegativeRollingPeriodFilter()); + manager.addFilter(new Base64Filter(gaenValidationUtils())); + manager.addFilter(new KeysMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils())); + manager.addFilter(new RollingStartNumberAfterDayAfterTomorrowFilter()); + manager.addFilter(new RollingStartNumberInRetentionPeriodFilter(gaenValidationUtils())); + manager.addFilter(new NonFakeKeysFilter()); + manager.addFilter(new ValidRollingPeriodFilter()); return manager; + } } ``` -- `NoBase64Filter` +- `Base64Filter` > This filter validates that the key actually is a correctly encoded base64 string. Since we are using 16 bytes of key data, those can be represented with exactly 24 characters. The validation of the length is already done during model validation and is assumed to be correct when reaching the filter. This filter _throws_ a `KeyIsNotBase64Exception` if any of the keys is wrongly encoded. Every key submitted _MUST_ have correct base64 encoding -- `KeysNotMatchingJWTFilter`: +- `KeysMatchingJWTFilter`: > This filter compares the supplied keys with information found in the JWT token. During the `exposed` request, the onset date, which will be set by the health authority and inserted as a claim into the JWT is the lower bound for allowed key dates. For the `exposednextday` the JWT contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual supplied key. -- `RollingStartNumberAfterDayAfterTomorrow`: - > Representing the maximum allowed time skew. Any key which is further in the future as the day after tomorrow is considered to be _maliciously_ uploaded and is hence filtered out. -- `RollingStartNumberBeforeRetentionDay`: - > Any key which was valid earlier than `RetentionPeriod` is considered to be outdated and not saved in the database. The key would be removed during the next database clean anyways. -- `FakeKeysFilter` - > Any key which has the `fake` flag is not inserted. -- `NegativeRollingPeriodFilter`: - > The `RollingPeriod` represents the 10 minutes interval of the key's validity. Negative numbers are not possible, hence any key having a negative rolling period is considered to be _maliciously_ uploaded. \ No newline at end of file +- `RollingStartNumberAfterDayAfterTomorrowFilter`: + > Representing the maximum allowed time skew. Any key which is further in the future as the day after tomorrow is considered to be _maliciously_ or faulty (for example because of wrong date time settings) uploaded and is hence filtered out. +- `RollingStartNumberInRetentionPeriodFilter`: + > Only keys with key date in the configured retention period are inserted into the datbase. Any key which was valid earlier than `RetentionPeriod` is considered to be outdated and not saved in the database. The key would be removed during the next database clean anyways. +- `NonFakeKeysFilter` + > Only keys that are non-fake are inserted into the database, more precicely keys that have the fake flag set to `0`. +- `ValidRollingPeriodFilter`: + > The `RollingPeriod` represents the 10 minutes interval of the key's validity. Negative numbers are not possible, hence any key having a negative rolling period is considered to be _maliciously_ uploaded. Further, according to [Apple/Googles documentation](https://github.com/google/exposure-notifications-server/blob/main/docs/server_functional_requirements.md) values must be in [1..144] + + +## Additonal Modifiers +- `IOSLegacyProblemRPLT144FModifier` + > This modifier makes sure, that rolling period is always set to 144. Default value according to EN is 144, so just set it to that. This allows to check for the Google-TEKs also on iOS. Because the Rolling Proximity Identifier is based on the TEK and the unix epoch, this should work. The only downside is that iOS will not be able to optimize verification of the TEKs, because it will have to consider each TEK for a whole day. +- `OldAndroid0RPModifier`: + > Some early builds of Google's Exposure Notification API returned TEKs with rolling period set to '0'. According to the specification, this is invalid and will cause both Android and iOS to drop/ignore the key. To mitigate ignoring TEKs from these builds alltogether, the rolling period is increased to '144' (one full day). This should not happen anymore and can be removed in the near future. Until then we are going to log whenever this happens to be able to monitor this problem. + + + +## Configuration +During construction, instances of `GAENDataService` and `ValidationUtils` are needed. Further, any filter or modifier can be added to the list with `addFilter(KeyInsertionFilter filter)` or `addModifier(KeyInsertionModifier)`. Ideally, this happens inside the [`WSBaseConfig`](../config/WSBaseConfig.java), where default filters are added right after constructing the `InsertManager`. + +To allow for conditional `KeyInsertionFilters` or `KeyInsertionModifiers` refer to the following snippet: + +```java +public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfigurer { + + // ... + + @ConditionalOnProperty( + value = "ws.app.gaen.insertmanager.iosrplt144modifier", + havingValue = "true", + matchIfMissing = false) + @Bean + public IOSLegacyProblemRPLT144Modifier iosLegacyProblemRPLT144(InsertManager manager) { + var iosModifier = new IOSLegacyProblemRPLT144Modifier(); + manager.addModifier(iosModifier); + return iosModifier; + } +} +``` + +This looks for a property, either supplied via a `application.properties` file, or via `java` arguments (e.g. `java -D ws.app.gaen.insertmanager.iosrplt144modifier`) and constructs and inserts the respective modifier bean into the modifier chain. For further `SpringBoot` `Conditional` annotations have a look at ["Spring Boot Conditionals"](https://reflectoring.io/spring-boot-conditionals/) + +Encapsulating the logic into smaller pieces of code allows for easier and better reviews of the respective filters. +Further, for each filter an extensive documentation can be provided, without cluttering the code with too many comments. From 019cce010fa19dc00b4d91201f54a64ff011f9ae Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Thu, 3 Sep 2020 13:14:41 +0200 Subject: [PATCH 05/10] InsertManager cleanup This PR updates the comments of the filters and modifiers to follow 'plain English': http://www.plainenglish.co.uk/how-to-write-in-plain-english.html Also changed ValidationUtils.checkForDelayedKeyDateClaim and its use in the KeysMatchingJWTFilter which was wrong. --- .../backend/sdk/ws/config/WSBaseConfig.java | 24 ++++++------- .../sdk/ws/controller/GaenController.java | 14 ++++---- .../backend/sdk/ws/insertmanager/README.md | 32 ++++++++++------- .../{Base64Filter.java => AssertBase64.java} | 20 ++++------- ...ter.java => EnforceMatchingJWTClaims.java} | 34 ++++++++++++------- ...ilter.java => EnforceRetentionPeriod.java} | 9 ++--- ...er.java => EnforceValidRollingPeriod.java} | 3 +- ...akeKeysFilter.java => RemoveFakeKeys.java} | 11 ++---- ...wFilter.java => RemoveKeysFromFuture.java} | 9 ++--- .../IOSLegacyProblemRPLT144Modifier.java | 15 +++++--- .../OldAndroid0RPModifier.java | 9 +---- .../backend/sdk/ws/util/ValidationUtils.java | 20 +++++------ 12 files changed, 94 insertions(+), 106 deletions(-) rename dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/{Base64Filter.java => AssertBase64.java} (55%) rename dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/{KeysMatchingJWTFilter.java => EnforceMatchingJWTClaims.java} (60%) rename dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/{RollingStartNumberInRetentionPeriodFilter.java => EnforceRetentionPeriod.java} (71%) rename dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/{ValidRollingPeriodFilter.java => EnforceValidRollingPeriod.java} (86%) rename dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/{NonFakeKeysFilter.java => RemoveFakeKeys.java} (66%) rename dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/{RollingStartNumberAfterDayAfterTomorrowFilter.java => RemoveKeysFromFuture.java} (67%) diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java index 16b7e16b..dd6a6979 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java @@ -35,12 +35,12 @@ import org.dpppt.backend.sdk.ws.controller.GaenController; import org.dpppt.backend.sdk.ws.filter.ResponseWrapperFilter; import org.dpppt.backend.sdk.ws.insertmanager.InsertManager; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.Base64Filter; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.KeysMatchingJWTFilter; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.NonFakeKeysFilter; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RollingStartNumberAfterDayAfterTomorrowFilter; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RollingStartNumberInRetentionPeriodFilter; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.ValidRollingPeriodFilter; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.AssertBase64; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceMatchingJWTClaims; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceRetentionPeriod; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceValidRollingPeriod; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RemoveFakeKeys; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RemoveKeysFromFuture; import org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier.IOSLegacyProblemRPLT144Modifier; import org.dpppt.backend.sdk.ws.insertmanager.insertionmodifier.OldAndroid0RPModifier; import org.dpppt.backend.sdk.ws.interceptor.HeaderInjector; @@ -216,12 +216,12 @@ public ProtoSignature gaenSigner() { @Bean public InsertManager insertManager() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); - manager.addFilter(new Base64Filter(gaenValidationUtils())); - manager.addFilter(new KeysMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils())); - manager.addFilter(new RollingStartNumberAfterDayAfterTomorrowFilter()); - manager.addFilter(new RollingStartNumberInRetentionPeriodFilter(gaenValidationUtils())); - manager.addFilter(new NonFakeKeysFilter()); - manager.addFilter(new ValidRollingPeriodFilter()); + manager.addFilter(new AssertBase64(gaenValidationUtils())); + manager.addFilter(new EnforceMatchingJWTClaims(gaenRequestValidator, gaenValidationUtils())); + manager.addFilter(new RemoveKeysFromFuture()); + manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); + manager.addFilter(new RemoveFakeKeys()); + manager.addFilter(new EnforceValidRollingPeriod()); return manager; } diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java index f5024ea6..35aacbbc 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java @@ -35,7 +35,7 @@ import org.dpppt.backend.sdk.utils.UTCInstant; import org.dpppt.backend.sdk.ws.insertmanager.InsertException; import org.dpppt.backend.sdk.ws.insertmanager.InsertManager; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.Base64Filter.KeyIsNotBase64Exception; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.AssertBase64.KeyIsNotBase64Exception; import org.dpppt.backend.sdk.ws.security.ValidateRequest; import org.dpppt.backend.sdk.ws.security.ValidateRequest.ClaimIsBeforeOnsetException; import org.dpppt.backend.sdk.ws.security.ValidateRequest.InvalidDateException; @@ -44,7 +44,7 @@ import org.dpppt.backend.sdk.ws.security.signature.ProtoSignature.ProtoSignatureWrapper; import org.dpppt.backend.sdk.ws.util.ValidationUtils; import org.dpppt.backend.sdk.ws.util.ValidationUtils.BadBatchReleaseTimeException; -import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateClaimIsWrong; +import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateClaimIsMissing; import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateIsInvalid; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -154,7 +154,7 @@ public GaenController( // configured Filters in the WSBaseConfig) insertIntoDatabaseIfJWTIsNotFake(gaenRequest.getGaenKeys(), userAgent, principal, now); - this.validationUtils.validateDelayedKeyDate( + this.validationUtils.assertDelayedKeyDate( now, UTCInstant.of(gaenRequest.getDelayedKeyDate(), GaenUnit.TenMinutes)); var responseBuilder = ResponseEntity.ok(); @@ -215,10 +215,12 @@ public GaenController( "JWT token that can be verified by the backend server, must have been created by" + " /v1/gaen/exposed and contain the delayedKeyDate") Object principal) - throws KeyIsNotBase64Exception, DelayedKeyDateClaimIsWrong { + throws KeyIsNotBase64Exception, DelayedKeyDateClaimIsMissing { var now = UTCInstant.now(); - validationUtils.checkForDelayedKeyDateClaim(principal, gaenSecondDay.getDelayedKey()); + // Throws an exception if the claim doesn't exist. The actual verification is done in the + // filters. + validationUtils.getDelayedKeyDateClaim(principal); // Filter out non valid keys and insert them into the database (c.f. InsertManager and // configured Filters in the WSBaseConfig) @@ -349,7 +351,7 @@ private void insertIntoDatabaseIfJWTIsNotFake( } } - @ExceptionHandler({DelayedKeyDateClaimIsWrong.class}) + @ExceptionHandler({DelayedKeyDateClaimIsMissing.class}) @ResponseStatus(HttpStatus.BAD_REQUEST) public ResponseEntity delayedClaimIsWrong() { return ResponseEntity.badRequest().body("DelayedKeyDateClaim is wrong"); diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md index 1803416b..840113ca 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md @@ -16,7 +16,7 @@ A valid key is defined as follows: - Rolling Period in [1..144] - Rolling start number inside the configured retention period - Rolling start number not too far in the future, more precisely not after the day after tomorrow at time of insertion -- Key date must honor the onset date which is given by the healt authority +- Key date must honor the onset date which is given by the health authority ## KeyInsertionFilter Interface @@ -55,6 +55,12 @@ public interface KeyInsertionModifier { It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. +## Names + +The filters should be one of +- `Assert` - lets either pass all keys or throws `InsertException` +- `Remove` - explains which keys are removed +- `Enforce` - explains which keys are kept ## InsertException @@ -72,28 +78,28 @@ public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfig @Bean public InsertManager insertManager() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); - manager.addFilter(new Base64Filter(gaenValidationUtils())); - manager.addFilter(new KeysMatchingJWTFilter(gaenRequestValidator, gaenValidationUtils())); - manager.addFilter(new RollingStartNumberAfterDayAfterTomorrowFilter()); - manager.addFilter(new RollingStartNumberInRetentionPeriodFilter(gaenValidationUtils())); - manager.addFilter(new NonFakeKeysFilter()); - manager.addFilter(new ValidRollingPeriodFilter()); + manager.addFilter(new AssertBase64(gaenValidationUtils())); + manager.addFilter(new EnforceMatchingJWTClaims(gaenRequestValidator, gaenValidationUtils())); + manager.addFilter(new RemoveKeysFromFuture()); + manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); + manager.addFilter(new RemoveFakeKeys()); + manager.addFilter(new EnforceValidRollingPeriod()); return manager; } } ``` -- `Base64Filter` +- `AssertBase64` > This filter validates that the key actually is a correctly encoded base64 string. Since we are using 16 bytes of key data, those can be represented with exactly 24 characters. The validation of the length is already done during model validation and is assumed to be correct when reaching the filter. This filter _throws_ a `KeyIsNotBase64Exception` if any of the keys is wrongly encoded. Every key submitted _MUST_ have correct base64 encoding -- `KeysMatchingJWTFilter`: +- `EnforceMatchingJWTClaims`: > This filter compares the supplied keys with information found in the JWT token. During the `exposed` request, the onset date, which will be set by the health authority and inserted as a claim into the JWT is the lower bound for allowed key dates. For the `exposednextday` the JWT contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual supplied key. -- `RollingStartNumberAfterDayAfterTomorrowFilter`: +- `RemoveKeysFromFuture`: > Representing the maximum allowed time skew. Any key which is further in the future as the day after tomorrow is considered to be _maliciously_ or faulty (for example because of wrong date time settings) uploaded and is hence filtered out. -- `RollingStartNumberInRetentionPeriodFilter`: +- `EnforceRetentionPeriod`: > Only keys with key date in the configured retention period are inserted into the datbase. Any key which was valid earlier than `RetentionPeriod` is considered to be outdated and not saved in the database. The key would be removed during the next database clean anyways. -- `NonFakeKeysFilter` +- `RemoveFakeKeys` > Only keys that are non-fake are inserted into the database, more precicely keys that have the fake flag set to `0`. -- `ValidRollingPeriodFilter`: +- `EnforceValidRollingPeriod`: > The `RollingPeriod` represents the 10 minutes interval of the key's validity. Negative numbers are not possible, hence any key having a negative rolling period is considered to be _maliciously_ uploaded. Further, according to [Apple/Googles documentation](https://github.com/google/exposure-notifications-server/blob/main/docs/server_functional_requirements.md) values must be in [1..144] diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/Base64Filter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/AssertBase64.java similarity index 55% rename from dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/Base64Filter.java rename to dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/AssertBase64.java index 63587dfc..a66bce92 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/Base64Filter.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/AssertBase64.java @@ -9,23 +9,17 @@ import org.dpppt.backend.sdk.ws.util.ValidationUtils; /** - * All keys must be valid Base64 encoded. Non valid Base64 keys are not allowed and are filtered - * out. This filter rejects the whole submitted batch of keys, if any of the keys is not valid - * Base64, as this is a client error. + * Rejects a batch of keys if any of them have an invalid base64 encoding. Invalid base64 encodings + * point to a client error. */ -public class Base64Filter implements KeyInsertionFilter { +public class AssertBase64 implements KeyInsertionFilter { private final ValidationUtils validationUtils; - public Base64Filter(ValidationUtils validationUtils) { + public AssertBase64(ValidationUtils validationUtils) { this.validationUtils = validationUtils; } - /** - * Loop through all keys and check for Base64 validity using {@link - * ValidationUtils#isValidBase64Key(String)} and count the number of invalid keys. If the count is - * > 0, a {@link KeyIsNotBase64Exception} is thrown which results in a client error. - */ @Override public List filter( UTCInstant now, @@ -36,10 +30,10 @@ public List filter( Object principal) throws InsertException { - var numberOfInvalidKeys = - content.stream().filter(key -> !validationUtils.isValidBase64Key(key.getKeyData())).count(); + var hasInvalidKeys = + content.stream().anyMatch(key -> !validationUtils.isValidBase64Key(key.getKeyData())); - if (numberOfInvalidKeys > 0) { + if (hasInvalidKeys) { throw new KeyIsNotBase64Exception(); } return content; diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeysMatchingJWTFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaims.java similarity index 60% rename from dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeysMatchingJWTFilter.java rename to dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaims.java index a831f6f2..2bbc4e54 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/KeysMatchingJWTFilter.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaims.java @@ -11,22 +11,26 @@ import org.dpppt.backend.sdk.ws.security.ValidateRequest.ClaimIsBeforeOnsetException; import org.dpppt.backend.sdk.ws.security.ValidateRequest.InvalidDateException; import org.dpppt.backend.sdk.ws.util.ValidationUtils; -import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateClaimIsWrong; +import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateClaimIsMissing; import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateIsInvalid; /** - * This filter compares the supplied keys with information found in the JWT token. During the - * `exposed` request, the onset date, which will be set by the health authority and inserted as a - * claim into the JWT is the lower bound for allowed key dates. For the `exposednextday` the JWT - * contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual - * supplied key. + * * This filter compares the supplied keys with information found in the JWT token. Depending on + * the request, the following checks are made: + * + *
    + *
  • `exposed`: the key dates must be >= the onset date, which was set by the health authority + * and is available as a claim in the JWT + *
  • `exposednextday`: the supplied key must match `delayedKeyDate`, which has been set as a + * claim by a previous call to `exposed` + *
*/ -public class KeysMatchingJWTFilter implements KeyInsertionFilter { +public class EnforceMatchingJWTClaims implements KeyInsertionFilter { private final ValidateRequest validateRequest; private final ValidationUtils validationUtils; - public KeysMatchingJWTFilter(ValidateRequest validateRequest, ValidationUtils utils) { + public EnforceMatchingJWTClaims(ValidateRequest validateRequest, ValidationUtils utils) { this.validateRequest = validateRequest; this.validationUtils = utils; } @@ -43,11 +47,17 @@ public List filter( .filter( key -> { try { - validationUtils.checkForDelayedKeyDateClaim(principal, key); + // getDelayedKeyDateClaim throws an exception if there is no delayedKeyDate claim + // available. + var delayedKeyDateClaim = validationUtils.getDelayedKeyDateClaim(principal); + // Found a delayedKeyDate claim, so it must be `/exposednextday` var delayedKeyDate = UTCInstant.of(key.getRollingStartNumber(), GaenUnit.TenMinutes); - return isValidDelayedKeyDate(now, delayedKeyDate); - } catch (DelayedKeyDateClaimIsWrong ex) { + return delayedKeyDateClaim.equals(delayedKeyDate) + && isValidDelayedKeyDate(now, delayedKeyDate); + + } catch (DelayedKeyDateClaimIsMissing ex) { + // Didn't find a delayedKeyDate claim, so it must be `/exposed` return isValidKeyDate(key, principal, now); } }) @@ -65,7 +75,7 @@ private boolean isValidKeyDate(GaenKey key, Object principal, UTCInstant now) { private boolean isValidDelayedKeyDate(UTCInstant now, UTCInstant delayedKeyDate) { try { - validationUtils.validateDelayedKeyDate(now, delayedKeyDate); + validationUtils.assertDelayedKeyDate(now, delayedKeyDate); return true; } catch (DelayedKeyDateIsInvalid ex) { return false; diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberInRetentionPeriodFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceRetentionPeriod.java similarity index 71% rename from dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberInRetentionPeriodFilter.java rename to dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceRetentionPeriod.java index 2d5f52bc..f7df3abe 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberInRetentionPeriodFilter.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceRetentionPeriod.java @@ -13,19 +13,14 @@ * Checks if a key is in the configured retention period. If a key is before the retention period it * is filtered out, as it will not be relevant for the system anymore. */ -public class RollingStartNumberInRetentionPeriodFilter implements KeyInsertionFilter { +public class EnforceRetentionPeriod implements KeyInsertionFilter { private final ValidationUtils validationUtils; - public RollingStartNumberInRetentionPeriodFilter(ValidationUtils validationUtils) { + public EnforceRetentionPeriod(ValidationUtils validationUtils) { this.validationUtils = validationUtils; } - /** - * Loops through all the keys and converts the rolling start number to a timestamp. Using {@link - * ValidationUtils#isBeforeRetention(UTCInstant, UTCInstant)} only keys are accepted that are not - * before the retention period. Keys before the retention period are filtered out. - */ @Override public List filter( UTCInstant now, diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/ValidRollingPeriodFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceValidRollingPeriod.java similarity index 86% rename from dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/ValidRollingPeriodFilter.java rename to dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceValidRollingPeriod.java index 5b4610cc..9eb7a8c3 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/ValidRollingPeriodFilter.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceValidRollingPeriod.java @@ -13,9 +13,8 @@ * "https://github.com/google/exposure-notifications-server/blob/main/docs/server_functional_requirements.md#publishing-temporary-exposure-keys" * >EN documentation */ -public class ValidRollingPeriodFilter implements KeyInsertionFilter { +public class EnforceValidRollingPeriod implements KeyInsertionFilter { - /** Loop through given keys and filter out keys which have rolling period < 1 or > 144. */ @Override public List filter( UTCInstant now, diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/NonFakeKeysFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RemoveFakeKeys.java similarity index 66% rename from dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/NonFakeKeysFilter.java rename to dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RemoveFakeKeys.java index 063f3977..d184c1a5 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/NonFakeKeysFilter.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RemoveFakeKeys.java @@ -7,16 +7,9 @@ import org.dpppt.backend.sdk.utils.UTCInstant; import org.dpppt.backend.sdk.ws.insertmanager.OSType; -/** - * Filters out fake keys from fake upload requests. Only Non-Fake keys are inserted into the - * database. - */ -public class NonFakeKeysFilter implements KeyInsertionFilter { +/** Keep only Non-Fake keys, so that fake keys are not stored in the database. */ +public class RemoveFakeKeys implements KeyInsertionFilter { - /** - * Loops through the list of given keys and checks the fake flag. Only return keys that have fake - * flag set to 0 - */ @Override public List filter( UTCInstant now, diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberAfterDayAfterTomorrowFilter.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RemoveKeysFromFuture.java similarity index 67% rename from dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberAfterDayAfterTomorrowFilter.java rename to dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RemoveKeysFromFuture.java index c45af883..ede8a4d9 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RollingStartNumberAfterDayAfterTomorrowFilter.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/RemoveKeysFromFuture.java @@ -9,15 +9,10 @@ import org.dpppt.backend.sdk.ws.insertmanager.OSType; /** - * Checks if a key has rolling start number after the day after tomorrow. If so, the key is filtered - * out, as this is not allowed by the system to insert keys too far in the future. + * Reject keys that are too far in the future. The `rollingStart` must not be later than tomorrow. */ -public class RollingStartNumberAfterDayAfterTomorrowFilter implements KeyInsertionFilter { +public class RemoveKeysFromFuture implements KeyInsertionFilter { - /** - * Loops through all the keys and converts the rolling start number to a timstamp. The it is - * checked if the timestamp is before now + 2 days. - */ @Override public List filter( UTCInstant now, diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/IOSLegacyProblemRPLT144Modifier.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/IOSLegacyProblemRPLT144Modifier.java index 9d255562..c62708fe 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/IOSLegacyProblemRPLT144Modifier.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/IOSLegacyProblemRPLT144Modifier.java @@ -7,11 +7,16 @@ import org.dpppt.backend.sdk.ws.insertmanager.OSType; /** - * This key modifier makes sure, that rolling period is always set to 144. Default value according - * to EN is 144, so just set it to that. This allows to check for the Google-TEKs also on iOS. - * Because the Rolling Proximity Identifier is based on the TEK and the unix epoch, this should - * work. The only downside is that iOS will not be able to optimize verification of the TEKs, - * because it will have to consider each TEK for a whole day. + * Overwrite the rolling period with the default value of 144 so that iOS does not reject the keys. + * Since version 1.5 of the GAEN on Android, TEKs with a rolling period < 144 can be released. + * Unfortunately these keys are rejected by iOS, so this filter sets the default value. There are + * two downsides to this: + * + *
    + *
  • some more work for the GAEN to verify the keys + *
  • a same-day key with original rolling period < 144 will be released later and thus delay + * detection of an eventual exposition + *
*/ public class IOSLegacyProblemRPLT144Modifier implements KeyInsertionModifier { diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/OldAndroid0RPModifier.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/OldAndroid0RPModifier.java index 731711b9..8c4038f9 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/OldAndroid0RPModifier.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionmodifier/OldAndroid0RPModifier.java @@ -20,11 +20,6 @@ public class OldAndroid0RPModifier implements KeyInsertionModifier { private static final Logger logger = LoggerFactory.getLogger(OldAndroid0RPModifier.class); - /** - * Loop through all the given keys and check if the rolling period is equal to 0. If so, set to - * 144. In case a key with rolling period 0 is received from an iOS client, an error log is - * printed. - */ @Override public List modify( UTCInstant now, @@ -35,9 +30,7 @@ public List modify( Object principal) { for (GaenKey gaenKey : content) { if (gaenKey.getRollingPeriod().equals(0)) { - if (osType.equals(OSType.IOS)) { - logger.error("We got a rollingPeriod of 0 ({},{},{})", osType, osVersion, appVersion); - } + logger.error("We got a rollingPeriod of 0 ({},{},{})", osType, osVersion, appVersion); gaenKey.setRollingPeriod(144); } } diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/util/ValidationUtils.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/util/ValidationUtils.java index 8e849d25..09cf6d4f 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/util/ValidationUtils.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/util/ValidationUtils.java @@ -11,7 +11,7 @@ import java.time.Duration; import java.util.Base64; -import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.model.gaen.GaenUnit; import org.dpppt.backend.sdk.utils.UTCInstant; import org.springframework.security.oauth2.jwt.Jwt; @@ -101,7 +101,7 @@ public boolean isValidBatchReleaseTime(UTCInstant batchReleaseTime, UTCInstant n return this.isDateInRange(batchReleaseTime, now); } - public void validateDelayedKeyDate(UTCInstant now, UTCInstant delayedKeyDate) + public void assertDelayedKeyDate(UTCInstant now, UTCInstant delayedKeyDate) throws DelayedKeyDateIsInvalid { if (delayedKeyDate.isBeforeDateOf(now.getLocalDate().minusDays(1)) || delayedKeyDate.isAfterDateOf(now.getLocalDate().plusDays(1))) { @@ -109,19 +109,15 @@ public void validateDelayedKeyDate(UTCInstant now, UTCInstant delayedKeyDate) } } - public void checkForDelayedKeyDateClaim(Object principal, GaenKey delayedKey) - throws DelayedKeyDateClaimIsWrong { - if (principal instanceof Jwt - && Boolean.FALSE.equals(((Jwt) principal).containsClaim("delayedKeyDate"))) { - throw new DelayedKeyDateClaimIsWrong(); - } + public UTCInstant getDelayedKeyDateClaim(Object principal) throws DelayedKeyDateClaimIsMissing { if (principal instanceof Jwt) { var jwt = (Jwt) principal; - var claimKeyDate = Integer.parseInt(jwt.getClaimAsString("delayedKeyDate")); - if (!delayedKey.getRollingStartNumber().equals(claimKeyDate)) { - throw new DelayedKeyDateClaimIsWrong(); + if (jwt.containsClaim("delayedKeyDate")) { + return UTCInstant.of( + Integer.parseInt(jwt.getClaimAsString("delayedKeyDate")), GaenUnit.TenMinutes); } } + throw new DelayedKeyDateClaimIsMissing(); } public boolean jwtIsFake(Object principal) { @@ -141,7 +137,7 @@ public class DelayedKeyDateIsInvalid extends Exception { private static final long serialVersionUID = -2667236967819549686L; } - public class DelayedKeyDateClaimIsWrong extends Exception { + public class DelayedKeyDateClaimIsMissing extends Exception { /** */ private static final long serialVersionUID = 4683923905451080793L; From 9014336372f8bb4b40ae5c4ddac94986f60fb760 Mon Sep 17 00:00:00 2001 From: alig Date: Fri, 4 Sep 2020 08:46:38 +0200 Subject: [PATCH 06/10] Split JWTClaims filter for Exposed and ExposedNextDay Simplify Exception Handling in controller and make InsertException abstract. --- .../backend/sdk/ws/config/WSBaseConfig.java | 22 +++++++-- .../sdk/ws/controller/GaenController.java | 39 +++++---------- .../sdk/ws/insertmanager/InsertException.java | 2 +- .../backend/sdk/ws/insertmanager/README.md | 25 ++++++++-- .../EnforceMatchingJWTClaimsForExposed.java | 47 +++++++++++++++++++ ...ceMatchingJWTClaimsForExposedNextDay.java} | 35 +++----------- 6 files changed, 103 insertions(+), 67 deletions(-) create mode 100644 dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaimsForExposed.java rename dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/{EnforceMatchingJWTClaims.java => EnforceMatchingJWTClaimsForExposedNextDay.java} (56%) diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java index dd6a6979..952a2f85 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java @@ -36,7 +36,8 @@ import org.dpppt.backend.sdk.ws.filter.ResponseWrapperFilter; import org.dpppt.backend.sdk.ws.insertmanager.InsertManager; import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.AssertBase64; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceMatchingJWTClaims; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceMatchingJWTClaimsForExposed; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceMatchingJWTClaimsForExposedNextDay; import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceRetentionPeriod; import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceValidRollingPeriod; import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.RemoveFakeKeys; @@ -214,10 +215,22 @@ public ProtoSignature gaenSigner() { } @Bean - public InsertManager insertManager() { + public InsertManager insertManagerExposed() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); manager.addFilter(new AssertBase64(gaenValidationUtils())); - manager.addFilter(new EnforceMatchingJWTClaims(gaenRequestValidator, gaenValidationUtils())); + manager.addFilter(new EnforceMatchingJWTClaimsForExposed(gaenRequestValidator)); + manager.addFilter(new RemoveKeysFromFuture()); + manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); + manager.addFilter(new RemoveFakeKeys()); + manager.addFilter(new EnforceValidRollingPeriod()); + return manager; + } + + @Bean + public InsertManager insertManagerExposedNextDay() { + var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); + manager.addFilter(new AssertBase64(gaenValidationUtils())); + manager.addFilter(new EnforceMatchingJWTClaimsForExposedNextDay(gaenValidationUtils())); manager.addFilter(new RemoveKeysFromFuture()); manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); manager.addFilter(new RemoveFakeKeys()); @@ -281,7 +294,8 @@ public GaenController gaenController() { theValidator = backupValidator(); } return new GaenController( - insertManager(), + insertManagerExposed(), + insertManagerExposedNextDay(), gaenDataService(), fakeKeyService(), theValidator, diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java index 35aacbbc..44fc6dc5 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java @@ -27,7 +27,6 @@ import org.dpppt.backend.sdk.data.gaen.FakeKeyService; import org.dpppt.backend.sdk.data.gaen.GAENDataService; import org.dpppt.backend.sdk.model.gaen.DayBuckets; -import org.dpppt.backend.sdk.model.gaen.GaenKey; import org.dpppt.backend.sdk.model.gaen.GaenRequest; import org.dpppt.backend.sdk.model.gaen.GaenSecondDay; import org.dpppt.backend.sdk.model.gaen.GaenUnit; @@ -86,7 +85,8 @@ public class GaenController { private final Duration requestTime; private final ValidateRequest validateRequest; private final ValidationUtils validationUtils; - private final InsertManager insertManager; + private final InsertManager insertManagerExposed; + private final InsertManager insertManagerExposedNextDay; private final GAENDataService dataService; private final FakeKeyService fakeKeyService; private final Duration exposedListCacheControl; @@ -94,7 +94,8 @@ public class GaenController { private final ProtoSignature gaenSigner; public GaenController( - InsertManager insertManager, + InsertManager insertManagerExposed, + InsertManager insertManagerExposedNextDay, GAENDataService dataService, FakeKeyService fakeKeyService, ValidateRequest validateRequest, @@ -104,7 +105,8 @@ public GaenController( Duration requestTime, Duration exposedListCacheControl, PrivateKey secondDayKey) { - this.insertManager = insertManager; + this.insertManagerExposed = insertManagerExposed; + this.insertManagerExposedNextDay = insertManagerExposedNextDay; this.dataService = dataService; this.fakeKeyService = fakeKeyService; this.releaseBucketDuration = releaseBucketDuration; @@ -145,14 +147,14 @@ public GaenController( @AuthenticationPrincipal @Documentation(description = "JWT token that can be verified by the backend server") Object principal) - throws WrongScopeException, KeyIsNotBase64Exception, DelayedKeyDateIsInvalid { + throws DelayedKeyDateIsInvalid, InsertException, WrongScopeException { var now = UTCInstant.now(); this.validateRequest.isValid(principal); // Filter out non valid keys and insert them into the database (c.f. InsertManager and // configured Filters in the WSBaseConfig) - insertIntoDatabaseIfJWTIsNotFake(gaenRequest.getGaenKeys(), userAgent, principal, now); + insertManagerExposed.insertIntoDatabase(gaenRequest.getGaenKeys(), userAgent, principal, now); this.validationUtils.assertDelayedKeyDate( now, UTCInstant.of(gaenRequest.getDelayedKeyDate(), GaenUnit.TenMinutes)); @@ -215,7 +217,7 @@ public GaenController( "JWT token that can be verified by the backend server, must have been created by" + " /v1/gaen/exposed and contain the delayedKeyDate") Object principal) - throws KeyIsNotBase64Exception, DelayedKeyDateClaimIsMissing { + throws DelayedKeyDateClaimIsMissing, InsertException { var now = UTCInstant.now(); // Throws an exception if the claim doesn't exist. The actual verification is done in the @@ -224,7 +226,8 @@ public GaenController( // Filter out non valid keys and insert them into the database (c.f. InsertManager and // configured Filters in the WSBaseConfig) - insertIntoDatabaseIfJWTIsNotFake(gaenSecondDay.getDelayedKey(), userAgent, principal, now); + insertManagerExposedNextDay.insertIntoDatabase( + List.of(gaenSecondDay.getDelayedKey()), userAgent, principal, now); return () -> { try { @@ -331,26 +334,6 @@ public GaenController( return ResponseEntity.ok(dayBuckets); } - private void insertIntoDatabaseIfJWTIsNotFake( - GaenKey key, String userAgent, Object principal, UTCInstant now) - throws KeyIsNotBase64Exception { - List keys = new ArrayList<>(); - keys.add(key); - insertIntoDatabaseIfJWTIsNotFake(keys, userAgent, principal, now); - } - - private void insertIntoDatabaseIfJWTIsNotFake( - List keys, String userAgent, Object principal, UTCInstant now) - throws KeyIsNotBase64Exception { - try { - insertManager.insertIntoDatabase(keys, userAgent, principal, now); - } catch (KeyIsNotBase64Exception ex) { - throw ex; - } catch (InsertException ex) { - logger.info("Unknown exception thrown: ", ex); - } - } - @ExceptionHandler({DelayedKeyDateClaimIsMissing.class}) @ResponseStatus(HttpStatus.BAD_REQUEST) public ResponseEntity delayedClaimIsWrong() { diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertException.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertException.java index 43564ac3..8c8f54e7 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertException.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/InsertException.java @@ -1,6 +1,6 @@ package org.dpppt.backend.sdk.ws.insertmanager; -public class InsertException extends Exception { +public abstract class InsertException extends Exception { /** */ private static final long serialVersionUID = 6476089262577182680L; diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md index 840113ca..439d9240 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md @@ -68,7 +68,7 @@ An `InsertException` can be thrown inside an implementation of the `InsertionFil ## Default Filters -Looking at the `WSBaseConfig`, we can see that during construction of the `InsertManager` bean, a set of default filters are added: +Looking at the `WSBaseConfig`, we can see that two instances of the `InsertManager` are constructed, one for the `exposed` request and one for the `exposedNextDay` request, both are supplied wit a set of default filters: ```java public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfigurer { @@ -76,23 +76,38 @@ public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfig // ... @Bean - public InsertManager insertManager() { + public InsertManager insertManagerExposed() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); manager.addFilter(new AssertBase64(gaenValidationUtils())); - manager.addFilter(new EnforceMatchingJWTClaims(gaenRequestValidator, gaenValidationUtils())); + manager.addFilter(new EnforceMatchingJWTClaimsForExposed(gaenRequestValidator)); manager.addFilter(new RemoveKeysFromFuture()); manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); manager.addFilter(new RemoveFakeKeys()); manager.addFilter(new EnforceValidRollingPeriod()); return manager; } + + @Bean + public InsertManager insertManagerExposedNextDay() { + var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); + manager.addFilter(new AssertBase64(gaenValidationUtils())); + manager.addFilter(new EnforceMatchingJWTClaimsForExposedNextDay(gaenValidationUtils())); + manager.addFilter(new RemoveKeysFromFuture()); + manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); + manager.addFilter(new RemoveFakeKeys()); + manager.addFilter(new EnforceValidRollingPeriod()); + return manager; + } + } ``` - `AssertBase64` > This filter validates that the key actually is a correctly encoded base64 string. Since we are using 16 bytes of key data, those can be represented with exactly 24 characters. The validation of the length is already done during model validation and is assumed to be correct when reaching the filter. This filter _throws_ a `KeyIsNotBase64Exception` if any of the keys is wrongly encoded. Every key submitted _MUST_ have correct base64 encoding -- `EnforceMatchingJWTClaims`: - > This filter compares the supplied keys with information found in the JWT token. During the `exposed` request, the onset date, which will be set by the health authority and inserted as a claim into the JWT is the lower bound for allowed key dates. For the `exposednextday` the JWT contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual supplied key. +- `EnforceMatchingJWTClaimsForExposed`: + > This filter compares the supplied keys with information found in the JWT token for the `exposed` request. It makes sure, that the onset date, which will be set by the health authority and inserted as a claim into the JWT is the lower bound for allowed key dates. +- `EnforceMatchingJWTClaimsForExposedNextDay`: + > This filter compares the supplied keys with information found in the JWT token for the `exposednextday` request. It makes sure, that the JWT contains the previously submitted and checked `delayedKeyDate`, which is compared to the actual supplied key. - `RemoveKeysFromFuture`: > Representing the maximum allowed time skew. Any key which is further in the future as the day after tomorrow is considered to be _maliciously_ or faulty (for example because of wrong date time settings) uploaded and is hence filtered out. - `EnforceRetentionPeriod`: diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaimsForExposed.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaimsForExposed.java new file mode 100644 index 00000000..b60a270b --- /dev/null +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaimsForExposed.java @@ -0,0 +1,47 @@ +package org.dpppt.backend.sdk.ws.insertmanager.insertionfilters; + +import java.util.List; +import java.util.stream.Collectors; +import org.dpppt.backend.sdk.model.gaen.GaenKey; +import org.dpppt.backend.sdk.semver.Version; +import org.dpppt.backend.sdk.utils.UTCInstant; +import org.dpppt.backend.sdk.ws.insertmanager.OSType; +import org.dpppt.backend.sdk.ws.security.ValidateRequest; +import org.dpppt.backend.sdk.ws.security.ValidateRequest.ClaimIsBeforeOnsetException; +import org.dpppt.backend.sdk.ws.security.ValidateRequest.InvalidDateException; + +/** + * This filter compares the supplied keys from the exposed request with information found in the JWT + * token: the key dates must be >= the onset date, which was set by the health authority and is + * available as a claim in the JWT + */ +public class EnforceMatchingJWTClaimsForExposed implements KeyInsertionFilter { + + private final ValidateRequest validateRequest; + + public EnforceMatchingJWTClaimsForExposed(ValidateRequest validateRequest) { + this.validateRequest = validateRequest; + } + + @Override + public List filter( + UTCInstant now, + List content, + OSType osType, + Version osVersion, + Version appVersion, + Object principal) { + return content.stream() + .filter(key -> isValidKeyDate(key, principal, now)) + .collect(Collectors.toList()); + } + + private boolean isValidKeyDate(GaenKey key, Object principal, UTCInstant now) { + try { + validateRequest.validateKeyDate(now, principal, key); + return true; + } catch (InvalidDateException | ClaimIsBeforeOnsetException es) { + return false; + } + } +} diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaims.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaimsForExposedNextDay.java similarity index 56% rename from dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaims.java rename to dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaimsForExposedNextDay.java index 2bbc4e54..266aa43e 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaims.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/EnforceMatchingJWTClaimsForExposedNextDay.java @@ -7,31 +7,20 @@ import org.dpppt.backend.sdk.semver.Version; import org.dpppt.backend.sdk.utils.UTCInstant; import org.dpppt.backend.sdk.ws.insertmanager.OSType; -import org.dpppt.backend.sdk.ws.security.ValidateRequest; -import org.dpppt.backend.sdk.ws.security.ValidateRequest.ClaimIsBeforeOnsetException; -import org.dpppt.backend.sdk.ws.security.ValidateRequest.InvalidDateException; import org.dpppt.backend.sdk.ws.util.ValidationUtils; import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateClaimIsMissing; import org.dpppt.backend.sdk.ws.util.ValidationUtils.DelayedKeyDateIsInvalid; /** - * * This filter compares the supplied keys with information found in the JWT token. Depending on - * the request, the following checks are made: - * - *
    - *
  • `exposed`: the key dates must be >= the onset date, which was set by the health authority - * and is available as a claim in the JWT - *
  • `exposednextday`: the supplied key must match `delayedKeyDate`, which has been set as a - * claim by a previous call to `exposed` - *
+ * This filter compares the supplied keys from the exposed next day request with information found + * in the JWT token: the supplied key must match `delayedKeyDate`, which has been set as a claim by + * a previous call to `exposed` */ -public class EnforceMatchingJWTClaims implements KeyInsertionFilter { +public class EnforceMatchingJWTClaimsForExposedNextDay implements KeyInsertionFilter { - private final ValidateRequest validateRequest; private final ValidationUtils validationUtils; - public EnforceMatchingJWTClaims(ValidateRequest validateRequest, ValidationUtils utils) { - this.validateRequest = validateRequest; + public EnforceMatchingJWTClaimsForExposedNextDay(ValidationUtils utils) { this.validationUtils = utils; } @@ -50,29 +39,17 @@ public List filter( // getDelayedKeyDateClaim throws an exception if there is no delayedKeyDate claim // available. var delayedKeyDateClaim = validationUtils.getDelayedKeyDateClaim(principal); - // Found a delayedKeyDate claim, so it must be `/exposednextday` var delayedKeyDate = UTCInstant.of(key.getRollingStartNumber(), GaenUnit.TenMinutes); return delayedKeyDateClaim.equals(delayedKeyDate) && isValidDelayedKeyDate(now, delayedKeyDate); - } catch (DelayedKeyDateClaimIsMissing ex) { - // Didn't find a delayedKeyDate claim, so it must be `/exposed` - return isValidKeyDate(key, principal, now); + return false; } }) .collect(Collectors.toList()); } - private boolean isValidKeyDate(GaenKey key, Object principal, UTCInstant now) { - try { - validateRequest.validateKeyDate(now, principal, key); - return true; - } catch (InvalidDateException | ClaimIsBeforeOnsetException es) { - return false; - } - } - private boolean isValidDelayedKeyDate(UTCInstant now, UTCInstant delayedKeyDate) { try { validationUtils.assertDelayedKeyDate(now, delayedKeyDate); From a92a4810bedc0b3e841d017ce7fcb30999fa16b1 Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Mon, 7 Sep 2020 13:36:12 +0200 Subject: [PATCH 07/10] rename base64 checks to keyformat, as also the length is checked --- .../dpppt/backend/sdk/ws/config/WSBaseConfig.java | 6 +++--- .../backend/sdk/ws/controller/DPPPTController.java | 4 ++-- .../backend/sdk/ws/controller/DebugController.java | 2 +- .../backend/sdk/ws/controller/GaenController.java | 4 ++-- .../dpppt/backend/sdk/ws/insertmanager/README.md | 14 +++++++++----- .../{AssertBase64.java => AssertKeyFormat.java} | 14 +++++++------- .../dpppt/backend/sdk/ws/util/ValidationUtils.java | 9 +++------ 7 files changed, 27 insertions(+), 26 deletions(-) rename dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/{AssertBase64.java => AssertKeyFormat.java} (73%) diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java index 952a2f85..53b27da3 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java @@ -35,7 +35,7 @@ import org.dpppt.backend.sdk.ws.controller.GaenController; import org.dpppt.backend.sdk.ws.filter.ResponseWrapperFilter; import org.dpppt.backend.sdk.ws.insertmanager.InsertManager; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.AssertBase64; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.AssertKeyFormat; import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceMatchingJWTClaimsForExposed; import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceMatchingJWTClaimsForExposedNextDay; import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.EnforceRetentionPeriod; @@ -217,7 +217,7 @@ public ProtoSignature gaenSigner() { @Bean public InsertManager insertManagerExposed() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); - manager.addFilter(new AssertBase64(gaenValidationUtils())); + manager.addFilter(new AssertKeyFormat(gaenValidationUtils())); manager.addFilter(new EnforceMatchingJWTClaimsForExposed(gaenRequestValidator)); manager.addFilter(new RemoveKeysFromFuture()); manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); @@ -229,7 +229,7 @@ public InsertManager insertManagerExposed() { @Bean public InsertManager insertManagerExposedNextDay() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); - manager.addFilter(new AssertBase64(gaenValidationUtils())); + manager.addFilter(new AssertKeyFormat(gaenValidationUtils())); manager.addFilter(new EnforceMatchingJWTClaimsForExposedNextDay(gaenValidationUtils())); manager.addFilter(new RemoveKeysFromFuture()); manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DPPPTController.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DPPPTController.java index b5ba7f3a..0c12991b 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DPPPTController.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DPPPTController.java @@ -123,7 +123,7 @@ public DPPPTController( if (!this.validateRequest.isValid(principal)) { return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); } - if (!validationUtils.isValidBase64Key(exposeeRequest.getKey())) { + if (!validationUtils.isValidKeyFormat(exposeeRequest.getKey())) { return new ResponseEntity<>("No valid base64 key", HttpStatus.BAD_REQUEST); } // TODO: should we give that information? @@ -186,7 +186,7 @@ public DPPPTController( List exposees = new ArrayList<>(); for (var exposedKey : exposeeRequests.getExposedKeys()) { - if (!validationUtils.isValidBase64Key(exposedKey.getKey())) { + if (!validationUtils.isValidKeyFormat(exposedKey.getKey())) { return new ResponseEntity<>("No valid base64 key", HttpStatus.BAD_REQUEST); } diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DebugController.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DebugController.java index 6dd653aa..cf842f0d 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DebugController.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/DebugController.java @@ -71,7 +71,7 @@ public DebugController( List nonFakeKeys = new ArrayList<>(); for (var key : gaenRequest.getGaenKeys()) { - if (!validationUtils.isValidBase64Key(key.getKeyData())) { + if (!validationUtils.isValidKeyFormat(key.getKeyData())) { return new ResponseEntity<>("No valid base64 key", HttpStatus.BAD_REQUEST); } this.validateRequest.validateKeyDate(now, principal, key); diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java index 44fc6dc5..9e6931ed 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/controller/GaenController.java @@ -34,7 +34,7 @@ import org.dpppt.backend.sdk.utils.UTCInstant; import org.dpppt.backend.sdk.ws.insertmanager.InsertException; import org.dpppt.backend.sdk.ws.insertmanager.InsertManager; -import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.AssertBase64.KeyIsNotBase64Exception; +import org.dpppt.backend.sdk.ws.insertmanager.insertionfilters.AssertKeyFormat.KeyFormatException; import org.dpppt.backend.sdk.ws.security.ValidateRequest; import org.dpppt.backend.sdk.ws.security.ValidateRequest.ClaimIsBeforeOnsetException; import org.dpppt.backend.sdk.ws.security.ValidateRequest.InvalidDateException; @@ -355,7 +355,7 @@ public ResponseEntity delayedKeyDateIsInvalid() { BadBatchReleaseTimeException.class, DateTimeParseException.class, ClaimIsBeforeOnsetException.class, - KeyIsNotBase64Exception.class + KeyFormatException.class }) @ResponseStatus(HttpStatus.BAD_REQUEST) public ResponseEntity invalidArguments() { diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md index 439d9240..1447572d 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md @@ -11,7 +11,7 @@ Encapsulating the logic into smaller pieces of code, should allow for easier and ## Valid Keys A valid key is defined as follows: -- Base64 Encoded key +- Base64 Encoded key with correct length of 32 bytes - Non Fake - Rolling Period in [1..144] - Rolling start number inside the configured retention period @@ -78,7 +78,7 @@ public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfig @Bean public InsertManager insertManagerExposed() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); - manager.addFilter(new AssertBase64(gaenValidationUtils())); + manager.addFilter(new AssertKeyFormst(gaenValidationUtils())); manager.addFilter(new EnforceMatchingJWTClaimsForExposed(gaenRequestValidator)); manager.addFilter(new RemoveKeysFromFuture()); manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); @@ -90,7 +90,7 @@ public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfig @Bean public InsertManager insertManagerExposedNextDay() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); - manager.addFilter(new AssertBase64(gaenValidationUtils())); + manager.addFilter(new AssertKeyFormat(gaenValidationUtils())); manager.addFilter(new EnforceMatchingJWTClaimsForExposedNextDay(gaenValidationUtils())); manager.addFilter(new RemoveKeysFromFuture()); manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); @@ -102,8 +102,12 @@ public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfig } ``` -- `AssertBase64` - > This filter validates that the key actually is a correctly encoded base64 string. Since we are using 16 bytes of key data, those can be represented with exactly 24 characters. The validation of the length is already done during model validation and is assumed to be correct when reaching the filter. This filter _throws_ a `KeyIsNotBase64Exception` if any of the keys is wrongly encoded. Every key submitted _MUST_ have correct base64 encoding +- `AssertKeyFormat` + > This filter validates that the key actually is a correctly encoded base64 string and has the correct length. Since + we are using 16 bytes of key data, those can be represented with exactly 24 characters. The validation of the + length is already done during model validation and is assumed to be correct when reaching the filter. This + filter _throws_ a `KeyFormatException` if any of the keys is wrongly encoded. Every key submitted _MUST_ have + correct base64 encoding and have the correct length. - `EnforceMatchingJWTClaimsForExposed`: > This filter compares the supplied keys with information found in the JWT token for the `exposed` request. It makes sure, that the onset date, which will be set by the health authority and inserted as a claim into the JWT is the lower bound for allowed key dates. - `EnforceMatchingJWTClaimsForExposedNextDay`: diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/AssertBase64.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/AssertKeyFormat.java similarity index 73% rename from dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/AssertBase64.java rename to dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/AssertKeyFormat.java index a66bce92..bd96b5e3 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/AssertBase64.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/insertionfilters/AssertKeyFormat.java @@ -9,14 +9,14 @@ import org.dpppt.backend.sdk.ws.util.ValidationUtils; /** - * Rejects a batch of keys if any of them have an invalid base64 encoding. Invalid base64 encodings - * point to a client error. + * Rejects a batch of keys if any of them have an invalid base64 encoding or doesn't have the + * correct length. Invalid base64 encodings or wrong key lengths point to a client error. */ -public class AssertBase64 implements KeyInsertionFilter { +public class AssertKeyFormat implements KeyInsertionFilter { private final ValidationUtils validationUtils; - public AssertBase64(ValidationUtils validationUtils) { + public AssertKeyFormat(ValidationUtils validationUtils) { this.validationUtils = validationUtils; } @@ -31,15 +31,15 @@ public List filter( throws InsertException { var hasInvalidKeys = - content.stream().anyMatch(key -> !validationUtils.isValidBase64Key(key.getKeyData())); + content.stream().anyMatch(key -> !validationUtils.isValidKeyFormat(key.getKeyData())); if (hasInvalidKeys) { - throw new KeyIsNotBase64Exception(); + throw new KeyFormatException(); } return content; } - public class KeyIsNotBase64Exception extends InsertException { + public class KeyFormatException extends InsertException { /** */ private static final long serialVersionUID = -918099046973553472L; diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/util/ValidationUtils.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/util/ValidationUtils.java index 09cf6d4f..f8274056 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/util/ValidationUtils.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/util/ValidationUtils.java @@ -36,18 +36,15 @@ public ValidationUtils(int keyLengthBytes, Duration retentionPeriod, Long releas } /** - * Check the validty of a base64 value + * Check the validity of a base64 encoded key by decoding it and checking the key length * * @param value representation of a base64 value * @return if _value_ is a valid representation */ - public boolean isValidBase64Key(String value) { + public boolean isValidKeyFormat(String value) { try { byte[] key = Base64.getDecoder().decode(value); - if (key.length != KEY_LENGTH_BYTES) { - return false; - } - return true; + return key.length == KEY_LENGTH_BYTES; } catch (Exception e) { return false; } From fe689f8ab19436b965b3a868dab6db7762e0a152 Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Tue, 8 Sep 2020 15:23:13 +0200 Subject: [PATCH 08/10] fix typo --- .../main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md index 1447572d..fa59c451 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md @@ -78,7 +78,7 @@ public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfig @Bean public InsertManager insertManagerExposed() { var manager = new InsertManager(gaenDataService(), gaenValidationUtils()); - manager.addFilter(new AssertKeyFormst(gaenValidationUtils())); + manager.addFilter(new AssertKeyFormat(gaenValidationUtils())); manager.addFilter(new EnforceMatchingJWTClaimsForExposed(gaenRequestValidator)); manager.addFilter(new RemoveKeysFromFuture()); manager.addFilter(new EnforceRetentionPeriod(gaenValidationUtils())); From 630d8bab7718357ef18d8d1e0d9955230547ba53 Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Tue, 8 Sep 2020 15:35:39 +0200 Subject: [PATCH 09/10] add figure to show data flow --- .../backend/sdk/ws/insertmanager/README.md | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md index fa59c451..0ed9849c 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/insertmanager/README.md @@ -1,15 +1,32 @@ # Insert-Manager -## Idea -The Insert-Manager was introduced to reduce logic in controllers. The idea is to provide a second abstraction layer next to the `DataServices` to provide for possible generic validation and normalization. For this there are two mechanisems: modifiers and filters +The Insert-Manager is used to reduce logic in the controllers. +It provides a second abstraction layer next to the `DataServices` to provide generic validation and normalization. +Encapsulating the logic into smaller pieces of code allows for easier and better reviews of the respective +filters and modifiers. +Furthermore, each filter or modifier can be documented individually, without having to document each place where it +is applied. -The Insert-Manager holds a list of `KeyInsertionFilter`, which provide some code, to either filter for invalid data data. Each filter can decide to either skip respective keys, or throw a `InsertException`. Throwing an exception aborts the current insert request, and throws to the controller. Inside the controller the exception can be mapped to a respective error message and http status code. +The insert-manager uses two lists: `modifiers` and `filters`. +First the modifies are run on the keys, then the filters, in the order as they're given to the `InsertManager` -Additionally the Insert-Manager can be configured to hold a list of `KeyInsertModifier`. Modifiers are can be used to modify incoming keys before inserting into the database. (for example to fix buggy clients) +```text + +Mobile Client -> Backend -> InsertManager ( Modifiers -> Filters ) -> Database + +``` + +The Insert-Manager holds a list of `KeyInsertionFilter`, which provide code to filter for invalid data. +Each filter can decide to either skip respective keys, or throw an `InsertException`. +Throwing an exception aborts the current insert request, and the exception is bubbled up to the controller. +Inside the controller the exception can be mapped to a specific error message and an HTTP status code. + +Additionally the Insert-Manager can be configured to hold a list of `KeyInsertModifier`. +Modifiers can modify incoming keys before inserting them into the database, for example to fix buggy clients. -Encapsulating the logic into smaller pieces of code, should allow for easier and better reviews of the respective filters and modifiers. Further, for each filter or modifier an extensive documentation can be provided, without cluttering the code with too many comments. ## Valid Keys + A valid key is defined as follows: - Base64 Encoded key with correct length of 32 bytes - Non Fake @@ -20,6 +37,7 @@ A valid key is defined as follows: ## KeyInsertionFilter Interface + The `KeyInsertionFilter` interface has the following signature: ```java @@ -37,7 +55,9 @@ public interface KeyInsertionFilter { It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. + ## KeyInsertionModifier Interface + The `KeyInsertionModifier` interface has the following signature: ```java @@ -55,6 +75,7 @@ public interface KeyInsertionModifier { It gets a `now` object representing _the time the request started_ from the controller , a list of keys, some OS and app related information taken from the `UserAgent` (c.f. `InsertManager@exctractOS` and following) and a possible principal object, representing a authenticated state (e.g. a `JWT`). The function is marked to throw a `InsertException` to stop the inserting process. + ## Names The filters should be one of @@ -62,10 +83,12 @@ The filters should be one of - `Remove` - explains which keys are removed - `Enforce` - explains which keys are kept + ## InsertException An `InsertException` can be thrown inside an implementation of the `InsertionFilter` interface to mark an insert as "unrecoverable" and abort it. Such "unrecoverable" states might be patterns in the uploaded model, which allows for packet sniffing and/or information gathering. + ## Default Filters Looking at the `WSBaseConfig`, we can see that two instances of the `InsertManager` are constructed, one for the `exposed` request and one for the `exposedNextDay` request, both are supplied wit a set of default filters: @@ -123,14 +146,15 @@ public abstract class WSBaseConfig implements SchedulingConfigurer, WebMvcConfig ## Additonal Modifiers + - `IOSLegacyProblemRPLT144FModifier` > This modifier makes sure, that rolling period is always set to 144. Default value according to EN is 144, so just set it to that. This allows to check for the Google-TEKs also on iOS. Because the Rolling Proximity Identifier is based on the TEK and the unix epoch, this should work. The only downside is that iOS will not be able to optimize verification of the TEKs, because it will have to consider each TEK for a whole day. - `OldAndroid0RPModifier`: > Some early builds of Google's Exposure Notification API returned TEKs with rolling period set to '0'. According to the specification, this is invalid and will cause both Android and iOS to drop/ignore the key. To mitigate ignoring TEKs from these builds alltogether, the rolling period is increased to '144' (one full day). This should not happen anymore and can be removed in the near future. Until then we are going to log whenever this happens to be able to monitor this problem. - ## Configuration + During construction, instances of `GAENDataService` and `ValidationUtils` are needed. Further, any filter or modifier can be added to the list with `addFilter(KeyInsertionFilter filter)` or `addModifier(KeyInsertionModifier)`. Ideally, this happens inside the [`WSBaseConfig`](../config/WSBaseConfig.java), where default filters are added right after constructing the `InsertManager`. To allow for conditional `KeyInsertionFilters` or `KeyInsertionModifiers` refer to the following snippet: From d5639043598e50a1d9406a5a15a2831ca4d7d079 Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Tue, 8 Sep 2020 16:57:16 +0200 Subject: [PATCH 10/10] documenting configuration of modifiers --- .../org/dpppt/backend/sdk/ws/config/WSBaseConfig.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java index 53b27da3..df059418 100644 --- a/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java +++ b/dpppt-backend-sdk/dpppt-backend-sdk-ws/src/main/java/org/dpppt/backend/sdk/ws/config/WSBaseConfig.java @@ -238,6 +238,10 @@ public InsertManager insertManagerExposedNextDay() { return manager; } + /** + * Even though there are probably no android devices left that send TEKs with rollingPeriod of 0, + * this modifier will not hurt. Every TEK with rollingPeriod of 0 will be reported. + */ @ConditionalOnProperty( value = "ws.app.gaen.insertmanager.android0rpmodifier", havingValue = "true", @@ -249,6 +253,11 @@ public OldAndroid0RPModifier oldAndroid0RPModifier(InsertManager manager) { return androidModifier; } + /** + * This modifier will most probably not be enabled, as there should be very little iOS devices + * left that cannot handle a non-144 rollingPeriod key. Also, up to 8th of September 2020, Android + * did not release same day keys. + */ @ConditionalOnProperty( value = "ws.app.gaen.insertmanager.iosrplt144modifier", havingValue = "true",