From f3a34de248680139e3a0cac029644722e6ebd502 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 13 Feb 2025 14:51:25 +1100 Subject: [PATCH 1/7] Add RotatingRuntimeConfigStore - Retrieves operator remote config from core endpoint --- .../service/RotatingRuntimeConfigStore.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/main/java/com/uid2/operator/service/RotatingRuntimeConfigStore.java diff --git a/src/main/java/com/uid2/operator/service/RotatingRuntimeConfigStore.java b/src/main/java/com/uid2/operator/service/RotatingRuntimeConfigStore.java new file mode 100644 index 000000000..8045897d0 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/RotatingRuntimeConfigStore.java @@ -0,0 +1,41 @@ +package com.uid2.operator.service; + +import com.uid2.shared.Utils; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.reader.IMetadataVersionedStore; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; + +import java.io.InputStream; + +import static com.uid2.operator.Const.Config.OperatorRuntimeConfigEventBus; + +public class RotatingRuntimeConfigStore implements IMetadataVersionedStore { + private final DownloadCloudStorage metadataStreamProvider; + private final String runtimeConfigPath; + private final Vertx vertx; + + public RotatingRuntimeConfigStore(Vertx vertx, DownloadCloudStorage metadataStreamProvider, String runtimeConfigPath) { + this.metadataStreamProvider = metadataStreamProvider; + this.runtimeConfigPath = runtimeConfigPath; + this.vertx = vertx; + } + + @Override + public JsonObject getMetadata() throws Exception { + try (InputStream s = this.metadataStreamProvider.download(this.runtimeConfigPath)) { + return Utils.toJsonObject(s); + } + } + + @Override + public long getVersion(JsonObject jsonObject) { + return jsonObject.getLong("version"); + } + + @Override + public long loadContent(JsonObject jsonObject) throws Exception { + vertx.eventBus().publish(OperatorRuntimeConfigEventBus, jsonObject); + return 1; + } +} From a9b3726e387d6edb684a87abd606670e690db666 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 13 Feb 2025 14:51:39 +1100 Subject: [PATCH 2/7] Add RotatingRuntimeConfigStoreTest --- .../RotatingRuntimeConfigStoreTest.java | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/test/java/com/uid2/operator/RotatingRuntimeConfigStoreTest.java diff --git a/src/test/java/com/uid2/operator/RotatingRuntimeConfigStoreTest.java b/src/test/java/com/uid2/operator/RotatingRuntimeConfigStoreTest.java new file mode 100644 index 000000000..7e6f35bce --- /dev/null +++ b/src/test/java/com/uid2/operator/RotatingRuntimeConfigStoreTest.java @@ -0,0 +1,77 @@ +package com.uid2.operator; + +import com.uid2.operator.service.RotatingRuntimeConfigStore; +import com.uid2.shared.cloud.EmbeddedResourceStorage; +import com.uid2.shared.cloud.ICloudStorage; +import io.vertx.core.Vertx; +import io.vertx.core.eventbus.EventBus; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +import static com.uid2.operator.Const.Config.OperatorRuntimeConfigEventBus; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RotatingRuntimeConfigStoreTest { + private AutoCloseable mocks; + @Mock + private ICloudStorage metadataStreamProvider; + @Mock + private Vertx vertx; + @Mock + private EventBus eventBus; + + private RotatingRuntimeConfigStore rotatingRuntimeConfigStore; + private final String runtimeConfigPath = "/com.uid2.core/test/runtime_config/metadata.json"; + + @BeforeEach + public void setup() { + mocks = MockitoAnnotations.openMocks(this); + when(vertx.eventBus()).thenReturn(eventBus); + rotatingRuntimeConfigStore = new RotatingRuntimeConfigStore(vertx, metadataStreamProvider, runtimeConfigPath); + } + + @AfterEach + public void teardown() throws Exception { + mocks.close(); + } + + @Test + public void testGetMetadata() throws Exception { + JsonObject expectedMetadata = new JsonObject().put("key", "value"); + when(metadataStreamProvider.download(runtimeConfigPath)) + .thenReturn(new ByteArrayInputStream(expectedMetadata.toString().getBytes(StandardCharsets.US_ASCII))); + JsonObject actualMetadata = rotatingRuntimeConfigStore.getMetadata(); + assertEquals(expectedMetadata, actualMetadata); + } + + @Test + public void testGetVersion() { + JsonObject jsonObject = new JsonObject().put("version", 123L); + long version = rotatingRuntimeConfigStore.getVersion(jsonObject); + assertEquals(123L, version); + } + + @Test + public void testLoadContent() throws Exception { + JsonObject jsonObject = new JsonObject().put("key", "value"); + long result = rotatingRuntimeConfigStore.loadContent(jsonObject); + verify(eventBus).publish(OperatorRuntimeConfigEventBus, jsonObject); + assertEquals(1L, result); + } + + @Test + public void loadFromEmbeddedResourceStorage() throws Exception { + rotatingRuntimeConfigStore = new RotatingRuntimeConfigStore(vertx, new EmbeddedResourceStorage(Main.class), runtimeConfigPath); + JsonObject m = rotatingRuntimeConfigStore.getMetadata(); + rotatingRuntimeConfigStore.loadContent(m); + } +} From be5b6f2ac33d1e358efb838854487c40480094ec Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 13 Feb 2025 14:54:59 +1100 Subject: [PATCH 3/7] Update Main to create RotatingRuntimeConfigStore - updated local and integ config - moved runtime-config-defaults.json to runtime_config/metadata.json in com.uid2.core - moved createStoreVerticles method to complete before initialiseConfigService --- conf/integ-config.json | 11 +----- conf/local-config.json | 10 +---- src/main/java/com/uid2/operator/Main.java | 38 +++++++++++-------- .../test/runtime_config/metadata.json | 1 + 4 files changed, 28 insertions(+), 32 deletions(-) rename conf/runtime-config-defaults.json => src/main/resources/com.uid2.core/test/runtime_config/metadata.json (92%) diff --git a/conf/integ-config.json b/conf/integ-config.json index 8f7fa2385..72905e5e5 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -15,13 +15,6 @@ "optout_api_uri": "http://localhost:8081/optout/replicate", "salts_expired_shutdown_hours": 12, "operator_type": "public", - "runtime_config_store": { - "type": "http", - "config" : { - "host": "localhost", - "port": 8088, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "http://localhost:8088/operator/config", + "runtime_config_refresh_ms": 5000 } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index 6a91eecb2..0ceb9bad9 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -37,12 +37,6 @@ "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, "operator_type": "public", - "runtime_config_store": { - "type": "file", - "config" : { - "path": "conf/runtime-config-defaults.json", - "format": "json" - }, - "config_scan_period_ms": 5000 - } + "runtime_config_path": "/com.uid2.core/test/runtime_config/metadata.json", + "runtime_config_refresh_ms": 5000 } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index bc285d4fa..eeed823c6 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -60,6 +60,7 @@ import java.util.function.Supplier; import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp; +import static com.uid2.operator.Const.Config.OperatorRuntimeConfigEventBus; import static io.micrometer.core.instrument.Metrics.globalRegistry; public class Main { @@ -77,6 +78,7 @@ public class Main { private final RotatingClientSideKeypairStore clientSideKeypairProvider; private final RotatingSaltProvider saltProvider; private final CloudSyncOptOutStore optOutStore; + private final RotatingRuntimeConfigStore runtimeConfigStore; private OperatorShutdownHandler shutdownHandler = null; private final OperatorMetrics metrics; private final boolean clientSideTokenGenerate; @@ -146,6 +148,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); + String runtimeConfigMdPath = this.config.getString("runtime_config_path"); + this.runtimeConfigStore = new RotatingRuntimeConfigStore(vertx, fsStores, runtimeConfigMdPath); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); if (this.validateServiceLinks) { @@ -269,14 +273,9 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } } - private Future initialiseConfigService() throws Exception { + private Future initialiseConfigService(ConfigRetriever dynamicConfigRetriever) throws Exception { Promise promise = Promise.promise(); - ConfigRetriever dynamicConfigRetriever = ConfigRetrieverFactory.create( - vertx, - config.getJsonObject("runtime_config_store"), - this.createOperatorKeyRetriever().retrieve() - ); Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); ConfigRetriever staticConfigRetriever = ConfigRetrieverFactory.create( @@ -284,8 +283,7 @@ private Future initialiseConfigService() throws Exception { new JsonObject() .put("type", "json") .put("config", config) - .put(ConfigScanPeriodMsProp, -1), - "" + .put(ConfigScanPeriodMsProp, -1) ); Future staticConfigFuture = ConfigService.create(staticConfigRetriever); @@ -337,9 +335,23 @@ private void run() throws Exception { this.createVertxInstancesMetric(); this.createVertxEventLoopsMetric(); - this.initialiseConfigService() - .compose(configService -> { + ConfigRetriever dynamicConfigRetriever = ConfigRetrieverFactory.create( + vertx, + new JsonObject() + .put("type", "event-bus") + .put("config", new JsonObject() + .put("address", OperatorRuntimeConfigEventBus)) + ); + this.createStoreVerticles() + .compose(v -> { + try { + return this.initialiseConfigService(dynamicConfigRetriever); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .compose(configService -> { Supplier operatorVerticleSupplier = () -> { UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; @@ -352,11 +364,6 @@ private void run() throws Exception { Promise compositePromise = Promise.promise(); List fs = new ArrayList<>(); fs.add(createAndDeployStatsCollector()); - try { - fs.add(createStoreVerticles()); - } catch (Exception e) { - throw new RuntimeException(e); - } CompositeFuture.all(fs).onComplete(ar -> { if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); @@ -424,6 +431,7 @@ private Future createStoreVerticles() throws Exception { fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("runtime_config", runtimeConfigStore, "runtime_config_refresh_ms")); fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync)); CompositeFuture.all(fs).onComplete(ar -> { if (ar.failed()) promise.fail(new Exception(ar.cause())); diff --git a/conf/runtime-config-defaults.json b/src/main/resources/com.uid2.core/test/runtime_config/metadata.json similarity index 92% rename from conf/runtime-config-defaults.json rename to src/main/resources/com.uid2.core/test/runtime_config/metadata.json index 817d714dd..1bf70dfcc 100644 --- a/conf/runtime-config-defaults.json +++ b/src/main/resources/com.uid2.core/test/runtime_config/metadata.json @@ -1,4 +1,5 @@ { + "version": 1, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, From 66a0661da7de04a12666865475f29e57827703a8 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 13 Feb 2025 14:56:01 +1100 Subject: [PATCH 4/7] Update ConfigServiceTest and ConfigRetrieverFactory to work with new Rotating runtime config store --- Dockerfile | 1 - src/main/java/com/uid2/operator/Const.java | 1 + .../service/ConfigRetrieverFactory.java | 8 +- .../com/uid2/operator/ConfigServiceTest.java | 80 ++++--------------- 4 files changed, 20 insertions(+), 70 deletions(-) diff --git a/Dockerfile b/Dockerfile index fa7d23d29..2ab5c5de3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,6 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz COPY ./conf/default-config.json ${EXTRA_CONFIG} /app/conf/ COPY ./conf/*.xml /app/conf/ -COPY ./conf/runtime-config-defaults.json /app/conf/ COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index c7d7f3dda..1504826e8 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -32,5 +32,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String ConfigScanPeriodMsProp = "config_scan_period_ms"; public static final String IdentityV3Prop = "identity_v3"; + public static final String OperatorRuntimeConfigEventBus = "operator.runtime.config"; } } diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 0df88955b..d99cb62f9 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -9,20 +9,16 @@ import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp; public class ConfigRetrieverFactory { - public static ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig, String operatorKey) { + public static ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { String type = bootstrapConfig.getString("type"); JsonObject storeConfig = bootstrapConfig.getJsonObject("config"); - if (type.equals("http")) { - storeConfig.put("headers", new JsonObject() - .put("Authorization", "Bearer " + operatorKey)); - } ConfigStoreOptions storeOptions = new ConfigStoreOptions() .setType(type) .setConfig(storeConfig); ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() - .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMsProp)) + .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMsProp, 5000L)) .addStore(storeOptions); return ConfigRetriever.create(vertx, retrieverOptions); diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index b1b448b02..75c3527e8 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -3,14 +3,9 @@ import com.uid2.operator.service.ConfigRetrieverFactory; import com.uid2.operator.service.ConfigService; import io.vertx.core.Future; -import io.vertx.core.Promise; import io.vertx.core.Vertx; -import io.vertx.core.http.HttpHeaders; -import io.vertx.core.http.HttpServer; import io.vertx.core.json.JsonObject; import io.vertx.config.ConfigRetriever; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.handler.BodyHandler; import org.junit.jupiter.api.*; import io.vertx.junit5.VertxExtension; import io.vertx.junit5.VertxTestContext; @@ -26,18 +21,11 @@ class ConfigServiceTest { private Vertx vertx; private JsonObject bootstrapConfig; private JsonObject runtimeConfig; - private HttpServer server; + private JsonObject invalidBootstrapConfig; @BeforeEach void setUp() { vertx = Vertx.vertx(); - bootstrapConfig = new JsonObject() - .put("type", "http") - .put("config", new JsonObject() - .put("host", "localhost") - .put("port", 8088) - .put("path", "/operator/config")) - .put(ConfigScanPeriodMsProp, 300000); runtimeConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) @@ -46,52 +34,32 @@ void setUp() { .put(MaxBidstreamLifetimeSecondsProp, 7200) .put(SharingTokenExpiryProp, 3600); + JsonObject invalidConfig = new JsonObject() + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); + + bootstrapConfig = new JsonObject() + .put("type", "json") + .put("config", runtimeConfig); + + invalidBootstrapConfig = new JsonObject() + .put("type", "json") + .put("config", invalidConfig); } @AfterEach void tearDown() { - if (server != null) { - server.close(); - } vertx.close(); } - private Future startMockServer(JsonObject config) { - if (server != null) { - server.close(); - } - - Promise promise = Promise.promise(); - - Router router = Router.router(vertx); - router.route().handler(BodyHandler.create()); - router.get("/operator/config").handler(ctx -> ctx.response() - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(config.encode())); - - server = vertx.createHttpServer() - .requestHandler(router) - .listen(Const.Port.ServicePortForCore,"127.0.0.1", http -> { - if (http.succeeded()) { - promise.complete(); - } else { - promise.fail(http.cause()); - } - }); - - return promise.future(); - } - @Test void testGetConfig(VertxTestContext testContext) { - ConfigRetriever configRetriever = ConfigRetrieverFactory.create(vertx, bootstrapConfig, ""); - JsonObject httpStoreConfig = runtimeConfig; - startMockServer(httpStoreConfig) - .compose(v -> ConfigService.create(configRetriever)) + ConfigRetriever configRetriever = ConfigRetrieverFactory.create(vertx, bootstrapConfig); + ConfigService.create(configRetriever) .compose(configService -> { JsonObject retrievedConfig = configService.getConfig(); assertNotNull(retrievedConfig, "Config retriever should initialise without error"); - assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); + assertTrue(retrievedConfig.fieldNames().containsAll(runtimeConfig.fieldNames()), "Retrieved config should contain all keys in runtime config"); return Future.succeededFuture(); }) .onComplete(testContext.succeedingThenComplete()); @@ -100,14 +68,7 @@ void testGetConfig(VertxTestContext testContext) { @Test void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { JsonObject lastConfig = new JsonObject().put("previous", "config"); - JsonObject invalidConfig = new JsonObject() - .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) - .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - JsonObject jsonBootstrapConfig = new JsonObject() - .put("type", "json") - .put("config", invalidConfig) - .put(ConfigScanPeriodMsProp, -1); - ConfigRetriever spyRetriever = spy(ConfigRetrieverFactory.create(vertx, jsonBootstrapConfig, "")); + ConfigRetriever spyRetriever = spy(ConfigRetrieverFactory.create(vertx, invalidBootstrapConfig)); when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); ConfigService.create(spyRetriever) .compose(configService -> { @@ -120,14 +81,7 @@ void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { @Test void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) { - JsonObject invalidConfig = new JsonObject() - .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) - .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - JsonObject jsonBootstrapConfig = new JsonObject() - .put("type", "json") - .put("config", invalidConfig) - .put(ConfigScanPeriodMsProp, -1); - ConfigRetriever configRetriever = ConfigRetrieverFactory.create(vertx, jsonBootstrapConfig, ""); + ConfigRetriever configRetriever = ConfigRetrieverFactory.create(vertx, invalidBootstrapConfig); ConfigService.create(configRetriever) .onComplete(testContext.failing(throwable -> { assertThrows(RuntimeException.class, () -> { From 83ad5474e184e059aed7fa2be3a24d623f9ac95f Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 13 Feb 2025 15:13:23 +1100 Subject: [PATCH 5/7] Update config files with new runtime_config_path and runtime_config_refresh_ms values --- conf/default-config.json | 4 +++- conf/docker-config.json | 10 ++-------- conf/integ-config.json | 2 +- conf/local-config.json | 2 +- conf/local-e2e-docker-private-config.json | 11 ++--------- conf/local-e2e-docker-public-config.json | 11 ++--------- conf/local-e2e-private-config.json | 11 ++--------- conf/local-e2e-public-config.json | 11 ++--------- conf/validator-latest-e2e-docker-public-config.json | 11 ++--------- scripts/aws/conf/euid-integ-config.json | 12 ++---------- scripts/aws/conf/euid-prod-config.json | 12 ++---------- scripts/aws/conf/uid2-integ-config.json | 12 ++---------- scripts/aws/conf/uid2-prod-config.json | 12 ++---------- scripts/azure-cc/conf/integ-uid2-config.json | 12 ++---------- scripts/azure-cc/conf/prod-uid2-config.json | 12 ++---------- scripts/gcp-oidc/conf/integ-config.json | 12 ++---------- scripts/gcp-oidc/conf/prod-config.json | 12 ++---------- 17 files changed, 33 insertions(+), 136 deletions(-) diff --git a/conf/default-config.json b/conf/default-config.json index a57739a7f..77028486c 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -35,5 +35,7 @@ "enclave_platform": null, "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, - "operator_type": "public" + "operator_type": "public", + "runtime_config_path": "runtime_config/metadata.json", + "runtime_config_refresh_ms": 300000 } diff --git a/conf/docker-config.json b/conf/docker-config.json index 17266d2bd..fd5f5f010 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -38,12 +38,6 @@ "failure_shutdown_wait_hours": 120, "salts_expired_shutdown_hours": 12, "operator_type": "public", - "runtime_config_store": { - "type": "file", - "config" : { - "path": "conf/runtime-config-defaults.json", - "format": "json" - }, - "config_scan_period_ms": 5000 - } + "runtime_config_path": "/com.uid2.core/test/runtime_config/metadata.json", + "runtime_config_refresh_ms": 300000 } diff --git a/conf/integ-config.json b/conf/integ-config.json index 72905e5e5..22ec6fddb 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -16,5 +16,5 @@ "salts_expired_shutdown_hours": 12, "operator_type": "public", "runtime_config_path": "http://localhost:8088/operator/config", - "runtime_config_refresh_ms": 5000 + "runtime_config_refresh_ms": 300000 } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index 0ceb9bad9..a8d6fcbe4 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -38,5 +38,5 @@ "salts_expired_shutdown_hours": 12, "operator_type": "public", "runtime_config_path": "/com.uid2.core/test/runtime_config/metadata.json", - "runtime_config_refresh_ms": 5000 + "runtime_config_refresh_ms": 300000 } diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index 218f64e2c..f9eecc5e7 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -28,13 +28,6 @@ "cloud_refresh_interval": 30, "salts_expired_shutdown_hours": 12, "operator_type": "private", - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core", - "port": 8088, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "http://core:8088/operator/config", + "runtime_config_refresh_ms": 300000 } diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 3357a2b85..565b3a060 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -34,13 +34,6 @@ "cloud_refresh_interval": 30, "salts_expired_shutdown_hours": 12, "operator_type": "public", - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core", - "port": 8088, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "http://core:8088/operator/config", + "runtime_config_refresh_ms": 300000 } diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 00d94d552..1f9cf8400 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -39,13 +39,6 @@ "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, "operator_type": "private", - "runtime_config_store": { - "type": "http", - "config" : { - "host": "localhost", - "port": 8088, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "http://localhost:8088/operator/config", + "runtime_config_refresh_ms": 300000 } diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index edd7af5cb..049a32328 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -40,13 +40,6 @@ "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, "operator_type": "public", - "runtime_config_store": { - "type": "http", - "config" : { - "host": "localhost", - "port": 8088, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "http://localhost:8088/operator/config", + "runtime_config_refresh_ms": 300000 } diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index f9b201ee8..c9ac87dce 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -33,13 +33,6 @@ "optout_delta_rotate_interval": 60, "cloud_refresh_interval": 30, "operator_type": "public", - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core", - "port": 8088, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "http://core:8088/operator/config", + "runtime_config_refresh_ms": 300000 } diff --git a/scripts/aws/conf/euid-integ-config.json b/scripts/aws/conf/euid-integ-config.json index 66a82183e..b718c90c2 100644 --- a/scripts/aws/conf/euid-integ-config.json +++ b/scripts/aws/conf/euid-integ-config.json @@ -12,14 +12,6 @@ "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", "optout_s3_folder": "optout/", "allow_legacy_api": false, - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core.integ.euid.eu", - "port": 443, - "ssl": true, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "https://core.integ.euid.eu/operator/config", + "runtime_config_refresh_ms": 300000 } \ No newline at end of file diff --git a/scripts/aws/conf/euid-prod-config.json b/scripts/aws/conf/euid-prod-config.json index ac00370bc..8537ae52b 100644 --- a/scripts/aws/conf/euid-prod-config.json +++ b/scripts/aws/conf/euid-prod-config.json @@ -28,14 +28,6 @@ "enable_phone_support": true, "enable_v1_phone_support": false, "enable_v2_encryption": true, - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core.prod.euid.eu", - "port": 443, - "ssl": true, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "https://core.prod.euid.eu/operator/config", + "runtime_config_refresh_ms": 300000 } \ No newline at end of file diff --git a/scripts/aws/conf/uid2-integ-config.json b/scripts/aws/conf/uid2-integ-config.json index 4daf69cf7..9d2f09a2c 100644 --- a/scripts/aws/conf/uid2-integ-config.json +++ b/scripts/aws/conf/uid2-integ-config.json @@ -12,14 +12,6 @@ "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", "allow_legacy_api": false, - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core-integ.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "https://core-integ.uidapi.com/operator/config", + "runtime_config_refresh_ms": 300000 } \ No newline at end of file diff --git a/scripts/aws/conf/uid2-prod-config.json b/scripts/aws/conf/uid2-prod-config.json index a46670746..aea212b4f 100644 --- a/scripts/aws/conf/uid2-prod-config.json +++ b/scripts/aws/conf/uid2-prod-config.json @@ -23,14 +23,6 @@ "refresh_token_expires_after_seconds": 2592000, "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false, - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core-prod.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "https://core-prod.uidapi.com/operator/config", + "runtime_config_refresh_ms": 300000 } \ No newline at end of file diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json index d1b697ad7..bc6ba3c51 100644 --- a/scripts/azure-cc/conf/integ-uid2-config.json +++ b/scripts/azure-cc/conf/integ-uid2-config.json @@ -11,14 +11,6 @@ "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core-integ.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "https://core-integ.uidapi.com/operator/config", + "runtime_config_refresh_ms": 300000 } diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index 1305fffcf..9e076b29b 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -12,14 +12,6 @@ "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", "identity_token_expires_after_seconds": 259200, - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core-prod.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "https://core-prod.uidapi.com/operator/config", + "runtime_config_refresh_ms": 300000 } diff --git a/scripts/gcp-oidc/conf/integ-config.json b/scripts/gcp-oidc/conf/integ-config.json index 6876be36d..c5a2b5fd3 100644 --- a/scripts/gcp-oidc/conf/integ-config.json +++ b/scripts/gcp-oidc/conf/integ-config.json @@ -11,14 +11,6 @@ "core_attest_url": "https://core.uidapi.com/attest", "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "https://core.uidapi.com/operator/config", + "runtime_config_refresh_ms": 300000 } \ No newline at end of file diff --git a/scripts/gcp-oidc/conf/prod-config.json b/scripts/gcp-oidc/conf/prod-config.json index 82c3a0d81..b89f3356c 100644 --- a/scripts/gcp-oidc/conf/prod-config.json +++ b/scripts/gcp-oidc/conf/prod-config.json @@ -12,14 +12,6 @@ "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", "identity_token_expires_after_seconds": 259200, - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } + "runtime_config_path": "https://core.uidapi.com/operator/config", + "runtime_config_refresh_ms": 300000 } From 0d6eae2c2d43352de51187b6f9ea76932675ca7e Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 13 Feb 2025 17:03:23 +1100 Subject: [PATCH 6/7] Add RuntimeConfigPathProp Const --- src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 1504826e8..b783aadea 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -33,5 +33,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String ConfigScanPeriodMsProp = "config_scan_period_ms"; public static final String IdentityV3Prop = "identity_v3"; public static final String OperatorRuntimeConfigEventBus = "operator.runtime.config"; + public static String RuntimeConfigPathProp = "runtime_config_path"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index eeed823c6..0eabc0d4a 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -148,7 +148,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); - String runtimeConfigMdPath = this.config.getString("runtime_config_path"); + String runtimeConfigMdPath = this.config.getString(Const.Config.RuntimeConfigPathProp); this.runtimeConfigStore = new RotatingRuntimeConfigStore(vertx, fsStores, runtimeConfigMdPath); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); From d5e1d4b0639b111f2c28a6e290db99d57035b2df Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 13 Feb 2025 17:51:56 +1100 Subject: [PATCH 7/7] Remove unnecessary exception throw --- src/main/java/com/uid2/operator/Main.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 0eabc0d4a..cc9842b3b 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -273,7 +273,7 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } } - private Future initialiseConfigService(ConfigRetriever dynamicConfigRetriever) throws Exception { + private Future initialiseConfigService(ConfigRetriever dynamicConfigRetriever) { Promise promise = Promise.promise(); Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); @@ -344,13 +344,7 @@ private void run() throws Exception { ); this.createStoreVerticles() - .compose(v -> { - try { - return this.initialiseConfigService(dynamicConfigRetriever); - } catch (Exception e) { - throw new RuntimeException(e); - } - }) + .compose(v -> this.initialiseConfigService(dynamicConfigRetriever)) .compose(configService -> { Supplier operatorVerticleSupplier = () -> { UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse);