[UID2-2005] Add new endpoint to check opt out status by raw UIDs (#557)

* Add new endpoint to check opt out status by raw UIDs

* Try loading optout deltas from local

* Revert "Try loading optout deltas from local"

This reverts commit d8e563a.

* Add a switch to disable endpoint and hashmap loading

* Add Snapshot store test for disabled status

* Add test cases for optout status endpoint processing

* Increase max request size to 5K. Refactor and update tests

* Update opt out status test

* Update default max size of opt out request

conf/default-config.json

@@ -17,6 +17,8 @@
   "optout_partition_interval": 86400,
   "optout_max_partitions": 30,
   "optout_heap_default_capacity": 8192,
+  "optout_status_api_enabled": false,
+  "optout_status_max_request_size": 5000,
   "cloud_download_threads": 8,
   "cloud_upload_threads": 2,
   "cloud_refresh_interval": 60,

conf/local-config.json

@@ -32,6 +32,7 @@
   "optout_heap_default_capacity": 8192,
   "optout_max_partitions": 30,
   "optout_partition_interval": 86400,
+  "optout_status_api_enabled": true,
   "client_side_token_generate": true,
   "client_side_token_generate_domain_name_check_enabled": true,
   "key_sharing_endpoint_provide_app_names": true,

conf/local-e2e-docker-public-config.json

@@ -31,6 +31,7 @@
   "optout_metadata_path": "/optout/refresh",
   "optout_api_uri": "http://optout:8081/optout/replicate",
   "optout_delta_rotate_interval": 60,
+  "optout_status_api_enabled": true,
   "cloud_refresh_interval": 30,
   "salts_expired_shutdown_hours": 12
 }

src/main/java/com/uid2/operator/Const.java

@@ -23,5 +23,7 @@ public class Config extends com.uid2.shared.Const.Config {
         public static final String AzureSecretNameProp = "azure_secret_name";
 
         public static final String GcpSecretVersionNameProp = "gcp_secret_version_name";
+        public static final String OptOutStatusApiEnabled = "optout_status_api_enabled";
+        public static final String OptOutStatusMaxRequestSize = "optout_status_max_request_size";
     }
 }

src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java

@@ -79,6 +79,11 @@ public Instant getLatestEntry(UserIdentity firstLevelHashIdentity) {
         return instant;
     }
 
+    @Override
+    public long getOptOutTimestampByAdId(String adId) {
+        return this.snapshot.get().getAdIdOptOutTimestamp(adId);
+    }
+
     @Override
     public void addEntry(UserIdentity firstLevelHashIdentity, byte[] advertisingId, Handler<AsyncResult<Instant>> handler) {
         if (remoteApiHost == null) {
@@ -344,6 +349,8 @@ public static class OptOutStoreSnapshot {
          */
         private final Map<String, Long> adIdToOptOutTimestamp;
 
+        private final boolean optoutStatusApiEnabled;
+
         // array of optout partitions
         private final OptOutPartition[] partitions;
 
@@ -373,6 +380,7 @@ public OptOutStoreSnapshot(DownloadCloudStorage fsLocal, JsonObject jsonConfig,
             this.heap = new OptOutHeap(heapCapacity);
 
             this.adIdToOptOutTimestamp = Collections.emptyMap();
+            this.optoutStatusApiEnabled = jsonConfig.getBoolean(Const.Config.OptOutStatusApiEnabled, false);
 
             // initially 1 partition
             this.partitions = new OptOutPartition[1];
@@ -384,7 +392,8 @@ public OptOutStoreSnapshot(DownloadCloudStorage fsLocal, JsonObject jsonConfig,
         }
 
         public OptOutStoreSnapshot(OptOutStoreSnapshot last, BloomFilter bf, OptOutHeap heap,
-                                   OptOutPartition[] newPartitions, IndexUpdateContext iuc) {
+                                   OptOutPartition[] newPartitions, IndexUpdateContext iuc,
+                                   boolean optoutStatusApiEnabled) {
             this.clock = last.clock;
             this.fsLocal = last.fsLocal;
             this.fileUtils = last.fileUtils;
@@ -400,14 +409,19 @@ public OptOutStoreSnapshot(OptOutStoreSnapshot last, BloomFilter bf, OptOutHeap
             newIndexedFiles.addAll(iuc.loadedPartitions.keySet());
             this.indexedFiles = Collections.unmodifiableSet(newIndexedFiles);
 
-            HashMap<String, Long> newOptOutTimestamps = new HashMap<>();
-            for (OptOutPartition partition : this.partitions) {
-                if (partition == null) continue;
-                partition.forEach(entry -> {
-                    newOptOutTimestamps.merge(entry.advertisingIdToB64(), entry.timestamp, OPT_OUT_TIMESTAMP_MERGE_STRATEGY);
-                });
+            this.optoutStatusApiEnabled = optoutStatusApiEnabled;
+            if (this.optoutStatusApiEnabled) {
+                HashMap<String, Long> newOptOutTimestamps = new HashMap<>();
+                for (OptOutPartition partition : this.partitions) {
+                    if (partition == null) continue;
+                    partition.forEach(entry -> {
+                        newOptOutTimestamps.merge(entry.advertisingIdToB64(), entry.timestamp, OPT_OUT_TIMESTAMP_MERGE_STRATEGY);
+                    });
+                }
+                this.adIdToOptOutTimestamp = Collections.unmodifiableMap(newOptOutTimestamps);
+            } else {
+                this.adIdToOptOutTimestamp = Collections.emptyMap();
             }
-            this.adIdToOptOutTimestamp = Collections.unmodifiableMap(newOptOutTimestamps);
 
             // update total entries
             totalEntries.set(size());
@@ -587,7 +601,7 @@ private OptOutStoreSnapshot processDeltas(IndexUpdateContext iuc) {
             newPartitions[0] = this.heap.isEmpty() ? null : this.heap.toPartition(true);
 
             OptOutStoreSnapshot.bloomFilterSize.set(this.bloomFilter.size());
-            return new OptOutStoreSnapshot(this, this.bloomFilter, this.heap, newPartitions, iuc);
+            return new OptOutStoreSnapshot(this, this.bloomFilter, this.heap, newPartitions, iuc, this.optoutStatusApiEnabled);
         }
 
         private OptOutStoreSnapshot processPartitions(IndexUpdateContext iuc) {
@@ -637,7 +651,7 @@ private OptOutStoreSnapshot processPartitions(IndexUpdateContext iuc) {
 
             OptOutStoreSnapshot.bloomFilterSize.set(newBf.size());
             OptOutStoreSnapshot.bloomFilterMax.set(newBf.capacity());
-            return new OptOutStoreSnapshot(this, newBf, newHeap, newPartitions, iuc);
+            return new OptOutStoreSnapshot(this, newBf, newHeap, newPartitions, iuc, this.optoutStatusApiEnabled);
         }
 
         // used for finding files to feed to index

src/main/java/com/uid2/operator/store/IOptOutStore.java

@@ -15,5 +15,7 @@ public interface IOptOutStore {
      */
     Instant getLatestEntry(UserIdentity firstLevelHashIdentity);
 
+    long getOptOutTimestampByAdId(String adId);
+
     void addEntry(UserIdentity firstLevelHashIdentity, byte[] advertisingId, Handler<AsyncResult<Instant>> handler);
 }

src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java

@@ -101,6 +101,8 @@ public class UIDOperatorVerticle extends AbstractVerticle {
     private final Map<Tuple.Tuple3<String, OptoutCheckPolicy, String>, Counter> _tokenGeneratePolicyCounters = new HashMap<>();
     private final Map<String, Tuple.Tuple2<Counter, Counter>> _identityMapUnmappedIdentifiers = new HashMap<>();
     private final Map<String, Counter> _identityMapRequestWithUnmapped = new HashMap<>();
+
+    private final Map<String, DistributionSummary> optOutStatusCounters = new HashMap<>();
     private final IdentityScope identityScope;
     private final V2PayloadHandler v2PayloadHandler;
     private final boolean phoneSupport;
@@ -121,6 +123,9 @@ public class UIDOperatorVerticle extends AbstractVerticle {
     protected boolean keySharingEndpointProvideAppNames;
     protected Instant lastInvalidOriginProcessTime = Instant.now();
 
+    private final int optOutStatusMaxRequestSize;
+    private final boolean optOutStatusApiEnabled;
+
     public UIDOperatorVerticle(JsonObject config,
                                boolean clientSideTokenGenerate,
                                ISiteStore siteProvider,
@@ -168,6 +173,8 @@ public UIDOperatorVerticle(JsonObject config,
         this.allowClockSkewSeconds = config.getInteger(Const.Config.AllowClockSkewSecondsProp, 1800);
         this.maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, config.getInteger(Const.Config.SharingTokenExpiryProp));
         this.saltRetrievalResponseHandler = saltRetrievalResponseHandler;
+        this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, false);
+        this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000);
     }
 
     @Override
@@ -278,7 +285,11 @@ private void setupV2Routes(Router mainRouter, BodyHandler bodyHandler) {
                 rc -> v2PayloadHandler.handle(rc, this::handleKeysBidstream), Role.ID_READER));
         v2Router.post("/token/logout").handler(bodyHandler).handler(auth.handleV1(
                 rc -> v2PayloadHandler.handleAsync(rc, this::handleLogoutAsyncV2), Role.OPTOUT));
-
+        if (this.optOutStatusApiEnabled) {
+            v2Router.post("/optout/status").handler(bodyHandler).handler(auth.handleV1(
+                    rc -> v2PayloadHandler.handle(rc, this::handleOptoutStatus),
+                    Role.MAPPER, Role.SHARER, Role.ID_READER));
+        }
 
         if (this.clientSideTokenGenerate)
             v2Router.post("/token/client-generate").handler(bodyHandler).handler(this::handleClientSideTokenGenerate);
@@ -1678,6 +1689,74 @@ private void recordIdentityMapStatsForServiceLinks(RoutingContext rc, String api
         }
     }
 
+    private List<String> parseOptoutStatusRequestPayload(RoutingContext rc) {
+        final JsonObject requestObj = (JsonObject) rc.data().get("request");
+        if (requestObj == null) {
+            ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Invalid request body");
+            return null;
+        }
+        final JsonArray rawUidsJsonArray = requestObj.getJsonArray("advertising_ids");
+        if (rawUidsJsonArray == null) {
+            ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Required Parameter Missing: advertising_ids");
+            return null;
+        }
+        if (rawUidsJsonArray.size() > optOutStatusMaxRequestSize) {
+            ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Request payload is too large");
+            return null;
+        }
+        List<String> rawUID2sInputList = new ArrayList<>(rawUidsJsonArray.size());
+        for (int i = 0; i < rawUidsJsonArray.size(); ++i) {
+            rawUID2sInputList.add(rawUidsJsonArray.getString(i));
+        }
+        return rawUID2sInputList;
+    }
+
+    private void handleOptoutStatus(RoutingContext rc) {
+        try {
+            // Parse request to get list of raw UID2 strings
+            List<String> rawUID2sInput = parseOptoutStatusRequestPayload(rc);
+            if (rawUID2sInput == null) {
+                return;
+            }
+            final JsonArray optedOutJsonArray = new JsonArray();
+            for (String rawUId : rawUID2sInput) {
+                // Call opt out service to get timestamp of opted out identities
+                long timestamp = optOutStore.getOptOutTimestampByAdId(rawUId);
+                if (timestamp != -1) {
+                    JsonObject optOutJsonObj = new JsonObject();
+                    optOutJsonObj.put("advertising_id", rawUId);
+                    optOutJsonObj.put("opted_out_since", timestamp);
+                    optedOutJsonArray.add(optOutJsonObj);
+                }
+            }
+            // Create response and return
+            final JsonObject bodyJsonObj = new JsonObject();
+            bodyJsonObj.put("opted_out", optedOutJsonArray);
+            ResponseUtil.SuccessV2(rc, bodyJsonObj);
+            recordOptOutStatusEndpointStats(rc, rawUID2sInput.size(), optedOutJsonArray.size());
+        } catch (Exception e) {
+            ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc,
+                    "Unknown error while getting optout status", e);
+        }
+    }
+
+    private void recordOptOutStatusEndpointStats(RoutingContext rc, int inputCount, int optOutCount) {
+        String apiContact = getApiContact(rc);
+        DistributionSummary inputDistSummary = optOutStatusCounters.computeIfAbsent(apiContact, k -> DistributionSummary
+                .builder("uid2.operator.optout.status.input_size")
+                .description("number of UIDs received in request")
+                .tags("api_contact", apiContact)
+                .register(Metrics.globalRegistry));
+        inputDistSummary.record(inputCount);
+
+        DistributionSummary optOutDistSummary = optOutStatusCounters.computeIfAbsent(apiContact, k -> DistributionSummary
+                .builder("uid2.operator.optout.status.optout_size")
+                .description("number of UIDs that have opted out")
+                .tags("api_contact", apiContact)
+                .register(Metrics.globalRegistry));
+        optOutDistSummary.record(optOutCount);
+    }
+
     private RefreshResponse refreshIdentity(RoutingContext rc, String tokenStr) {
         final RefreshToken refreshToken;
         try {

src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java

@@ -42,7 +42,6 @@
 import io.vertx.ext.web.client.WebClient;
 import io.vertx.junit5.VertxExtension;
 import io.vertx.junit5.VertxTestContext;
-import org.apache.commons.collections4.CollectionUtils;
 import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -96,6 +95,8 @@ public class UIDOperatorVerticleTest {
     private static final String clientSideTokenGeneratePrivateKey = "UID2-Y-L-MEECAQAwEwYHKoZIzj0CAQYIKoZIzj0DAQcEJzAlAgEBBCBop1Dw/IwDcstgicr/3tDoyR3OIpgAWgw8mD6oTO+1ug==";
     private static final int clientSideTokenGenerateSiteId = 123;
 
+    private static final int optOutStatusMaxRequestSize = 1000;
+
     private AutoCloseable mocks;
     @Mock private ISiteStore siteProvider;
     @Mock private IClientKeyProvider clientKeyProvider;
@@ -159,6 +160,8 @@ private void setupConfig(JsonObject config) {
         config.put("client_side_token_generate_log_invalid_http_origins", true);
 
         config.put(Const.Config.AllowClockSkewSecondsProp, 3600);
+        config.put(Const.Config.OptOutStatusApiEnabled, true);
+        config.put(Const.Config.OptOutStatusMaxRequestSize, optOutStatusMaxRequestSize);
     }
 
     private static byte[] makeAesKey(String prefix) {
@@ -2115,6 +2118,103 @@ void identityMapBatchRequestTooLarge(String apiVersion, Vertx vertx, VertxTestCo
         send(apiVersion, vertx, apiVersion + "/identity/map", false, null, req, 413, json -> testContext.completeNow());
     }
 
+    private static Stream<Arguments> optOutStatusRequestData() {
+        List<String> rawUIDS = Arrays.asList("RUQbFozFwnmPVjDx8VMkk9vJoNXUJImKnz2h9RfzzM24",
+            "qAmIGxqLk_RhOtm4f1nLlqYewqSma8fgvjEXYnQ3Jr0K",
+            "r3wW2uvJkwmeFcbUwSeM6BIpGF8tX38wtPfVc4wYyo71",
+            "e6SA-JVAXnvk8F1MUtzsMOyWuy5Xqe15rLAgqzSGiAbz");
+        Map<String, Long> optedOutIdsCase1 = new HashMap<>();
+
+        optedOutIdsCase1.put(rawUIDS.get(0), Instant.now().minus(1, ChronoUnit.DAYS).getEpochSecond());
+        optedOutIdsCase1.put(rawUIDS.get(1), Instant.now().minus(2, ChronoUnit.DAYS).getEpochSecond());
+        optedOutIdsCase1.put(rawUIDS.get(2), -1L);
+        optedOutIdsCase1.put(rawUIDS.get(3), -1L);
+
+        Map<String, Long> optedOutIdsCase2 = new HashMap<>();
+        optedOutIdsCase2.put(rawUIDS.get(2), -1L);
+        optedOutIdsCase2.put(rawUIDS.get(3), -1L);
+        return Stream.of(
+            Arguments.arguments(optedOutIdsCase1, 2, Role.MAPPER),
+            Arguments.arguments(optedOutIdsCase1, 2, Role.ID_READER),
+            Arguments.arguments(optedOutIdsCase1, 2, Role.SHARER),
+            Arguments.arguments(optedOutIdsCase2, 0, Role.MAPPER)
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("optOutStatusRequestData")
+    void optOutStatusRequest(Map<String, Long> optedOutIds, int optedOutCount, Role role, Vertx vertx, VertxTestContext testContext) {
+        fakeAuth(126, role);
+        setupSalts();
+        setupKeys();
+
+        JsonArray rawUIDs = new JsonArray();
+        for (String rawUID2 : optedOutIds.keySet()) {
+            when(this.optOutStore.getOptOutTimestampByAdId(rawUID2)).thenReturn(optedOutIds.get(rawUID2));
+            rawUIDs.add(rawUID2);
+        }
+        JsonObject requestJson = new JsonObject();
+        requestJson.put("advertising_ids", rawUIDs);
+
+        send("v2", vertx, "v2/optout/status", false, null, requestJson, 200, respJson -> {
+            assertEquals("success", respJson.getString("status"));
+            JsonArray optOutJsonArray = respJson.getJsonObject("body").getJsonArray("opted_out");
+            assertEquals(optedOutCount, optOutJsonArray.size());
+            for (int i = 0; i < optOutJsonArray.size(); ++i) {
+                JsonObject optOutObject = optOutJsonArray.getJsonObject(i);
+                assertEquals(optedOutIds.get(optOutObject.getString("advertising_id")),
+                        optOutObject.getLong("opted_out_since"));
+            }
+            testContext.completeNow();
+        });
+    }
+
+    private static Stream<Arguments> optOutStatusValidationErrorData() {
+        // Test case 1
+        JsonArray rawUIDs = new JsonArray();
+
+        for (int i = 0; i <= optOutStatusMaxRequestSize; ++i) {
+            byte[] rawUid2Bytes = Random.getBytes(32);
+            rawUIDs.add(Utils.toBase64String(rawUid2Bytes));
+        }
+
+        JsonObject requestJson1 = new JsonObject();
+        requestJson1.put("advertising_ids", rawUIDs);
+        // Test case 2
+        JsonObject requestJson2 = new JsonObject();
+        requestJson2.put("advertising", rawUIDs);
+        return Stream.of(
+                Arguments.arguments(requestJson1, "Request payload is too large"),
+                Arguments.arguments(requestJson2, "Required Parameter Missing: advertising_ids")
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("optOutStatusValidationErrorData")
+    void optOutStatusValidationError(JsonObject requestJson, String errorMsg, Vertx vertx, VertxTestContext testContext) {
+        fakeAuth(126, Role.MAPPER);
+        setupSalts();
+        setupKeys();
+
+        send("v2", vertx, "v2/optout/status", false, null, requestJson, 400, respJson -> {
+            assertEquals(com.uid2.shared.Const.ResponseStatus.ClientError, respJson.getString("status"));
+            assertEquals(errorMsg, respJson.getString("message"));
+            testContext.completeNow();
+        });
+    }
+
+    @Test
+    void optOutStatusUnauthorized(Vertx vertx, VertxTestContext testContext) {
+        fakeAuth(126, Role.GENERATOR);
+        setupSalts();
+        setupKeys();
+
+        send("v2", vertx, "v2/optout/status", false, null, new JsonObject(), 401, respJson -> {
+            assertEquals(com.uid2.shared.Const.ResponseStatus.Unauthorized, respJson.getString("status"));
+            testContext.completeNow();
+        });
+    }
+
     @Test
     void LogoutV2(Vertx vertx, VertxTestContext testContext) {
         final int clientSiteId = 201;

src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java

@@ -197,6 +197,11 @@ public Instant getLatestEntry(UserIdentity firstLevelHashIdentity) {
         public void addEntry(UserIdentity firstLevelHashIdentity, byte[] advertisingId, Handler<AsyncResult<Instant>> handler) {
             // noop
         }
+
+        @Override
+        public long getOptOutTimestampByAdId(String adId) {
+            return -1;
+        }
     }
 
 }