Handle child to parent broker migration

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift

@@ -25,6 +25,13 @@ struct DataBrokerScheduleConfig: Codable {
     let confirmOptOutScan: Int
     let maintenanceScan: Int
     let maxAttempts: Int
+
+    // Used when scheduling the subsequent opt-out attempt following a successful opt-out request submission
+    // This value should be less than `confirmOptOutScan` to ensure the next attempt occurs before
+    // the confirmation scan.
+    var hoursUntilNextOptOutAttempt: Int {
+        maintenanceScan
+    }
 }
 
 extension Int {

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift

@@ -115,7 +115,7 @@ class DataBrokerOperation: Operation, @unchecked Sendable {
         }
     }
 
-    private func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] {
+    static func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] {
         let operationsData: [BrokerJobData]
 
         switch operationType {
@@ -131,8 +131,8 @@ class DataBrokerOperation: Operation, @unchecked Sendable {
 
         if let priorityDate = priorityDate {
             filteredAndSortedOperationsData = operationsData
-                .filter { $0.preferredRunDate != nil && $0.preferredRunDate! <= priorityDate }
-                .sorted { $0.preferredRunDate! < $1.preferredRunDate! }
+                .eligibleForRun(byDate: priorityDate)
+                .sortedByPreferredRunDate()
         } else {
             filteredAndSortedOperationsData = operationsData
         }
@@ -152,9 +152,9 @@ class DataBrokerOperation: Operation, @unchecked Sendable {
 
         let brokerProfileQueriesData = allBrokerProfileQueryData.filter { $0.dataBroker.id == dataBrokerID }
 
-        let filteredAndSortedOperationsData = filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData,
-                                                                          operationType: operationType,
-                                                                          priorityDate: priorityDate)
+        let filteredAndSortedOperationsData = Self.filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData,
+                                                                               operationType: operationType,
+                                                                               priorityDate: priorityDate)
 
         Logger.dataBrokerProtection.log("filteredAndSortedOperationsData count: \(filteredAndSortedOperationsData.count, privacy: .public) for brokerID \(self.dataBrokerID, privacy: .public)")
 
@@ -215,3 +215,40 @@ class DataBrokerOperation: Operation, @unchecked Sendable {
     }
 }
 // swiftlint:enable explicit_non_final_class
+
+extension Array where Element == BrokerJobData {
+    /// Filters jobs based on their preferred run date:
+    /// - Opt-out jobs with no preferred run date are included.
+    /// - Jobs with a preferred run date on or before the priority date are included.
+    ///
+    /// Note: Opt-out jobs without a preferred run date may be:
+    /// 1. From child brokers (will be skipped during runOptOutOperation).
+    /// 2. From former child brokers now acting as parent brokers (will be processed if extractedProfile hasn't been removed).
+    func eligibleForRun(byDate priorityDate: Date) -> [BrokerJobData] {
+        filter { jobData in
+            guard let preferredRunDate = jobData.preferredRunDate else {
+                return jobData is OptOutJobData
+            }
+
+            return preferredRunDate <= priorityDate
+        }
+    }
+
+    /// Sorts BrokerJobData array based on their preferred run dates.
+    /// - Jobs with non-nil preferred run dates are sorted in ascending order (earliest date first).
+    /// - Opt-out jobs with nil preferred run dates come last, maintaining their original relative order.
+    func sortedByPreferredRunDate() -> [BrokerJobData] {
+        sorted { lhs, rhs in
+            switch (lhs.preferredRunDate, rhs.preferredRunDate) {
+            case (nil, nil):
+                return false
+            case (_, nil):
+                return true
+            case (nil, _):
+                return false
+            case (let lhsRunDate?, let rhsRunDate?):
+                return lhsRunDate < rhsRunDate
+            }
+        }
+    }
+}

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift

@@ -291,11 +291,11 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager {
         }
 
         guard extractedProfile.removedDate == nil else {
-            Logger.dataBrokerProtection.log("Profile already extracted, skipping...")
+            Logger.dataBrokerProtection.log("Profile already removed, skipping...")
             return
         }
 
-        guard let optOutStep = brokerProfileQueryData.dataBroker.optOutStep(), optOutStep.optOutType != .parentSiteOptOut else {
+        guard !brokerProfileQueryData.dataBroker.performsOptOutWithinParent() else {
             Logger.dataBrokerProtection.log("Broker opts out in parent, skipping...")
             return
         }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift

@@ -81,8 +81,14 @@ struct OperationPreferredDateCalculator {
             return date.now.addingTimeInterval(calculateNextRunDateOnError(schedulingConfig: schedulingConfig, historyEvents: historyEvents))
         case .optOutStarted, .scanStarted, .noMatchFound:
             return currentPreferredRunDate
-        case .optOutConfirmed, .optOutRequested:
+        case .optOutConfirmed:
             return nil
+        case .optOutRequested:
+            // Previously, opt-out jobs with `nil` preferredRunDate were never executed,
+            // but we need this following the child-to-parent-broker transition
+            // to prevent repeated scheduling of those former child broker opt-out jobs.
+            // https://app.asana.com/0/0/1208832818650310/f
+            return date.now.addingTimeInterval(schedulingConfig.hoursUntilNextOptOutAttempt.hoursToSeconds)
         }
     }
 

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift

@@ -172,6 +172,10 @@ extension Date {
     static func nowMinus(hours: Int) -> Date {
         Calendar.current.date(byAdding: .hour, value: -hours, to: Date()) ?? Date()
     }
+
+    static func nowPlus(hours: Int) -> Date {
+        nowMinus(hours: -hours)
+    }
 }
 
 final class DataBrokerProtectionStatsPixels: StatsPixels {

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift

@@ -40,7 +40,7 @@ struct MapperToUI {
             brokerQueryGroup.scannedBrokers
         }
 
-        let scanProgress = DBPUIScanProgress(currentScans: partiallyScannedBrokers.currentScans,
+        let scanProgress = DBPUIScanProgress(currentScans: partiallyScannedBrokers.completeBrokerScansCount,
                                              totalScans: totalScans,
                                              scannedBrokers: partiallyScannedBrokers)
 
@@ -452,8 +452,7 @@ fileprivate extension Array where Element == BrokerProfileQueryData {
 }
 
 extension Array where Element == DBPUIScanProgress.ScannedBroker {
-    /// Number of completed broker scans
-    var currentScans: Int {
+    var completeBrokerScansCount: Int {
         reduce(0) { accumulator, scannedBrokers in
             scannedBrokers.status == .completed ? accumulator + 1 : accumulator
         }

LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift

@@ -0,0 +1,99 @@
+//
+//  DataBrokerOperationTests.swift
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+@testable import DataBrokerProtection
+import XCTest
+
+final class DataBrokerOperationTests: XCTestCase {
+    lazy var mockOptOutQueryData: [BrokerProfileQueryData] = {
+        let brokerId: Int64 = 1
+
+        let mockNilPreferredRunDateQueryData = Array(1...10).map {
+            BrokerProfileQueryData.mock(preferredRunDate: nil, optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: nil)])
+        }
+        let mockPastQueryData = Array(1...10).map {
+            BrokerProfileQueryData.mock(preferredRunDate: .nowMinus(hours: $0), optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: .nowMinus(hours: $0))])
+        }
+        let mockFutureQueryData = Array(1...10).map {
+            BrokerProfileQueryData.mock(preferredRunDate: .nowPlus(hours: $0), optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: .nowPlus(hours: $0))])
+        }
+
+        return mockNilPreferredRunDateQueryData + mockPastQueryData + mockFutureQueryData
+    }()
+
+    lazy var mockScanQueryData: [BrokerProfileQueryData] = {
+        let mockNilPreferredRunDateQueryData = Array(1...10).map { _ in
+            BrokerProfileQueryData.mock(preferredRunDate: nil)
+        }
+        let mockPastQueryData = Array(1...10).map {
+            BrokerProfileQueryData.mock(preferredRunDate: .nowMinus(hours: $0))
+        }
+        let mockFutureQueryData = Array(1...10).map {
+            BrokerProfileQueryData.mock(preferredRunDate: .nowPlus(hours: $0))
+        }
+
+        return mockNilPreferredRunDateQueryData + mockPastQueryData + mockFutureQueryData
+    }()
+
+    func testWhenFilteringOptOutOperationData_thenAllButFuturePreferredRunDateIsReturned() {
+        let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: nil)
+        let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .now)
+        let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .distantPast)
+        let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .distantFuture)
+
+        XCTAssertEqual(operationData1.count, 30) // all jobs
+        XCTAssertEqual(operationData2.count, 20) // nil preferred run date + past jobs
+        XCTAssertEqual(operationData3.count, 10) // nil preferred run date jobs
+        XCTAssertEqual(operationData4.count, 30) // all jobs
+    }
+
+    func testWhenFilteringScanOperationData_thenPreferredRunDatePriorToPriorityDateIsReturned() {
+        let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .scheduledScan, priorityDate: nil)
+        let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .manualScan, priorityDate: .now)
+        let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .scheduledScan, priorityDate: .distantPast)
+        let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .manualScan, priorityDate: .distantFuture)
+
+        XCTAssertEqual(operationData1.count, 30) // all jobs
+        XCTAssertEqual(operationData2.count, 10) // past jobs
+        XCTAssertEqual(operationData3.count, 0) // no jobs
+        XCTAssertEqual(operationData4.count, 20) // past + future jobs
+    }
+
+    func testFilteringAllOperationData() {
+        let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: nil)
+        let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .now)
+        let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .distantPast)
+        let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .distantFuture)
+
+        XCTAssertEqual(operationData1.filter { $0 is ScanJobData }.count, 30) // all jobs
+        XCTAssertEqual(operationData1.filter { $0 is OptOutJobData }.count, 30) // all jobs
+        XCTAssertEqual(operationData1.count, 30+30)
+
+        XCTAssertEqual(operationData2.filter { $0 is ScanJobData }.count, 10) // past jobs
+        XCTAssertEqual(operationData2.filter { $0 is OptOutJobData }.count, 20) // nil preferred run date + past jobs
+        XCTAssertEqual(operationData2.count, 10+20)
+
+        XCTAssertEqual(operationData3.filter { $0 is ScanJobData }.count, 0) // no jobs
+        XCTAssertEqual(operationData3.filter { $0 is OptOutJobData }.count, 10) // nil preferred run date jobs
+        XCTAssertEqual(operationData3.count, 0+10)
+
+        XCTAssertEqual(operationData4.filter { $0 is ScanJobData }.count, 20) // past + future jobs
+        XCTAssertEqual(operationData4.filter { $0 is OptOutJobData }.count, 30) // all jobs
+        XCTAssertEqual(operationData4.count, 20+30)
+    }
+}

LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift

@@ -369,24 +369,17 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase {
         }
     }
 
-    func testWhenRemovedProfileIsFound_thenOptOutConfirmedIsAddedRemoveDateIsUpdatedAndPreferredRunDateIsSetToNil() async {
+    func testWhenRemovedProfileIsFound_thenOptOutConfirmedIsAddedRemoveDateIsUpdated() async {
         do {
-            let extractedProfileId: Int64 = 1
-            let brokerId: Int64 = 1
-            let profileQueryId: Int64 = 1
-            let mockHistoryEvent = HistoryEvent(extractedProfileId: extractedProfileId, brokerId: brokerId, profileQueryId: profileQueryId, type: .optOutRequested)
-            let mockBrokerProfileQuery = BrokerProfileQueryData(
-                dataBroker: .mock,
-                profileQuery: .mock,
-                scanJobData: .mock,
-                optOutJobData: [.mock(with: .mockWithoutRemovedDate, preferredRunDate: Date(), historyEvents: [mockHistoryEvent])]
-            )
-
             mockWebOperationRunner.scanResults = [.mockWithoutId]
-            mockDatabase.brokerProfileQueryDataToReturn = [mockBrokerProfileQuery]
             _ = try await sut.runScanOperation(
                 on: mockWebOperationRunner,
-                brokerProfileQueryData: mockBrokerProfileQuery,
+                brokerProfileQueryData: .init(
+                    dataBroker: .mock,
+                    profileQuery: .mock,
+                    scanJobData: .mock,
+                    optOutJobData: [OptOutJobData.mock(with: .mockWithoutRemovedDate)]
+                ),
                 database: mockDatabase,
                 notificationCenter: .default,
                 pixelHandler: MockDataBrokerProtectionPixelsHandler(),
@@ -396,8 +389,6 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase {
             XCTAssertTrue(mockDatabase.optOutEvents.contains(where: { $0.type == .optOutConfirmed }))
             XCTAssertTrue(mockDatabase.wasUpdateRemoveDateCalled)
             XCTAssertNotNil(mockDatabase.extractedProfileRemovedDate)
-            XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled)
-            XCTAssertNil(mockDatabase.lastPreferredRunDateOnOptOut)
         } catch {
             XCTFail("Should not throw")
         }
@@ -841,7 +832,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase {
         XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: Date().addingTimeInterval(schedulingConfig.confirmOptOutScan.hoursToSeconds)))
     }
 
-    func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToNil() throws {
+    func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToOptOutReattempt() throws {
         let brokerId: Int64 = 1
         let profileQueryId: Int64 = 1
         let extractedProfileId: Int64 = 1
@@ -851,7 +842,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase {
         try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: schedulingConfig, database: mockDatabase)
 
         XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled)
-        XCTAssertNil(mockDatabase.lastPreferredRunDateOnOptOut)
+        XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(schedulingConfig.hoursUntilNextOptOutAttempt.hoursToSeconds)))
     }
 
     func testWhenUpdatingDatesAndLastEventIsMatchesFound_thenWeSetScanPreferredDateToMaintanence() throws {
@@ -918,7 +909,9 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase {
 
         // If the date is not going to be set, we don't call the database function
         XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForScanCalled)
-        XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled)
+
+        XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled)
+        XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.hoursUntilNextOptOutAttempt.hoursToSeconds)))
     }
 
     func testUpdatingScanDateFromScan_thenScanDoesNotRespectMostRecentDate() throws {
@@ -942,8 +935,10 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase {
         try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: config, database: mockDatabase)
 
         XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled)
-        XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled)
         XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: expectedPreferredRunDate), "\(String(describing: mockDatabase.lastPreferredRunDateOnScan)) is not equal to \(expectedPreferredRunDate)")
+
+        XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled)
+        XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.hoursUntilNextOptOutAttempt.hoursToSeconds)))
     }
 }