Fix to keep resume import from import_start_time and continue import pending dates, #PG-3704

Author: AltamashShaikh

Commands/ImportGA4Reports.php

@@ -198,8 +198,21 @@ protected function executeImpl() : int
             $output->writeln(LogToSingleFileProcessor::$cliOutputPrefix . "Importing the following date ranges in order: " . $dateRangesText);
             // NOTE: date ranges to reimport are handled first, then we go back to the main import (which could be
             // continuous)
+
+            // If no date ranges to re-import and import reached till startDate, check if futureDates need to be imported or not and update the start and EdnDate
+            if (count($dateRangesToImport) == 1 && !empty($status['last_day_archived'])  && $status['last_day_archived'] === $status['import_range_start'] && $dates[1]->isLater(Date::factory('yesterday')) && !empty($status['import_start_time'])) {
+                $dateRangesToImport[0][0] = Date::factory($status['import_start_time'])->subDay(1);
+                $status['do_not_import_latest_dates_first'] = true;
+                $status['main_import_progress'] = $dateRangesToImport[0][0]->toString();
+                $importStatus->saveStatus($status);
+            }
+
             foreach (array_values($dateRangesToImport) as $index => $datesToImport) {
                 $status = $importStatus->getImportStatus($idSite);
+                $isDoNotImportLatestDatesFirst = !empty($status['do_not_import_latest_dates_first']);
+                if ($isDoNotImportLatestDatesFirst) {
+                    $status['last_date_imported'] = null;
+                }
                 // can change in the meantime, so we refetch
                 if (!is_array($datesToImport) || count($datesToImport) != 2) {
                     $output->writeln(LogToSingleFileProcessor::$cliOutputPrefix . "Found broken entry in date ranges to import (entry #{$index}) with improper type, skipping.");
@@ -218,7 +231,9 @@ protected function executeImpl() : int
                 if (!empty($lastDateImported) && $isFutureDateImport) {
                     $startDate = Date::factory($status['future_resume_date']);
                 } else {
-                    if (!empty($lastDateImported) && Date::factory($lastDateImported)->subDay(1)->isEarlier($endDate)) {
+                    if ($isDoNotImportLatestDatesFirst) {
+                        $startDate =  Date::factory($lastDateImported)->addDay(1);
+                    } elseif (!empty($lastDateImported) && Date::factory($lastDateImported)->subDay(1)->isEarlier($endDate)) {
                         $endDate = Date::factory($lastDateImported)->subDay(1);
                     }
                 }
@@ -227,15 +242,15 @@ protected function executeImpl() : int
                     $importStatus->removeReImportEntry($idSite, $datesToImport);
                     continue;
                 }
-                if ($endDate->isEarlier($startDate)) {
+                if ($endDate->isEarlier($startDate) && !$isDoNotImportLatestDatesFirst) {
                     $output->writeln(LogToSingleFileProcessor::$cliOutputPrefix . "(Entry #{$index}) is finished, moving on.");
                     $importStatus->removeReImportEntry($idSite, $datesToImport);
                     continue;
                 }
                 $output->writeln(LogToSingleFileProcessor::$cliOutputPrefix . "Importing reports for date range {$startDate} - {$endDate} from GA property {$property}.");
                 try {
                     $importer->setIsMainImport($isMainImport);
-                    $aborted = $importer->import($idSite, $property, $startDate, $endDate, $lock, '', $streamIds);
+                    $aborted = $importer->import($idSite, $property, $startDate, $endDate, $lock, '', $streamIds, $isDoNotImportLatestDatesFirst);
                     if ($aborted == -1) {
                         $shouldFinishImportIfNothingLeft = \false;
                     }

ImportStatus.php

@@ -70,11 +70,11 @@ public function getImportedDateRange($idSite)
         }
         return $dates;
     }
-    public function dayImportFinished($idSite, Date $date, $isMainImport = \true)
+    public function dayImportFinished($idSite, Date $date, $isMainImport = \true, $skipRecentDateImportFirst = false)
     {
         $status = $this->getImportStatus($idSite);
         $status['status'] = self::STATUS_ONGOING;
-        if (empty($status['last_date_imported']) || !Date::factory($status['last_date_imported'])->isEarlier($date) || !empty($status['future_resume_date']) && Date::factory($status['last_date_imported'])->isEarlier($date)) {
+        if (empty($status['last_date_imported']) || $skipRecentDateImportFirst || !Date::factory($status['last_date_imported'])->isEarlier($date) || !empty($status['future_resume_date']) && Date::factory($status['last_date_imported'])->isEarlier($date)) {
             $status['last_date_imported'] = $date->toString();
             $this->setImportedDateRange($idSite, $startDate = null, $date);
             if ($isMainImport) {

ImporterGA4.php

@@ -300,7 +300,7 @@ private function importCustomVariableSlots()
         $command .= ' customvariables:set-max-custom-variables ' . $numCustomVarSlots;
         passthru($command);
     }
-    public function import($idSite, $propertyId, Date $start, Date $end, Lock $lock, $segment = '', $streamIds = [])
+    public function import($idSite, $propertyId, Date $start, Date $end, Lock $lock, $segment = '', $streamIds = [], $skipRecentDateImportFirst = false)
     {
         $date = null;
         try {
@@ -314,7 +314,11 @@ public function import($idSite, $propertyId, Date $start, Date $end, Lock $lock,
             }
             $recordImporters = $this->getRecordImporters($idSite, $propertyId, $streamIds);
             $site = new Site($idSite);
-            $dates = $this->getRecentDatesToImport($start, $endPlusOne, Date::today()->getTimestamp());
+            if (!$skipRecentDateImportFirst) {
+                $dates = $this->getRecentDatesToImport($start, $endPlusOne, Date::today()->getTimestamp());
+            } else {
+                $dates = $this->getDatesToImport($start, $end);
+            }
             foreach ($dates as $date) {
                 if ($date->isToday() || $date->isLater(Date::yesterday())) {
                     $this->logger->info("Encountered Future Date while Importing data for GA4 Property {propertyID} for date {date}, the import would be stopped", ['viewId' => $propertyId, 'date' => $date->toString()]);
@@ -328,7 +332,7 @@ public function import($idSite, $propertyId, Date $start, Date $end, Lock $lock,
                     // force delete all tables in case they aren't all freed
                     \Piwik\DataTable\Manager::getInstance()->deleteAll();
                 }
-                $this->importStatus->dayImportFinished($idSite, $date, $this->isMainImport);
+                $this->importStatus->dayImportFinished($idSite, $date, $this->isMainImport, $skipRecentDateImportFirst);
             }
             $this->importStatus->finishImportIfNothingLeft($idSite);
             unset($recordImporters);
@@ -591,4 +595,18 @@ public function getRecentDatesToImport(Date $startDate, Date $endPlusOne, $thres
         }
         return $dates;
     }
+
+    /**
+     * @param Date $startDate
+     * @param Date $endDate
+     * @return array of dates between $startDate and $endDate
+     */
+    public function getDatesToImport(Date $startDate, Date $endDate)
+    {
+        $dates = [];
+        for ($date = $startDate; $date->getTimestamp() <= $endDate->getTimestamp(); $date = $date->addDay(1)) {
+            array_push($dates, $date);
+        }
+        return $dates;
+    }
 }

tests/Integration/ImporterTestGA4.php

@@ -65,6 +65,17 @@ public function test_getRecentDatesToImport_Past()
         }
         $this->assertEquals(['2019-07-13', '2019-07-12', '2019-07-11', '2019-07-10', '2019-07-09', '2019-07-08', '2019-07-07'], $processed);
     }
+    public function test_getDatesToImport()
+    {
+        $startDate = Date::factory('2022-07-07');
+        $endDate = Date::factory('2022-07-13');
+        $dates = $this->importer->getDatesToImport($startDate, $endDate);
+        $processed = [];
+        foreach ($dates as $dateObj) {
+            $processed[] = $dateObj->toString();
+        }
+        $this->assertEquals(['2022-07-07', '2022-07-08', '2022-07-09', '2022-07-10', '2022-07-11', '2022-07-12', '2022-07-13'], $processed);
+    }
     public function makeMockService()
     {
         return new \Piwik\Plugins\GoogleAnalyticsImporter\tests\Integration\MockGoogleServiceAnalytics($this);