show-progress to shows MB/s for each anonymization type

galfit82 · May 20, 2020 · 1e3bea5 · 1e3bea5
1 parent 9a5f232
commit 1e3bea5
Show file tree

Hide file tree

Showing 10 changed files with 38 additions and 21 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,5 @@
 /.vagrant/
 /*-console.log
 
+/.scannerwork/
+/.deptrac.cache
diff --git a/Dockerfile b/Dockerfile
@@ -6,4 +6,4 @@ RUN composer install --no-dev
 FROM php:7.4-cli-alpine
 COPY --from=build-env /build /anonymizer
 ENTRYPOINT ["php", "/anonymizer/bin/mysql-dump-anonymize.php"]
-CMD ["--show-progress=0", "--config=./columns_must_anonymize.yml"]
+CMD ["--show-progress=0", "--config=/anonymizer/columns_must_anonymize.yml"]
diff --git a/lib/Anonymizer.php b/lib/Anonymizer.php
@@ -132,7 +132,12 @@ private function anonymizeValue(ValueAnonymizerInterface $valueAnonymizer, Value
         if ($this->anonymizationProvider->isAnonymization($valueAnonymizer) === false) {
             $this->observer->notify(Observer::EVENT_NO_ANONYMIZATION, null);
         }
-        $this->observer->notify(Observer::EVENT_ANONYMIZATION_START, get_class($valueAnonymizer));
+
+        $this->observer->notify(Observer::EVENT_ANONYMIZATION_START, [
+            'anonymizationType' => get_class($valueAnonymizer),
+            'dataSize' => strlen($value->getUnEscapedValue())
+        ]);
+
         $ret = $valueAnonymizer->anonymize($value, $row);
         $this->observer->notify(Observer::EVENT_ANONYMIZATION_END, get_class($valueAnonymizer));
         return $ret;

diff --git a/lib/Application/Observer.php b/lib/Application/Observer.php
@@ -45,7 +45,7 @@ public function notify($event, $data) : void
                     $observer->onFinishReadLine($data);
                     break;
                 case self::EVENT_ANONYMIZATION_START:
-                    $observer->onAnonymizationStart($data);
+                    $observer->onAnonymizationStart($data['anonymizationType'], $data['dataSize']);
                     break;
                 case self::EVENT_ANONYMIZATION_END:
                     $observer->onAnonymizationFinish($data);

diff --git a/lib/Application/Observer/ProcessObserverInterface.php b/lib/Application/Observer/ProcessObserverInterface.php
@@ -14,7 +14,7 @@ public function onFinishReadLine(int $lineLength): void;
 
     public function onNullValue(string $anonymizationType): void;
 
-    public function onAnonymizationStart(string $anonymizationType): void;
+    public function onAnonymizationStart(string $anonymizationType, int $dataSize): void;
 
     public function onAnonymizationFinish(string $anonymizationType): void;
 

diff --git a/lib/Application/Observer/Progress.php b/lib/Application/Observer/Progress.php
@@ -80,9 +80,11 @@ public function onNullValue(string $anonymizationType): void
         $this->getAnonymizationType($anonymizationType)['nulls']++;
     }
 
-    public function onAnonymizationStart(string $anonymizationType): void
+    public function onAnonymizationStart(string $anonymizationType, int $dataSize): void
     {
         $this->getAnonymizationType($anonymizationType)['count']++;
+        $this->getAnonymizationType($anonymizationType)['size'] += $dataSize;
+
         $this->anonymizationTypeStart = microtime(true);
     }
 
@@ -133,6 +135,7 @@ private function &getAnonymizationType($key): array
                 'nulls' => 0,
                 'time' => 0,
                 'count' => 0,
+                'size' => 0,
             ];
         }
 
@@ -215,7 +218,14 @@ private function show(): void
             } else {
                 $div = 0;
             }
-            $output .= $this->pad($anonymizationType) . $this->round($microtime['time']) . 's (' . $this->round($div * 100) . ' %)  '
+            $speed = 0;
+            if ($microtime['size'] > 0) {
+                $speed = round(($microtime['size'] / 1024 / 1024) / $microtime['time'],3);
+            }
+
+            $output .= $this->pad($anonymizationType) . $this->round($microtime['time']) . 's'
+                . ' ('.str_pad((string)$speed,6, ' ',STR_PAD_LEFT).' MB/s)'
+                . ' (' . $this->round($div * 100) . ' %)  '
                 . ' (x ' . $this->round($this->anonymizationTypes[$anonymizationType]['count'], 0) . ')'
                 . ' (NULL: ' . $this->round($this->anonymizationTypes[$anonymizationType]['nulls'], 0) . ')'
                 . PHP_EOL;

diff --git a/tests/AnonymizerTest.php b/tests/AnonymizerTest.php
@@ -248,24 +248,24 @@ public function testRun(): void
         $this->observerMock->expects($this->exactly(25))->method('notify')->withConsecutive(
             [Observer::EVENT_START_READ, null],
             [Observer::EVENT_END_READ, 6],
-            [Observer::EVENT_ANONYMIZATION_START, $this->isType('string')],
+            [Observer::EVENT_ANONYMIZATION_START, $this->isType('array')],
             [Observer::EVENT_ANONYMIZATION_END, $this->isType('string')],
-            [Observer::EVENT_ANONYMIZATION_START, $this->isType('string')],
+            [Observer::EVENT_ANONYMIZATION_START, $this->isType('array')],
             [Observer::EVENT_ANONYMIZATION_END, $this->isType('string')],
-            [Observer::EVENT_ANONYMIZATION_START, $this->isType('string')],
+            [Observer::EVENT_ANONYMIZATION_START, $this->isType('array')],
             [Observer::EVENT_ANONYMIZATION_END, $this->isType('string')],
-            [Observer::EVENT_ANONYMIZATION_START, $this->isType('string')],
+            [Observer::EVENT_ANONYMIZATION_START, $this->isType('array')],
             [Observer::EVENT_ANONYMIZATION_END, $this->isType('string')],
             [Observer::EVENT_AFTER_LINE_PROCESSING, null],
             [Observer::EVENT_START_READ, null],
             [Observer::EVENT_END_READ, 5],
-            [Observer::EVENT_ANONYMIZATION_START, $this->isType('string')],
+            [Observer::EVENT_ANONYMIZATION_START, $this->isType('array')],
             [Observer::EVENT_ANONYMIZATION_END, $this->isType('string')],
-            [Observer::EVENT_ANONYMIZATION_START, $this->isType('string')],
+            [Observer::EVENT_ANONYMIZATION_START, $this->isType('array')],
             [Observer::EVENT_ANONYMIZATION_END, $this->isType('string')],
-            [Observer::EVENT_ANONYMIZATION_START, $this->isType('string')],
+            [Observer::EVENT_ANONYMIZATION_START, $this->isType('array')],
             [Observer::EVENT_ANONYMIZATION_END, $this->isType('string')],
-            [Observer::EVENT_ANONYMIZATION_START, $this->isType('string')],
+            [Observer::EVENT_ANONYMIZATION_START, $this->isType('array')],
             [Observer::EVENT_ANONYMIZATION_END, $this->isType('string')],
             [Observer::EVENT_AFTER_LINE_PROCESSING, null],
             [Observer::EVENT_START_READ, null],

diff --git a/tests/Application/Observer/ProgressTest.php b/tests/Application/Observer/ProgressTest.php
@@ -59,9 +59,9 @@ public function testOnAnonymizationStart(): void
 
     public function testOnAnonymizationFinish(): void
     {
-        $this->sut->onAnonymizationStart('namespace\\FreeText');
-        $this->sut->onAnonymizationStart('namespace\\FreeText');
-        $this->sut->onAnonymizationStart('namespace\\SomeType');
+        $this->sut->onAnonymizationStart('namespace\\FreeText', 1);
+        $this->sut->onAnonymizationStart('namespace\\FreeText', 1);
+        $this->sut->onAnonymizationStart('namespace\\SomeType',1);
         $this->assertSame(2, $this->getProperty('anonymizationTypes')['FreeText']['count']);
         $this->assertSame(1, $this->getProperty('anonymizationTypes')['SomeType']['count']);
 
@@ -99,12 +99,12 @@ public function testOnAfterEachLineProcessing(): void
 
         $this->sut->onBegin(10000);
         $this->sut->onStartReadLine();
-        $this->sut->onAnonymizationStart('namespace\\Something');
+        $this->sut->onAnonymizationStart('namespace\\Something',1);
         $this->sut->onAnonymizationFinish('namespace\\Something');
         $this->sut->onFinishReadLine(10);
 
         $this->sut->onStartReadLine();
-        $this->sut->onAnonymizationStart('namespace\\SomethingElse');
+        $this->sut->onAnonymizationStart('namespace\\SomethingElse',1);
         $this->sut->onAnonymizationFinish('namespace\\SomethingElse');
         $this->sut->onFinishReadLine(15);
 

diff --git a/tests/Application/ObserverTest.php b/tests/Application/ObserverTest.php
@@ -40,7 +40,7 @@ public function providerEventsMethods(): array
         return [
             [Observer::EVENT_START_READ, 'onStartReadLine'],
             [Observer::EVENT_END_READ, 'onFinishReadLine', 123],
-            [Observer::EVENT_ANONYMIZATION_START, 'onAnonymizationStart', 'data'],
+            [Observer::EVENT_ANONYMIZATION_START, 'onAnonymizationStart', ['anonymizationType'=>'test','dataSize'=>10]],
             [Observer::EVENT_ANONYMIZATION_END, 'onAnonymizationFinish', 'data'],
             [Observer::EVENT_AFTER_LINE_PROCESSING, 'onAfterEachLineProcessing'],
             [Observer::EVENT_NOT_AN_INSERT, 'onNotAnInsertLine'],

diff --git a/tests/WriteDump/MysqlLineDumpTest.php b/tests/WriteDump/MysqlLineDumpTest.php
@@ -25,7 +25,7 @@ public function setUp(): void
     public function testDump(): void
     {
         /** @noinspection SqlResolve */
-        $expected = "INSERT INTO `t` (`c1`, `c2`) VALUES (NULL,'r1c2'),('r2c1','r2c2');".PHP_EOL;
+        $expected = "INSERT INTO `t` (`c1`, `c2`) VALUES (NULL,'r1c2'),('r2c1','r2c2');"."\n";
 
         $actual = $this->sut->rebuildInsertLine('t', ['c1', 'c2'], [[
             AnonymizedValue::fromRawValue('NULL'),