diff --git a/README.md b/README.md index 1cfda84..a19253b 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ $etl = (new EtlExecutor()) }) ->loadInto(function (array $query, EtlState $state) { /** @var \PDO $pdo */ - $pdo = $state->destination; + $pdo = $state->destination; // See below - $state->destination corresponds to the $destination argument of the $etl->process() method. [$sql, $params] = $query; $stmt = $pdo->prepare($sql); $stmt->execute($params); @@ -132,6 +132,20 @@ As you can see: The `EtlState` object contains all elements relative to the state of your ETL workflow being running. +Difference between `yield` and `return` in transformers +------------------------------------------------------ + +The `EtlExecutor::transformWith()` method accepts an unlimited number of transformers as arguments. + +When you chain transformers, keep in mind that every transformer will get: +- Either the returned value passed from the previous transformer +- Either an array of every yielded value from the previous transformer + +But the last transformer of the chain (or your only one transformer) is deterministic to know what will be passed to the loader (either a return value, or a generator): +- If your transformer `returns` a value, this value will be passed to the loader. +- If your transformer `returns` an array of values (or whatever iterable), that return value will be passed to the loader. +- If your transformer `yields` values, each yielded value will be passed to the loader. + Using events ------------ diff --git a/src/EtlExecutor.php b/src/EtlExecutor.php index d8c22be..6ac1f6a 100644 --- a/src/EtlExecutor.php +++ b/src/EtlExecutor.php @@ -22,6 +22,7 @@ use BenTools\ETL\Internal\EtlBuilderTrait; use BenTools\ETL\Internal\EtlExceptionsTrait; use BenTools\ETL\Internal\Ref; +use BenTools\ETL\Internal\TransformResult; use BenTools\ETL\Loader\InMemoryLoader; use BenTools\ETL\Loader\LoaderInterface; use BenTools\ETL\Transformer\NullTransformer; @@ -137,10 +138,12 @@ private function extract(Ref $stateHolder): Generator private function transform(mixed $item, EtlState $state): array { try { - $transformed = $this->transformer->transform($item, $state); - $items = $transformed instanceof Generator ? [...$transformed] : [$transformed]; + $transformResult = TransformResult::create($this->transformer->transform($item, $state)); - return $this->dispatch(new TransformEvent($state, $items))->items; + $event = $this->dispatch(new TransformEvent($state, $transformResult)); + $transformResult = TransformResult::create($event->transformResult); + + return [...$transformResult]; } catch (SkipRequest|StopRequest $e) { throw $e; } catch (Throwable $e) { diff --git a/src/EventDispatcher/Event/TransformEvent.php b/src/EventDispatcher/Event/TransformEvent.php index dd4bac0..7ddac84 100644 --- a/src/EventDispatcher/Event/TransformEvent.php +++ b/src/EventDispatcher/Event/TransformEvent.php @@ -12,12 +12,9 @@ final class TransformEvent extends Event implements StoppableEventInterface { use StoppableEventTrait; - /** - * @param list $items - */ public function __construct( public readonly EtlState $state, - public array $items, + public mixed $transformResult, ) { } } diff --git a/src/Internal/TransformResult.php b/src/Internal/TransformResult.php new file mode 100644 index 0000000..5d2eded --- /dev/null +++ b/src/Internal/TransformResult.php @@ -0,0 +1,50 @@ + + */ +final class TransformResult implements IteratorAggregate +{ + public mixed $value; + public bool $iterable; + + private function __construct() + { + } + + public function getIterator(): Traversable + { + if ($this->iterable) { + yield from $this->value; + } else { + yield $this->value; + } + } + + public static function create(mixed $value): self + { + static $prototype; + $prototype ??= new self(); + + $that = clone $prototype; + if ($value instanceof Generator || $value instanceof self) { + $that->value = [...$value]; + $that->iterable = true; + } else { + $that->value = $value; + $that->iterable = false; + } + + return $that; + } +} diff --git a/src/Recipe/LoggerRecipe.php b/src/Recipe/LoggerRecipe.php index f3b6000..cc33495 100644 --- a/src/Recipe/LoggerRecipe.php +++ b/src/Recipe/LoggerRecipe.php @@ -83,7 +83,7 @@ public function decorate(EtlExecutor $executor): EtlExecutor [ 'key' => $event->state->currentItemKey, 'state' => $event->state, - 'items' => $event->items, + 'items' => $event->transformResult, ], ), $this->priorities[TransformEvent::class] ?? $this->defaultPriority, diff --git a/src/Transformer/ChainTransformer.php b/src/Transformer/ChainTransformer.php index be6b2c2..956f1de 100644 --- a/src/Transformer/ChainTransformer.php +++ b/src/Transformer/ChainTransformer.php @@ -5,6 +5,7 @@ namespace BenTools\ETL\Transformer; use BenTools\ETL\EtlState; +use BenTools\ETL\Internal\TransformResult; final readonly class ChainTransformer implements TransformerInterface { @@ -35,7 +36,7 @@ public function transform(mixed $item, EtlState $state): mixed { $output = $item; foreach ($this->transformers as $transformer) { - $output = $transformer->transform($output, $state); + $output = TransformResult::create($transformer->transform($output, $state))->value; } return $output; diff --git a/tests/Behavior/Events/TransformEventTest.php b/tests/Behavior/Events/TransformEventTest.php index f70c50b..907553e 100644 --- a/tests/Behavior/Events/TransformEventTest.php +++ b/tests/Behavior/Events/TransformEventTest.php @@ -19,7 +19,7 @@ yield strtoupper($value); }) ->onTransform(function (TransformEvent $e) use (&$transformedItems) { - $transformedItems = [...$transformedItems, ...$e->items]; + $transformedItems = [...$transformedItems, ...$e->transformResult]; }); // When diff --git a/tests/Behavior/SkipTest.php b/tests/Behavior/SkipTest.php index 7426dc9..3b71505 100644 --- a/tests/Behavior/SkipTest.php +++ b/tests/Behavior/SkipTest.php @@ -63,7 +63,7 @@ $cities[] = $city; }) ->onTransform(function (TransformEvent $event) { - if ('Tokyo' === [...$event->items][0]) { + if ('Tokyo' === [...$event->transformResult][0]) { $event->state->skip(); } }); diff --git a/tests/Behavior/StopTest.php b/tests/Behavior/StopTest.php index 95185c2..ac6a983 100644 --- a/tests/Behavior/StopTest.php +++ b/tests/Behavior/StopTest.php @@ -58,7 +58,7 @@ $cities[] = $city; }) ->onTransform(function (TransformEvent $event) { - if ('Shanghai' === [...$event->items][0]) { + if ('Shanghai' === [...$event->transformResult][0]) { $event->state->stop(); } }); diff --git a/tests/Unit/Transformer/ChainTransformerTest.php b/tests/Unit/Transformer/ChainTransformerTest.php index 4118e84..dedfa1c 100644 --- a/tests/Unit/Transformer/ChainTransformerTest.php +++ b/tests/Unit/Transformer/ChainTransformerTest.php @@ -23,7 +23,11 @@ function (string $item): Generator { yield strtoupper($item); }, )) - ->with(fn (Generator $items): array => [...$items]) + ->with(function (array $items): array { + $items[] = 'hey'; + + return $items; + }) ->with(fn (array $items): string => implode('-', $items)); // When @@ -32,8 +36,8 @@ function (string $item): Generator { // Then expect($report->output)->toBe([ - 'oof-OOF', - 'rab-RAB', + 'oof-OOF-hey', + 'rab-RAB-hey', ]); }); @@ -48,7 +52,11 @@ function (string $item): Generator { yield $item; yield strtoupper($item); }, - fn (Generator $items): array => [...$items], + function (array $items): array { + $items[] = 'hey'; + + return $items; + }, fn (array $items): string => implode('-', $items) ); @@ -57,7 +65,7 @@ function (string $item): Generator { // Then expect($report->output)->toBe([ - 'oof-OOF', - 'rab-RAB', + 'oof-OOF-hey', + 'rab-RAB-hey', ]); });