Skip to content

Commit

Permalink
EntityPreloader: refactor preloading has many associations
Browse files Browse the repository at this point in the history
  • Loading branch information
@JanTvrdik
JanTvrdik committed Oct 7, 2024
1 parent 381d5fd commit f6cb6b1
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 98 deletions.
3 changes: 1 addition & 2 deletions composer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
],
"require": {
"php": "^8.1",
"doctrine/orm": "^3",
"doctrine/persistence": "^3.1"
"doctrine/orm": "^3"
},
"require-dev": {
"doctrine/collections": "^2.2",
Expand Down
204 changes: 108 additions & 96 deletions src/EntityPreloader.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
use Doctrine\ORM\Mapping\ClassMetadata;
use Doctrine\ORM\PersistentCollection;
use Doctrine\ORM\QueryBuilder;
use Doctrine\Persistence\Proxy;
use LogicException;
use ReflectionProperty;
use function array_chunk;
use function array_values;
use function count;
Expand All @@ -20,7 +20,7 @@
class EntityPreloader
{

private const BATCH_SIZE = 1_000;
private const PRELOAD_ENTITY_DEFAULT_BATCH_SIZE = 1_000;
private const PRELOAD_COLLECTION_DEFAULT_BATCH_SIZE = 100;

public function __construct(
Expand Down Expand Up @@ -64,15 +64,15 @@ public function preload(
}

$maxFetchJoinSameFieldCount ??= 1;
$sourceEntities = $this->loadProxies($sourceClassMetadata, $sourceEntities, $batchSize ?? self::BATCH_SIZE, $maxFetchJoinSameFieldCount);
$sourceEntities = $this->loadProxies($sourceClassMetadata, $sourceEntities, $batchSize ?? self::PRELOAD_ENTITY_DEFAULT_BATCH_SIZE, $maxFetchJoinSameFieldCount);

return match ($associationMapping->type()) {
ClassMetadata::ONE_TO_MANY => $this->preloadOneToMany($sourceEntities, $sourceClassMetadata, $sourcePropertyName, $targetClassMetadata, $batchSize, $maxFetchJoinSameFieldCount),
ClassMetadata::MANY_TO_MANY => $this->preloadManyToMany($sourceEntities, $sourceClassMetadata, $sourcePropertyName, $targetClassMetadata, $batchSize, $maxFetchJoinSameFieldCount),
ClassMetadata::ONE_TO_ONE,
ClassMetadata::MANY_TO_ONE => $this->preloadToOne($sourceEntities, $sourceClassMetadata, $sourcePropertyName, $targetClassMetadata, $batchSize, $maxFetchJoinSameFieldCount),
$preloader = match (true) {
$associationMapping->isToOne() => $this->preloadToOne(...),
$associationMapping->isToMany() => $this->preloadToMany(...),
default => throw new LogicException("Unsupported association mapping type {$associationMapping->type()}"),
};

return $preloader($sourceEntities, $sourceClassMetadata, $sourcePropertyName, $targetClassMetadata, $batchSize, $maxFetchJoinSameFieldCount);
}

/**
Expand Down Expand Up @@ -135,7 +135,7 @@ private function loadProxies(
$entityKey = (string) $entityId;
$uniqueEntities[$entityKey] = $entity;

if ($entity instanceof Proxy && !$entity->__isInitialized()) {
if ($this->entityManager->isUninitializedObject($entity)) {
$uninitializedIds[$entityKey] = $entityId;
}
}
Expand All @@ -157,7 +157,7 @@ private function loadProxies(
* @template S of E
* @template T of E
*/
private function preloadOneToMany(
private function preloadToMany(
array $sourceEntities,
ClassMetadata $sourceClassMetadata,
string $sourcePropertyName,
Expand All @@ -168,12 +168,9 @@ private function preloadOneToMany(
{
$sourceIdentifierReflection = $sourceClassMetadata->getSingleIdReflectionProperty(); // e.g. Order::$id reflection
$sourcePropertyReflection = $sourceClassMetadata->getReflectionProperty($sourcePropertyName); // e.g. Order::$items reflection

$targetIdentifierReflection = $targetClassMetadata->getSingleIdReflectionProperty();
$targetPropertyName = $sourceClassMetadata->getAssociationMappedByTargetField($sourcePropertyName); // e.g. 'order'
$targetPropertyReflection = $targetClassMetadata->getReflectionProperty($targetPropertyName); // e.g. Item::$order reflection

if ($sourceIdentifierReflection === null || $sourcePropertyReflection === null || $targetIdentifierReflection === null || $targetPropertyReflection === null) {
if ($sourceIdentifierReflection === null || $sourcePropertyReflection === null || $targetIdentifierReflection === null) {
throw new LogicException('Doctrine should use RuntimeReflectionService which never returns null.');
}

Expand All @@ -198,21 +195,30 @@ private function preloadOneToMany(
}

foreach ($sourceEntityCollection as $targetEntity) {
$targetEntityId = $targetIdentifierReflection->getValue($targetEntity);
$targetEntityKey = (string) $targetEntityId;
$targetEntityKey = (string) $targetIdentifierReflection->getValue($targetEntity);
$targetEntities[$targetEntityKey] = $targetEntity;
}
}

foreach (array_chunk($uninitializedSourceEntityIds, $batchSize, preserve_keys: true) as $uninitializedSourceEntityIdsChunk) {
$targetEntitiesChunk = $this->loadEntitiesBy($targetClassMetadata, $targetPropertyName, array_values($uninitializedSourceEntityIdsChunk), $maxFetchJoinSameFieldCount);

foreach ($targetEntitiesChunk as $targetEntity) {
$sourceEntity = $targetPropertyReflection->getValue($targetEntity);
$sourceEntityKey = (string) $sourceIdentifierReflection->getValue($sourceEntity);
$uninitializedCollections[$sourceEntityKey]->add($targetEntity);
$innerLoader = match ($sourceClassMetadata->getAssociationMapping($sourcePropertyName)->type()) {
ClassMetadata::ONE_TO_MANY => $this->preloadOneToManyInner(...),
ClassMetadata::MANY_TO_MANY => $this->preloadManyToManyInner(...),
default => throw new LogicException('Unsupported association mapping type'),
};

$targetEntityKey = (string) $targetIdentifierReflection->getValue($targetEntity);
foreach (array_chunk($uninitializedSourceEntityIds, $batchSize, preserve_keys: true) as $uninitializedSourceEntityIdsChunk) {
$targetEntitiesChunk = $innerLoader(
sourceClassMetadata: $sourceClassMetadata,
sourceIdentifierReflection: $sourceIdentifierReflection,
sourcePropertyName: $sourcePropertyName,
targetClassMetadata: $targetClassMetadata,
targetIdentifierReflection: $targetIdentifierReflection,
uninitializedSourceEntityIdsChunk: array_values($uninitializedSourceEntityIdsChunk),
uninitializedCollections: $uninitializedCollections,
maxFetchJoinSameFieldCount: $maxFetchJoinSameFieldCount,
);

foreach ($targetEntitiesChunk as $targetEntityKey => $targetEntity) {
$targetEntities[$targetEntityKey] = $targetEntity;
}
}
Expand All @@ -226,105 +232,109 @@ private function preloadOneToMany(
}

/**
* @param list<S> $sourceEntities
* @param ClassMetadata<S> $sourceClassMetadata
* @param ClassMetadata<T> $targetClassMetadata
* @param positive-int|null $batchSize
* @param list<mixed> $uninitializedSourceEntityIdsChunk
* @param array<string, PersistentCollection<int, T>> $uninitializedCollections
* @param non-negative-int $maxFetchJoinSameFieldCount
* @return list<T>
* @return array<string, T>
* @template S of E
* @template T of E
*/
private function preloadManyToMany(
array $sourceEntities,
private function preloadOneToManyInner(
ClassMetadata $sourceClassMetadata,
ReflectionProperty $sourceIdentifierReflection,
string $sourcePropertyName,
ClassMetadata $targetClassMetadata,
?int $batchSize,
ReflectionProperty $targetIdentifierReflection,
array $uninitializedSourceEntityIdsChunk,
array $uninitializedCollections,
int $maxFetchJoinSameFieldCount,
): array
{
$sourceIdentifierReflection = $sourceClassMetadata->getSingleIdReflectionProperty();
$sourceIdentifierName = $sourceClassMetadata->getSingleIdentifierFieldName();
$sourcePropertyReflection = $sourceClassMetadata->getReflectionProperty($sourcePropertyName);

$targetIdentifierReflection = $targetClassMetadata->getSingleIdReflectionProperty();
$targetIdentifierName = $targetClassMetadata->getSingleIdentifierFieldName();
$targetPropertyName = $sourceClassMetadata->getAssociationMappedByTargetField($sourcePropertyName); // e.g. 'order'
$targetPropertyReflection = $targetClassMetadata->getReflectionProperty($targetPropertyName); // e.g. Item::$order reflection
$targetEntities = [];

if ($sourceIdentifierReflection === null || $sourcePropertyReflection === null || $targetIdentifierReflection === null) {
if ($targetPropertyReflection === null) {
throw new LogicException('Doctrine should use RuntimeReflectionService which never returns null.');
}

$batchSize ??= self::PRELOAD_COLLECTION_DEFAULT_BATCH_SIZE;
$targetEntities = [];
$uninitializedSourceEntityIds = [];
$uninitializedCollections = [];
foreach ($this->loadEntitiesBy($targetClassMetadata, $targetPropertyName, $uninitializedSourceEntityIdsChunk, $maxFetchJoinSameFieldCount) as $targetEntity) {
$sourceEntity = $targetPropertyReflection->getValue($targetEntity);
$sourceEntityKey = (string) $sourceIdentifierReflection->getValue($sourceEntity);
$uninitializedCollections[$sourceEntityKey]->add($targetEntity);

foreach ($sourceEntities as $sourceEntity) {
$sourceEntityId = $sourceIdentifierReflection->getValue($sourceEntity);
$sourceEntityKey = (string) $sourceEntityId;
$sourceEntityCollection = $sourcePropertyReflection->getValue($sourceEntity);
$targetEntityKey = (string) $targetIdentifierReflection->getValue($targetEntity);
$targetEntities[$targetEntityKey] = $targetEntity;
}

if (
$sourceEntityCollection instanceof PersistentCollection
&& !$sourceEntityCollection->isInitialized()
&& !$sourceEntityCollection->isDirty() // preloading dirty collection is too hard to handle
) {
$uninitializedSourceEntityIds[$sourceEntityKey] = $sourceEntityId;
$uninitializedCollections[$sourceEntityKey] = $sourceEntityCollection;
continue;
}
return $targetEntities;
}

foreach ($sourceEntityCollection as $targetEntity) {
$targetEntityId = $targetIdentifierReflection->getValue($targetEntity);
$targetEntityKey = (string) $targetEntityId;
$targetEntities[$targetEntityKey] = $targetEntity;
}
}
/**
* @param ClassMetadata<S> $sourceClassMetadata
* @param ClassMetadata<T> $targetClassMetadata
* @param list<mixed> $uninitializedSourceEntityIdsChunk
* @param array<string, PersistentCollection<int, T>> $uninitializedCollections
* @param non-negative-int $maxFetchJoinSameFieldCount
* @return array<string, T>
* @template S of E
* @template T of E
*/
private function preloadManyToManyInner(
ClassMetadata $sourceClassMetadata,
ReflectionProperty $sourceIdentifierReflection,
string $sourcePropertyName,
ClassMetadata $targetClassMetadata,
ReflectionProperty $targetIdentifierReflection,
array $uninitializedSourceEntityIdsChunk,
array $uninitializedCollections,
int $maxFetchJoinSameFieldCount,
): array
{
$sourceIdentifierName = $sourceClassMetadata->getSingleIdentifierFieldName();
$targetIdentifierName = $targetClassMetadata->getSingleIdentifierFieldName();

foreach (array_chunk($uninitializedSourceEntityIds, $batchSize, preserve_keys: true) as $uninitializedSourceEntityIdsChunk) {
$manyToManyRows = $this->entityManager->createQueryBuilder()
->select("source.{$sourceIdentifierName} AS sourceId", "target.{$targetIdentifierName} AS targetId")
->from($sourceClassMetadata->getName(), 'source')
->join("source.{$sourcePropertyName}", 'target')
->andWhere('source IN (:sourceEntityIds)')
->setParameter('sourceEntityIds', array_values($uninitializedSourceEntityIdsChunk))
->getQuery()
->getResult();

$uninitializedTargetEntityIds = [];

foreach ($manyToManyRows as $manyToManyRow) {
$targetEntityId = $manyToManyRow['targetId'];
$targetEntityKey = (string) $targetEntityId;
$targetEntity = $this->entityManager->getUnitOfWork()->tryGetById($targetEntityId, $targetClassMetadata->getName());

if ($targetEntity !== false && (!$targetEntity instanceof Proxy || $targetEntity->__isInitialized())) {
$targetEntities[$targetEntityKey] = $targetEntity;
continue;
}
$manyToManyRows = $this->entityManager->createQueryBuilder()
->select("source.{$sourceIdentifierName} AS sourceId", "target.{$targetIdentifierName} AS targetId")
->from($sourceClassMetadata->getName(), 'source')
->join("source.{$sourcePropertyName}", 'target')
->andWhere('source IN (:sourceEntityIds)')
->setParameter('sourceEntityIds', $uninitializedSourceEntityIdsChunk)
->getQuery()
->getResult();

$uninitializedTargetEntityIds[$targetEntityKey] = $targetEntityId;
}
$targetEntities = [];
$uninitializedTargetEntityIds = [];

foreach ($this->loadEntitiesBy($targetClassMetadata, $targetIdentifierName, array_values($uninitializedTargetEntityIds), $maxFetchJoinSameFieldCount) as $targetEntity) {
$targetEntityKey = (string) $targetIdentifierReflection->getValue($targetEntity);
foreach ($manyToManyRows as $manyToManyRow) {
$targetEntityId = $manyToManyRow['targetId'];
$targetEntityKey = (string) $targetEntityId;

/** @var T|false $targetEntity */
$targetEntity = $this->entityManager->getUnitOfWork()->tryGetById($targetEntityId, $targetClassMetadata->getName());

if ($targetEntity !== false && !$this->entityManager->isUninitializedObject($targetEntity)) {
$targetEntities[$targetEntityKey] = $targetEntity;
continue;
}

foreach ($manyToManyRows as $manyToManyRow) {
$sourceEntityKey = (string) $manyToManyRow['sourceId'];
$targetEntityKey = (string) $manyToManyRow['targetId'];
$uninitializedCollections[$sourceEntityKey]->add($targetEntities[$targetEntityKey]);
}
$uninitializedTargetEntityIds[$targetEntityKey] = $targetEntityId;
}

foreach ($uninitializedCollections as $sourceEntityCollection) {
$sourceEntityCollection->setInitialized(true);
$sourceEntityCollection->takeSnapshot();
foreach ($this->loadEntitiesBy($targetClassMetadata, $targetIdentifierName, array_values($uninitializedTargetEntityIds), $maxFetchJoinSameFieldCount) as $targetEntity) {
$targetEntityKey = (string) $targetIdentifierReflection->getValue($targetEntity);
$targetEntities[$targetEntityKey] = $targetEntity;
}

return array_values($targetEntities);
foreach ($manyToManyRows as $manyToManyRow) {
$sourceEntityKey = (string) $manyToManyRow['sourceId'];
$targetEntityKey = (string) $manyToManyRow['targetId'];
$uninitializedCollections[$sourceEntityKey]->add($targetEntities[$targetEntityKey]);
}

return $targetEntities;
}

/**
Expand All @@ -347,12 +357,14 @@ private function preloadToOne(
): array
{
$sourcePropertyReflection = $sourceClassMetadata->getReflectionProperty($sourcePropertyName); // e.g. Item::$order reflection
$targetEntities = [];

if ($sourcePropertyReflection === null) {
throw new LogicException('Doctrine should use RuntimeReflectionService which never returns null.');
}

$batchSize ??= self::PRELOAD_ENTITY_DEFAULT_BATCH_SIZE;
$targetEntities = [];

foreach ($sourceEntities as $sourceEntity) {
$targetEntity = $sourcePropertyReflection->getValue($sourceEntity);

Expand All @@ -363,7 +375,7 @@ private function preloadToOne(
$targetEntities[] = $targetEntity;
}

return $this->loadProxies($targetClassMetadata, $targetEntities, $batchSize ?? self::BATCH_SIZE, $maxFetchJoinSameFieldCount);
return $this->loadProxies($targetClassMetadata, $targetEntities, $batchSize, $maxFetchJoinSameFieldCount);
}

/**
Expand Down

0 comments on commit f6cb6b1

Please sign in to comment.