Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

!!! TASK: Split dimensionspacepoints into separate table to reduce data duplication #4790

Merged
merged 12 commits into from
Mar 18, 2024
Prev Previous commit
Next Next commit
Some code cleanup
kitsunet committed Mar 13, 2024

Unverified

No user is associated with the committer email.
commit d468dd3290a56bc8a550a3cf164b8e6ae0fdc48f
Original file line number Diff line number Diff line change
@@ -57,7 +57,7 @@ private function createNodeTable(): Table
private function createHierarchyRelationTable(): Table
{
$table = new Table($this->tableNamePrefix . '_hierarchyrelation', [
(new Column('name', Type::getType(Types::STRING)))->setLength(255)->setNotnull(false)->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION),
(new Column('name', Type::getType(Types::STRING)))->setLength(255)->setNotnull(false)->setCustomSchemaOption('charset', 'ascii')->setCustomSchemaOption('collation', 'ascii_general_ci'),
(new Column('position', Type::getType(Types::INTEGER)))->setNotnull(true),
DbalSchemaFactory::columnForContentStreamId('contentstreamid')->setNotnull(true),
DbalSchemaFactory::columnForDimensionSpacePointHash('dimensionspacepointhash')->setNotnull(true),
@@ -66,10 +66,7 @@ private function createHierarchyRelationTable(): Table
]);

return $table
->addIndex(['childnodeanchor'])
->addIndex(['contentstreamid'])
->addIndex(['parentnodeanchor'])
->addIndex(['contentstreamid', 'childnodeanchor', 'dimensionspacepointhash'])
->setPrimaryKey(['childnodeanchor', 'contentstreamid', 'dimensionspacepointhash', 'parentnodeanchor'])
->addIndex(['contentstreamid', 'dimensionspacepointhash']);
Comment on lines 68 to 70
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lol after debugging i found out that this change here slows down the replay of ContentStreamWasForked events dramatically

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can confirm, this fixes replay for me:
image

I would suggest that we revert that part for now as a "hotfix". Afterwards, we could investigate why that kills the database

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See #5009

}

Original file line number Diff line number Diff line change
@@ -56,6 +56,9 @@
* - cn -> child node
* - h -> the hierarchy edge connecting parent and child
* - ph -> the hierarchy edge incoming to the parent (sometimes relevant)
* - dsp -> dimension space point, resolves hashes to full dimension coordinates
* - cdsp -> child dimension space point, same as dsp for child queries
* - pdsp -> parent dimension space point, same as dsp for parent queries
*
* @internal the parent interface {@see ContentGraphInterface} is API
*/
Original file line number Diff line number Diff line change
@@ -15,9 +15,7 @@
namespace Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository;

use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
use Neos\ContentRepository\Core\DimensionSpace\AbstractDimensionSpacePoint;
use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint;
use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint;

/**
@@ -28,9 +26,9 @@
final class DimensionSpacePoints
{
/**
* @var string[]
* @var array<string, string>
*/
private array $dimensionspacePoints = [];
private array $dimensionspacePointsRuntimeCache = [];

public function __construct(
private readonly Connection $databaseConnection,
@@ -40,16 +38,13 @@ public function __construct(

public function insertDimensionSpacePoint(AbstractDimensionSpacePoint $dimensionSpacePoint): void
{
try {
$this->databaseConnection->executeStatement(
'INSERT INTO ' . $this->tableNamePrefix . '_dimensionspacepoints (hash, dimensionspacepoint) VALUES (:dimensionspacepointhash, :dimensionspacepoint)',
[
'dimensionspacepointhash' => $dimensionSpacePoint->hash,
'dimensionspacepoint' => $dimensionSpacePoint->toJson()
]
);
} catch (UniqueConstraintViolationException $_) {
if ($this->getCoordinatesByHashFromRuntimeCache($dimensionSpacePoint->hash) !== null) {
return;
}

$this->dimensionspacePointsRuntimeCache[$dimensionSpacePoint->hash] = $dimensionSpacePoint->toJson();
$this->writeDimensionSpacePoint($dimensionSpacePoint->hash, $dimensionSpacePoint->toJson());

}

/**
@@ -60,39 +55,51 @@ public function insertDimensionSpacePoint(AbstractDimensionSpacePoint $dimension
*/
public function insertDimensionSpacePointByHashAndCoordinates(string $hash, array $dimensionSpacePointCoordinates): void
{
try {
$this->databaseConnection->executeStatement(
'INSERT INTO ' . $this->tableNamePrefix . '_dimensionspacepoints (hash, dimensionspacepoint) VALUES (:dimensionspacepointhash, :dimensionspacepoint)',
[
'dimensionspacepointhash' => $hash,
'dimensionspacepoint' => json_encode($dimensionSpacePointCoordinates, JSON_THROW_ON_ERROR)
]
);
} catch (UniqueConstraintViolationException $_) {
if ($this->getCoordinatesByHashFromRuntimeCache($hash) !== null) {
return;
}

$dimensionSpacePointCoordinatesJson = json_encode($dimensionSpacePointCoordinates, JSON_THROW_ON_ERROR);
$this->dimensionspacePointsRuntimeCache[$hash] = $dimensionSpacePointCoordinatesJson;
$this->writeDimensionSpacePoint($hash, $dimensionSpacePointCoordinatesJson);
}

public function getDimensionSpacePointByHash(string $hash): DimensionSpacePoint
public function getOriginDimensionSpacePointByHash(string $hash): OriginDimensionSpacePoint
{
if (!isset($this->dimensionspacePoints[$hash])) {
$this->fillInternalIndex();
$coordinates = $this->getCoordinatesByHashFromRuntimeCache($hash);
if ($coordinates === null) {
$this->fillRuntimeCacheFromDatabase();
$coordinates = $this->getCoordinatesByHashFromRuntimeCache($hash);
}

if ($coordinates === null) {
throw new \RuntimeException(sprintf('A DimensionSpacePoint record with the given hash "%s" was not found in the projection, cannot determine coordinates.', $hash), 1710335509);
}

return DimensionSpacePoint::fromJsonString($this->dimensionspacePoints[$hash]);
return OriginDimensionSpacePoint::fromJsonString($coordinates);
}

public function getOriginDimensionSpacePointByHash(string $hash): OriginDimensionSpacePoint
private function writeDimensionSpacePoint(string $hash, string $dimensionSpacePointCoordinatesJson): void
{
if (!isset($this->dimensionspacePoints[$hash])) {
$this->fillInternalIndex();
}
$this->databaseConnection->executeStatement(
'INSERT IGNORE INTO ' . $this->tableNamePrefix . '_dimensionspacepoints (hash, dimensionspacepoint) VALUES (:dimensionspacepointhash, :dimensionspacepoint)',
[
'dimensionspacepointhash' => $hash,
'dimensionspacepoint' => $dimensionSpacePointCoordinatesJson
]
);
}

return OriginDimensionSpacePoint::fromJsonString($this->dimensionspacePoints[$hash]);
private function getCoordinatesByHashFromRuntimeCache(string $hash): ?string
{
return $this->dimensionspacePointsRuntimeCache[$hash] ?? null;
}

private function fillInternalIndex(): void
private function fillRuntimeCacheFromDatabase(): void
{
$allDimensionSpacePoints = $this->databaseConnection->fetchAllAssociativeIndexed('SELECT hash,dimensionspacepoint FROM ' . $this->tableNamePrefix . '_dimensionspacepoints');
$this->dimensionspacePoints = array_map(static fn ($item) => $item['dimensionspacepoint'], $allDimensionSpacePoints);
$allDimensionSpacePoints = $this->databaseConnection->fetchAllAssociative('SELECT hash, dimensionspacepoint FROM ' . $this->tableNamePrefix . '_dimensionspacepoints');
foreach ($allDimensionSpacePoints as $dimensionSpacePointRow) {
$this->dimensionspacePointsRuntimeCache[(string)$dimensionSpacePointRow['hash']] = (string)$dimensionSpacePointRow['dimensionspacepoint'];
}
}
}