Skip to content

Commit

Permalink
Merge pull request #127 from keboola/CT-1169-put-column-definition-en…
Browse files Browse the repository at this point in the history 
…dpoint

CT-1169 add parsing of precision&scale to SNFLK
  • Loading branch information
@tomasfejfar
tomasfejfar authored Feb 15, 2024
2 parents 60a42b3 + f3b83ba commit 1b4307e
Show file tree
Hide file tree
Showing 5 changed files with 334 additions and 0 deletions.
6 changes: 6 additions & 0 deletions packages/php-datatypes/src/Definition/Common.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ abstract class Common implements DefinitionInterface
public const KBC_METADATA_KEY_COMPRESSION = 'KBC.datatype.compression';
public const KBC_METADATA_KEY_FORMAT = 'KBC.datatype.format';

public const KBC_METADATA_KEYS_FOR_COLUMNS_SYNC = [
self::KBC_METADATA_KEY_NULLABLE,
self::KBC_METADATA_KEY_LENGTH,
self::KBC_METADATA_KEY_DEFAULT,
];

protected string $type;

protected ?string $length = null;
Expand Down
36 changes: 36 additions & 0 deletions packages/php-datatypes/src/Definition/Snowflake.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

class Snowflake extends Common
{
public const TYPES_WITH_COMPLEX_LENGTH = [self::TYPE_NUMBER, self::TYPE_DECIMAL, self::TYPE_NUMERIC];
public const METADATA_BACKEND = 'snowflake';
public const TYPE_NUMBER = 'NUMBER';
public const TYPE_DECIMAL = 'DECIMAL';
Expand Down Expand Up @@ -195,6 +196,41 @@ private function getLengthFromArray(array $lengthOptions): ?string
return $numericPrecision === null ? null : (string) $numericPrecision;
}

/**
* @return array{
* character_maximum?:string|int|null,
* numeric_precision?:string|int|null,
* numeric_scale?:string|int|null
* }
*/
public function getArrayFromLength(): array
{
if ($this->isTypeWithComplexLength()) {
if ($this->getLength() === null || $this->getLength() === '') {
$parsed = [];
} else {
$parsed = array_map(intval(...), explode(',', (string) $this->getLength()));
}
$parsed = $parsed + [38, 0];
return ['numeric_precision' => $parsed[0], 'numeric_scale' => $parsed[1]];
}
return ['character_maximum' => $this->getLength()];
}

/**
* @phpstan-assert-if-true array{
* numeric_precision:string,
* numeric_scale:string
* } $this->getArrayFromLength()
* @phpstan-assert-if-false array{
* character_maximum:string
* } $this->getArrayFromLength()
*/
public function isTypeWithComplexLength(): bool
{
return in_array($this->getType(), self::TYPES_WITH_COMPLEX_LENGTH, true);
}

/**
* @throws InvalidTypeException
*/
Expand Down
40 changes: 40 additions & 0 deletions packages/php-datatypes/tests/SnowflakeDatatypeTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace Keboola\DatatypeTest;

use Generator;
use Keboola\Datatype\Definition\Exception\InvalidLengthException;
use Keboola\Datatype\Definition\Exception\InvalidOptionException;
use Keboola\Datatype\Definition\Exception\InvalidTypeException;
Expand Down Expand Up @@ -381,4 +382,43 @@ public function testGetTypeByBasetype(): void
$this->expectExceptionMessage('Base type "FOO" is not valid.');
Snowflake::getTypeByBasetype('foo');
}

/**
* @param array<string, mixed> $expectedArray
* @dataProvider arrayFromLengthProvider
*/
public function testArrayFromLength(string $type, ?string $length, array $expectedArray): void
{
$definition = new Snowflake($type, ['length' => $length]);
$this->assertSame($expectedArray, $definition->getArrayFromLength());
}

public function arrayFromLengthProvider(): Generator
{
yield 'simple' => [
'VARCHAR',
'10',
['character_maximum' => '10'],
];
yield 'decimal' => [
'NUMERIC',
'38,2',
['numeric_precision' => 38, 'numeric_scale' => 2],
];
yield 'with zero scale' => [
'NUMERIC',
'38,0',
['numeric_precision' => 38, 'numeric_scale' => 0],
];
yield 'with null length' => [
'NUMERIC',
null,
['numeric_precision' => 38, 'numeric_scale' => 0],
];
yield 'numeric with int length' => [
'NUMERIC',
'10',
['numeric_precision' => 10, 'numeric_scale' => 0],
];
}
}
136 changes: 136 additions & 0 deletions packages/php-table-backend-utils/src/Table/Snowflake/SnowflakeTableQueryBuilder.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@

class SnowflakeTableQueryBuilder implements TableQueryBuilderInterface
{
private const CANNOT_CHANGE_DEFAULT_VALUE = 'cannotChangeDefaultValue';
private const CANNOT_CHANGE_SCALE = 'cannotChangeScale';
private const CANNOT_DECREASE_LENGTH = 'cannotDecreaseLength';
private const CANNOT_DECREASE_PRECISION = 'cannotDecreasePrecision';
private const CANNOT_INTRODUCE_COMPLEX_LENGTH = 'cannotIntroduceComplexLength';
private const CANNOT_REDUCE_COMPLEX_LENGTH = 'cannotReduceComplexLength';
private const INVALID_PKS_FOR_TABLE = 'invalidPKs';
private const INVALID_TABLE_NAME = 'invalidTableName';
public const TEMP_TABLE_PREFIX = '__temp_';
Expand Down Expand Up @@ -214,4 +220,134 @@ public static function buildTempTableName(string $realTableName): string
{
return self::TEMP_TABLE_PREFIX . $realTableName;
}

public function getUpdateColumnFromDefinitionQuery(
Snowflake $existingColumnDefinition,
Snowflake $desiredColumnDefinition,
string $schemaName,
string $tableName,
string $columnName,
): string {
$sql = sprintf(
'ALTER TABLE %s.%s MODIFY ',
SnowflakeQuote::quoteSingleIdentifier($schemaName),
SnowflakeQuote::quoteSingleIdentifier($tableName),
);
$sqlParts = [];
// allowed from https://docs.snowflake.com/en/sql-reference/sql/alter-table-column

// drop default
if ($existingColumnDefinition->getDefault() !== null
&& $desiredColumnDefinition->getDefault() === null) {
$sqlParts[] = 'DROP DEFAULT';
} elseif ($existingColumnDefinition->getDefault() !== $desiredColumnDefinition->getDefault()) {
throw new QueryBuilderException(
sprintf(
'Cannot change default value of column "%s" from "%s" to "%s"',
$columnName,
$existingColumnDefinition->getDefault(),
$desiredColumnDefinition->getDefault(),
),
self::CANNOT_CHANGE_DEFAULT_VALUE,
);
}

if ($existingColumnDefinition->isNullable() !== $desiredColumnDefinition->isNullable()) {
$sqlParts[] = $desiredColumnDefinition->isNullable() ? 'DROP NOT NULL' : 'SET NOT NULL';
}

$notSameLength = $existingColumnDefinition->getLength() !== $desiredColumnDefinition->getLength();
$isNewLengthBigger = $existingColumnDefinition->getLength() < $desiredColumnDefinition->getLength();

// increase precision
if ($existingColumnDefinition->isTypeWithComplexLength() && $notSameLength) {
if (!$desiredColumnDefinition->isTypeWithComplexLength()) {
throw new QueryBuilderException(
sprintf(
'Cannot reduce column "%s" with complex length "%s" to simple length "%s"',
$columnName,
$existingColumnDefinition->getLength(),
$desiredColumnDefinition->getLength(),
),
self::CANNOT_REDUCE_COMPLEX_LENGTH,
);
}
[
'numeric_precision' => $existingPrecision,
'numeric_scale' => $existingScale,
] = $existingColumnDefinition->getArrayFromLength();
[
'numeric_precision' => $desiredPrecision,
'numeric_scale' => $desiredScale,
] = $desiredColumnDefinition->getArrayFromLength();

if ($existingScale !== $desiredScale) {
throw new QueryBuilderException(
sprintf(
'Cannot change scale of a column "%s" from "%s" to "%s"',
$columnName,
$existingScale,
$desiredScale,
),
self::CANNOT_CHANGE_SCALE,
);
}

if ($existingPrecision < $desiredPrecision) {
$sqlParts[] = sprintf(
'SET DATA TYPE %s(%s, %s)',
$desiredColumnDefinition->getType(),
$desiredPrecision,
$desiredScale,
);
} else {
throw new QueryBuilderException(
sprintf(
'Cannot decrease precision of column "%s" from "%s" to "%s"',
$columnName,
$existingPrecision,
$desiredPrecision,
),
self::CANNOT_DECREASE_PRECISION,
);
}
} elseif ($notSameLength && $isNewLengthBigger) {
if ($desiredColumnDefinition->isTypeWithComplexLength()) {
throw new QueryBuilderException(
sprintf(
'Cannot convert column "%s" from simple length "%s" to complex length "%s"',
$columnName,
$existingColumnDefinition->getLength(),
$desiredColumnDefinition->getLength(),
),
self::CANNOT_INTRODUCE_COMPLEX_LENGTH,
);
}
// increase length
$sqlParts[] = sprintf(
'SET DATA TYPE %s(%s)',
$desiredColumnDefinition->getType(),
$desiredColumnDefinition->getLength(),
);
} elseif ($notSameLength) {
throw new QueryBuilderException(
sprintf(
'Cannot decrease length of column "%s" from "%s" to "%s"',
$columnName,
$existingColumnDefinition->getLength(),
$desiredColumnDefinition->getLength(),
),
self::CANNOT_DECREASE_LENGTH,
);
}

$partsWithColumnPrefix = array_map(function (string $part) use ($columnName) {
return sprintf(
'COLUMN %s %s',
SnowflakeQuote::quoteSingleIdentifier($columnName),
$part,
);
}, $sqlParts);
return $sql . implode(', ', $partsWithColumnPrefix);
}
}
116 changes: 116 additions & 0 deletions ...ges/php-table-backend-utils/tests/Unit/Table/Snowflake/SnowflakeTableQueryBuilderTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,4 +93,120 @@ public function testGetTruncateTable(): void
$dropTableCommand = $this->qb->getTruncateTableCommand('testDb', 'testTable');
self::assertEquals('TRUNCATE TABLE "testDb"."testTable"', $dropTableCommand);
}

/**
* @dataProvider provideGetColumnDefinitionUpdate
*/
public function testGetColumnDefinitionUpdate(
Snowflake $existingColumn,
Snowflake $desiredColumn,
string $expectedQuery,
): void {
$existingColumnDefinition = $existingColumn;
$desiredColumnDefinition = $desiredColumn;
$sql = $this->qb->getUpdateColumnFromDefinitionQuery(
$existingColumnDefinition,
$desiredColumnDefinition,
'testDb',
'testTable',
'testColumn',
);
self::assertEquals(
$expectedQuery,
$sql,
);
}

/**
* @dataProvider provideInvalidGetColumnDefinitionUpdate
*/
public function testInvalidGetColumnDefinitionUpdate(
Snowflake $existingColumn,
Snowflake $desiredColumn,
string $expectedExceptionMessage,
): void {
$existingColumnDefinition = $existingColumn;
$desiredColumnDefinition = $desiredColumn;
$this->expectExceptionMessage($expectedExceptionMessage);
$this->qb->getUpdateColumnFromDefinitionQuery(
$existingColumnDefinition,
$desiredColumnDefinition,
'testDb',
'testTable',
'testColumn',
);
}

/**
* @return \Generator<string, array{Snowflake,Snowflake,string}>
*/
public function provideGetColumnDefinitionUpdate(): Generator
{
yield 'drop default' => [
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => true, 'default' => '10']),
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => true, 'default' => null]),
/** @lang Snowflake */
'ALTER TABLE "testDb"."testTable" MODIFY COLUMN "testColumn" DROP DEFAULT',
];
yield 'add nullable' => [
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => false, 'default' => '']),
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => true, 'default' => '']),
/** @lang Snowflake */
'ALTER TABLE "testDb"."testTable" MODIFY COLUMN "testColumn" DROP NOT NULL',
];
yield 'drop nullable' => [
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => true, 'default' => '']),
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => false, 'default' => '']),
/** @lang Snowflake */
'ALTER TABLE "testDb"."testTable" MODIFY COLUMN "testColumn" SET NOT NULL',
];
yield 'increase length of text column' => [
new Snowflake('VARCHAR', ['length' => '12', 'nullable' => true, 'default' => '']),
new Snowflake('VARCHAR', ['length' => '38', 'nullable' => true, 'default' => '']),
/** @lang Snowflake */
'ALTER TABLE "testDb"."testTable" MODIFY COLUMN "testColumn" SET DATA TYPE VARCHAR(38)',
];
yield 'increase precision of numeric column' => [
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => true, 'default' => '']),
new Snowflake('NUMERIC', ['length' => '14,8', 'nullable' => true, 'default' => '']),
/** @lang Snowflake */
'ALTER TABLE "testDb"."testTable" MODIFY COLUMN "testColumn" SET DATA TYPE NUMERIC(14, 8)',
];
yield 'full set of changes (increase precision, drop nullable, drop default)' => [
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => true, 'default' => 'grunbread']),
new Snowflake('NUMERIC', ['length' => '14,8', 'nullable' => false, 'default' => '']),
/** @lang Snowflake */
'ALTER TABLE "testDb"."testTable" MODIFY COLUMN "testColumn" DROP DEFAULT, '
. 'COLUMN "testColumn" SET NOT NULL, COLUMN "testColumn" SET DATA TYPE NUMERIC(14, 8)',
];
}

public function provideInvalidGetColumnDefinitionUpdate(): Generator
{
yield 'add default' => [
new Snowflake('VARCHAR', ['length' => '10', 'nullable' => true, 'default' => '']),
new Snowflake('VARCHAR', ['length' => '10', 'nullable' => true, 'default' => 'Bedight']),
'Cannot change default value of column "testColumn" from "" to "Bedight"',
];
yield 'change default' => [
new Snowflake('VARCHAR', ['length' => '10', 'nullable' => true, 'default' => 'Bedight']),
new Snowflake('VARCHAR', ['length' => '10', 'nullable' => true, 'default' => 'Brabble']),
'Cannot change default value of column "testColumn" from "Bedight" to "Brabble"',
];
yield 'descrease length of string' => [
new Snowflake('VARCHAR', ['length' => '10', 'nullable' => true, 'default' => 'Bedight']),
new Snowflake('VARCHAR', ['length' => '8', 'nullable' => true, 'default' => 'Bedight']),
'Cannot decrease length of column "testColumn" from "10" to "8"',
];
yield 'descrease precision of number' => [
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => true, 'default' => '']),
new Snowflake('NUMERIC', ['length' => '10,8', 'nullable' => true, 'default' => '']),
'Cannot decrease precision of column "testColumn" from "12" to "10"',
];
yield 'change scale of number' => [
new Snowflake('NUMERIC', ['length' => '12,8', 'nullable' => true, 'default' => '']),
new Snowflake('NUMERIC', ['length' => '12,10', 'nullable' => true, 'default' => '']),
'Cannot change scale of a column "testColumn" from "8" to "10"',
];
}
}

0 comments on commit 1b4307e

Please sign in to comment.