DMD-33 snowflake definitions directly from SchemaReflection

.github/workflows/build-php-db-import-export.yml

@@ -25,10 +25,6 @@ on:
         required: true
       ABS_TERADATA_PASSWORD:
         required: true
-      EXASOL_PASSWORD:
-        required: true
-      EXA_SAAS_TOKEN:
-        required: true
       OAUTH_TOKEN_GITHUB:
         required: true
       GCS_CREDENTIALS:
@@ -78,14 +74,6 @@ env:
   ABS_TERADATA_PASSWORD: ${{ secrets.ABS_TERADATA_PASSWORD }}
   ABS_TERADATA_PORT: 1025
   ABS_TERADATA_DATABASE: ci_ielib_abs
-  # Exasol
-  EXASOL_HOST: mbgghigkizhshorgb53ivhkrsu.clusters.exasol.com:8563
-  EXASOL_USERNAME: devel
-  EXASOL_PASSWORD: ${{ secrets.EXASOL_PASSWORD }}
-  EXA_SAAS_DB_ID: 5ThvKt2NQEqTf-QVEBcNeg
-  EXA_SAAS_HOST: https://cloud.exasol.com
-  EXA_SAAS_USER_ID: org_W0xrKhY2DEnGwmSO
-  EXA_SAAS_TOKEN: ${{ secrets.EXA_SAAS_TOKEN }}
   # Bigquery
   BQ_KEY_FILE: ${{ secrets.BQ_KEY_FILE }}
   BQ_BUCKET_NAME: ie-ci-files-bucket
@@ -481,98 +469,19 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ env.S3_AWS_SECRET_ACCESS_KEY }}
         run: docker compose run ci-php-db-import-export composer tests-${{ matrix.suite }}
 
-  pre-exasol-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    concurrency: exasol-tests
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v4
-      -
-        name: Exasol start
-        run: |
-          bash ./provisioning/scripts/exasolRun.sh -w || exit 1
-
-  exasol-tests:
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: read
-    needs: [load-s3, pre-exasol-tests]
-    continue-on-error: true
-    strategy:
-      fail-fast: false
-      matrix:
-        suite:
-          - exasol
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v4
-      -
-        name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: ${{ env.AWS_CREDENTIALS_ROLE_TO_ASSUME }}
-          aws-region: ${{ env.AWS_CREDENTIALS_REGION }}
-
-      -
-        name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      -
-        name: Pull image
-        env:
-          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: ${{ env.ECR_REPOSITORY }}
-          IMAGE_TAG: ci-php-db-import-export-${{ github.sha }}
-          LOCAL_IMAGE: ${{ env.LOCAL_IMAGE }}
-        run: |
-          docker pull $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
-          docker tag $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $LOCAL_IMAGE:latest
-      #
-      # TESTS
-      #
-      -
-        name: Tests ${{ matrix.suite }}
-        env:
-          SUITE: ${{ matrix.suite }}
-          AWS_ACCESS_KEY_ID: ${{ env.S3_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ env.S3_AWS_SECRET_ACCESS_KEY }}
-        run: docker compose run ci-php-db-import-export composer tests-${{ matrix.suite }}
-
-  post-exasol-tests:
-    runs-on: ubuntu-latest
-    needs: exasol-tests
-    if: |
-      always() && (needs.exasol-tests.result != 'skipped')
-    concurrency: exasol-tests
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v4
-      -
-        name: Exasol stop
-        if: ${{ always() }}
-        run: |
-          bash ./provisioning/scripts/exasolRun.sh -p || exit 1
-
   #
   # Clean up uploaded data
   clean-s3:
     runs-on: ubuntu-latest
     permissions:
       id-token: write
       contents: read
-    needs: [testsS3, testsAll, exasol-tests]
+    needs: [testsS3, testsAll]
     if: |
       always() 
       && (
         (needs.testsS3.result != 'skipped')
         || (needs.testsAll.result != 'skipped')
-        || (needs.exasol-tests.result != 'skipped')
       )
     steps:
       -

.github/workflows/main.yml

@@ -123,8 +123,6 @@ jobs:
       AZURE_CLIENT_SECRET: ${{ secrets.IE_AZURE_CLIENT_SECRET }}
       TERADATA_PASSWORD: ${{ secrets.IE_TERADATA_PASSWORD }}
       ABS_TERADATA_PASSWORD: ${{ secrets.IE_ABS_TERADATA_PASSWORD }}
-      EXASOL_PASSWORD: ${{ secrets.IE_EXASOL_PASSWORD }}
-      EXA_SAAS_TOKEN: ${{ secrets.IE_EXA_SAAS_TOKEN }}
       BQ_KEY_FILE: ${{ secrets.IE_BQ_KEY_FILE }}
       OAUTH_TOKEN_GITHUB: ${{ secrets.IE_OAUTH_TOKEN_GITHUB }}
       GCS_CREDENTIALS: ${{ secrets.IE_GCS_CREDENTIALS }}

packages/php-table-backend-utils/src/Schema/Snowflake/SnowflakeSchemaReflection.php

@@ -5,9 +5,13 @@
 namespace Keboola\TableBackendUtils\Schema\Snowflake;
 
 use Doctrine\DBAL\Connection;
-use Keboola\TableBackendUtils\DataHelper;
+use Keboola\TableBackendUtils\Column\ColumnCollection;
+use Keboola\TableBackendUtils\Column\Snowflake\SnowflakeColumn;
 use Keboola\TableBackendUtils\Escaping\Snowflake\SnowflakeQuote;
 use Keboola\TableBackendUtils\Schema\SchemaReflectionInterface;
+use Keboola\TableBackendUtils\Table\Snowflake\SnowflakeTableDefinition;
+use Keboola\TableBackendUtils\Table\TableType;
+use RuntimeException;
 
 final class SnowflakeSchemaReflection implements SchemaReflectionInterface
 {
@@ -52,4 +56,99 @@ public function getViewsNames(): array
 
         return array_map(static fn($table) => $table['name'], $tables);
     }
+
+    /**
+     * @return array<string, SnowflakeTableDefinition>
+     */
+    public function getDefinitions(): array
+    {
+        $informationsQuery = sprintf(
+            'SELECT TABLE_NAME, TABLE_TYPE, BYTES, ROW_COUNT '.
+            'FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = %s ORDER BY TABLE_NAME;',
+            SnowflakeQuote::quote($this->schemaName),
+        );
+
+        $columnsQuery = sprintf(
+            'SELECT TABLE_NAME, '.
+            'COLUMN_NAME AS "name", DATA_TYPE AS "type", COLUMN_DEFAULT AS "default", IS_NULLABLE AS "null?" '.
+            'FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = %s ORDER BY TABLE_NAME;',
+            SnowflakeQuote::quote($this->schemaName),
+        );
+
+        $primaryKeyQuery = sprintf(
+            'SHOW PRIMARY KEYS',
+        );
+
+        /** @var array<int, array{TABLE_NAME: string, TABLE_TYPE: string, BYTES: int, ROW_COUNT: int}> $informations */
+        $informations = $this->connection->fetchAllAssociative($informationsQuery);
+        /** @var array<int, array{TABLE_NAME: string, name: string, type: string, default: string, null?: string}> $columns */
+        $columns = $this->connection->fetchAllAssociative($columnsQuery);
+        /** @var array<int, array{
+         *     created_on: string,
+         *     database_name: string,
+         *     schema_name: string,
+         *     table_name: string,
+         *     column_name: string,
+         *     key_sequence: int,
+         *     constraint_name: string,
+         *     rely: bool,
+         *     comment: ?string}> $primaryKeys */
+        $primaryKeys = $this->connection->fetchAllAssociative($primaryKeyQuery);
+
+        $tables = [];
+
+        foreach ($informations as $information) {
+            $tables[$information['TABLE_NAME']]['PROPS'] = $information;
+
+            switch (strtoupper($information['TABLE_TYPE'])) {
+                case 'BASE TABLE':
+                    $tables[$information['TABLE_NAME']]['PROPS']['TEMPORARY'] = false;
+                    $tables[$information['TABLE_NAME']]['PROPS']['TABLE_TYPE'] = TableType::TABLE;
+                    break;
+                case 'EXTERNAL TABLE':
+                    $tables[$information['TABLE_NAME']]['PROPS']['TEMPORARY'] = false;
+                    $tables[$information['TABLE_NAME']]['PROPS']['TABLE_TYPE'] = TableType::SNOWFLAKE_EXTERNAL;
+                    break;
+                case 'LOCAL TEMPORARY':
+                case 'TEMPORARY TABLE':
+                    $tables[$information['TABLE_NAME']]['PROPS']['TEMPORARY'] = true;
+                    $tables[$information['TABLE_NAME']]['PROPS']['TABLE_TYPE'] = TableType::TABLE;
+                    break;
+                case 'VIEW':
+                    $tables[$information['TABLE_NAME']]['PROPS']['TEMPORARY'] = false;
+                    $tables[$information['TABLE_NAME']]['PROPS']['TABLE_TYPE'] = TableType::VIEW;
+                    break;
+                default:
+                    throw new RuntimeException(sprintf(
+                        'Table type "%s" is not known.',
+                        $information['TABLE_TYPE'],
+                    ));
+            }
+        }
+
+        foreach ($columns as $column) {
+            // Offset 'null?' does not exist on
+            // array{TABLE_NAME: string, name: string, type: string, default: string, null?: string}.
+            // @phpstan-ignore-next-line
+            $column['null?'] = ($column['null?'] === 'YES' ? 'Y' : 'N');
+            $tables[$column['TABLE_NAME']]['COLUMNS'][] = SnowflakeColumn::createFromDB($column);
+        }
+
+        foreach ($primaryKeys as $primaryKey) {
+            $tables[$primaryKey['table_name']]['PRIMARY_KEYS'][] = $primaryKey['column_name'];
+        }
+
+        $definitions = [];
+        foreach ($tables as $tableName => $table) {
+            $definitions[$tableName] = new SnowflakeTableDefinition(
+                $this->schemaName,
+                $tableName,
+                $table['PROPS']['TEMPORARY'],
+                new ColumnCollection($table['COLUMNS'] ?? []),
+                $table['PRIMARY_KEYS'] ?? [],
+                $table['PROPS']['TABLE_TYPE'],
+            );
+        }
+        return $definitions;
+    }
 }

packages/php-table-backend-utils/tests/Functional/Snowflake/Schema/SnowflakeSchemaReflectionTest.php

@@ -7,6 +7,7 @@
 use Keboola\TableBackendUtils\Escaping\Snowflake\SnowflakeQuote;
 use Keboola\TableBackendUtils\Schema\Snowflake\SnowflakeSchemaReflection;
 use Tests\Keboola\TableBackendUtils\Functional\Snowflake\SnowflakeBaseCase;
+use function PHPUnit\Framework\assertEquals;
 
 class SnowflakeSchemaReflectionTest extends SnowflakeBaseCase
 {
@@ -78,4 +79,80 @@ public function testListViews(): void
         $this->connection->executeQuery($sql);
         self::assertSame([$viewName], $this->schemaRef->getViewsNames());
     }
+
+    public function testGetDefinitions(): void
+    {
+        $this->initTable();
+
+        // create transient table
+        $this->connection->executeQuery(
+            sprintf(
+                'CREATE OR REPLACE TRANSIENT TABLE %s.%s (
+            "id" INTEGER,
+    "first_name" VARCHAR(100),
+    "last_name" VARCHAR(100)
+);',
+                SnowflakeQuote::quoteSingleIdentifier(self::TEST_SCHEMA),
+                SnowflakeQuote::quoteSingleIdentifier('transient_table'),
+            ),
+        );
+
+        // create temporary table
+        $this->connection->executeQuery(
+            sprintf(
+                'CREATE OR REPLACE TEMPORARY TABLE %s.%s (
+            "id" INTEGER,
+    "first_name" VARCHAR(100),
+    "last_name" VARCHAR(100)
+);',
+                SnowflakeQuote::quoteSingleIdentifier(self::TEST_SCHEMA),
+                SnowflakeQuote::quoteSingleIdentifier('temporary_table'),
+            ),
+        );
+
+        $tableName = self::TABLE_GENERIC;
+        $schemaName = self::TEST_SCHEMA;
+        $viewName = self::VIEW_GENERIC;
+        $sql = sprintf(
+            '
+CREATE VIEW %s.%s AS
+     SELECT   "first_name",
+              "last_name" 
+     FROM %s.%s;
+',
+            SnowflakeQuote::quoteSingleIdentifier($schemaName),
+            SnowflakeQuote::quoteSingleIdentifier($viewName),
+            SnowflakeQuote::quoteSingleIdentifier($schemaName),
+            SnowflakeQuote::quoteSingleIdentifier($tableName),
+        );
+        $this->connection->executeQuery($sql);
+
+        $definitions = $this->schemaRef->getDefinitions();
+
+        self::assertCount(4, $definitions);
+
+        $temporaryTableKey = 'temporary_table';
+        self::assertEquals('temporary_table', $definitions[$temporaryTableKey]->getTableName());
+        self::assertEquals(3, $definitions[$temporaryTableKey]->getColumnsDefinitions()->count());
+        self::assertTrue($definitions[$temporaryTableKey]->isTemporary());
+        self::assertEquals('table', $definitions[$temporaryTableKey]->getTableType()->value);
+
+        $transientTableKey = 'transient_table';
+        self::assertEquals('transient_table', $definitions[$transientTableKey]->getTableName());
+        self::assertEquals(3, $definitions[$transientTableKey]->getColumnsDefinitions()->count());
+        self::assertFalse($definitions[$transientTableKey]->isTemporary());
+        self::assertEquals('table', $definitions[$transientTableKey]->getTableType()->value);
+
+        $genericTableKey = self::TABLE_GENERIC;
+        self::assertEquals(self::TABLE_GENERIC, $definitions[$genericTableKey]->getTableName());
+        self::assertEquals(3, $definitions[$genericTableKey]->getColumnsDefinitions()->count());
+        self::assertFalse($definitions[$genericTableKey]->isTemporary());
+        self::assertEquals('table', $definitions[$genericTableKey]->getTableType()->value);
+
+        $genericViewKey = self::VIEW_GENERIC;
+        self::assertEquals(self::VIEW_GENERIC, $definitions[$genericViewKey]->getTableName());
+        self::assertEquals(2, $definitions[$genericViewKey]->getColumnsDefinitions()->count());
+        self::assertFalse($definitions[$genericViewKey]->isTemporary());
+        self::assertEquals('view', $definitions[$genericViewKey]->getTableType()->value);
+    }
 }