diff --git a/pkg/datasources/external_tables.go b/pkg/datasources/external_tables.go index 47dac65a27..32b2897462 100644 --- a/pkg/datasources/external_tables.go +++ b/pkg/datasources/external_tables.go @@ -1,12 +1,14 @@ package datasources import ( + "context" "database/sql" - "errors" - "fmt" "log" - "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/snowflake" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/helpers" + + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) @@ -58,34 +60,31 @@ func ExternalTables() *schema.Resource { func ReadExternalTables(d *schema.ResourceData, meta interface{}) error { db := meta.(*sql.DB) + ctx := context.Background() + client := sdk.NewClientFromDB(db) databaseName := d.Get("database").(string) schemaName := d.Get("schema").(string) - currentExternalTables, err := snowflake.ListExternalTables(databaseName, schemaName, db) - if errors.Is(err, sql.ErrNoRows) { - // If not found, mark resource to be removed from state file during apply or refresh - log.Printf("[DEBUG] external tables in schema (%s) not found", d.Id()) - d.SetId("") - return nil - } else if err != nil { - log.Printf("[DEBUG] unable to parse external tables in schema (%s)", d.Id()) + schemaId := sdk.NewDatabaseObjectIdentifier(databaseName, schemaName) + showIn := sdk.NewShowExternalTableInRequest().WithSchema(schemaId) + externalTables, err := client.ExternalTables.Show(ctx, sdk.NewShowExternalTableRequest().WithIn(showIn)) + if err != nil { + log.Printf("[DEBUG] failed when searching external tables in schema (%s), err = %s", schemaId.FullyQualifiedName(), err.Error()) d.SetId("") return nil } - externalTables := []map[string]interface{}{} - - for _, externalTable := range currentExternalTables { - externalTableMap := map[string]interface{}{} - - externalTableMap["name"] = externalTable.ExternalTableName.String - externalTableMap["database"] = externalTable.DatabaseName.String - externalTableMap["schema"] = externalTable.SchemaName.String - externalTableMap["comment"] = externalTable.Comment.String - - externalTables = append(externalTables, externalTableMap) + externalTablesObjects := make([]map[string]any, len(externalTables)) + for i, externalTable := range externalTables { + externalTablesObjects[i] = map[string]any{ + "name": externalTable.Name, + "database": externalTable.DatabaseName, + "schema": externalTable.SchemaName, + "comment": externalTable.Comment, + } } - d.SetId(fmt.Sprintf(`%v|%v`, databaseName, schemaName)) - return d.Set("external_tables", externalTables) + d.SetId(helpers.EncodeSnowflakeID(schemaId)) + + return d.Set("external_tables", externalTablesObjects) } diff --git a/pkg/resources/external_table.go b/pkg/resources/external_table.go index b236a0c4cf..ddba4027a4 100644 --- a/pkg/resources/external_table.go +++ b/pkg/resources/external_table.go @@ -1,21 +1,16 @@ package resources import ( - "bytes" + "context" "database/sql" - "encoding/csv" "fmt" "log" - "strings" - "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/snowflake" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/helpers" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) -const ( - externalTableIDDelimiter = '|' -) - var externalTableSchema = map[string]*schema.Schema{ "name": { Type: schema.TypeString, @@ -50,10 +45,11 @@ var externalTableSchema = map[string]*schema.Schema{ ForceNew: true, }, "type": { - Type: schema.TypeString, - Required: true, - Description: "Column type, e.g. VARIANT", - ForceNew: true, + Type: schema.TypeString, + Required: true, + Description: "Column type, e.g. VARIANT", + ForceNew: true, + ValidateFunc: IsDataType(), }, "as": { Type: schema.TypeString, @@ -144,204 +140,141 @@ func ExternalTable() *schema.Resource { } } -type externalTableID struct { - DatabaseName string - SchemaName string - ExternalTableName string -} - -// String() takes in a externalTableID object and returns a pipe-delimited string: -// DatabaseName|SchemaName|ExternalTableName. -func (si *externalTableID) String() (string, error) { - var buf bytes.Buffer - csvWriter := csv.NewWriter(&buf) - csvWriter.Comma = externalTableIDDelimiter - dataIdentifiers := [][]string{{si.DatabaseName, si.SchemaName, si.ExternalTableName}} - if err := csvWriter.WriteAll(dataIdentifiers); err != nil { - return "", err - } - strExternalTableID := strings.TrimSpace(buf.String()) - return strExternalTableID, nil -} - -// externalTableIDFromString() takes in a pipe-delimited string: DatabaseName|SchemaName|ExternalTableName -// and returns a externalTableID object. -func externalTableIDFromString(stringID string) (*externalTableID, error) { - reader := csv.NewReader(strings.NewReader(stringID)) - reader.Comma = externalTableIDDelimiter - lines, err := reader.ReadAll() - if err != nil { - return nil, fmt.Errorf("not CSV compatible") - } - - if len(lines) != 1 { - return nil, fmt.Errorf("1 line at a time") - } - if len(lines[0]) != 3 { - return nil, fmt.Errorf("3 fields allowed") - } - - externalTableResult := &externalTableID{ - DatabaseName: lines[0][0], - SchemaName: lines[0][1], - ExternalTableName: lines[0][2], - } - return externalTableResult, nil -} - // CreateExternalTable implements schema.CreateFunc. -func CreateExternalTable(d *schema.ResourceData, meta interface{}) error { +func CreateExternalTable(d *schema.ResourceData, meta any) error { db := meta.(*sql.DB) + ctx := context.Background() + client := sdk.NewClientFromDB(db) + database := d.Get("database").(string) - dbSchema := d.Get("schema").(string) + schema := d.Get("schema").(string) name := d.Get("name").(string) - - // This type conversion is due to the test framework in the terraform-plugin-sdk having limited support - // for data types in the HCL2ValueFromConfigValue method. - columns := []map[string]string{} - for _, column := range d.Get("column").([]interface{}) { + id := sdk.NewSchemaObjectIdentifier(database, schema, name) + location := d.Get("location").(string) + fileFormat := d.Get("file_format").(string) + req := sdk.NewCreateExternalTableRequest(id, location).WithRawFileFormat(&fileFormat) + + tableColumns := d.Get("column").([]any) + columnRequests := make([]*sdk.ExternalTableColumnRequest, len(tableColumns)) + for i, col := range tableColumns { columnDef := map[string]string{} - for key, val := range column.(map[string]interface{}) { + for key, val := range col.(map[string]any) { columnDef[key] = val.(string) } - columns = append(columns, columnDef) + + name := columnDef["name"] + dataTypeString := columnDef["type"] + dataType, err := sdk.ToDataType(dataTypeString) + if err != nil { + return fmt.Errorf(`failed to parse datatype: %s`, dataTypeString) + } + as := columnDef["as"] + columnRequests[i] = sdk.NewExternalTableColumnRequest(name, dataType, as) } - builder := snowflake.NewExternalTableBuilder(name, database, dbSchema) - builder.WithColumns(columns) - builder.WithFileFormat(d.Get("file_format").(string)) - builder.WithLocation(d.Get("location").(string)) + req.WithColumns(columnRequests) - builder.WithAutoRefresh(d.Get("auto_refresh").(bool)) - builder.WithRefreshOnCreate(d.Get("refresh_on_create").(bool)) - builder.WithCopyGrants(d.Get("copy_grants").(bool)) + req.WithAutoRefresh(sdk.Bool(d.Get("auto_refresh").(bool))) + req.WithRefreshOnCreate(sdk.Bool(d.Get("refresh_on_create").(bool))) + req.WithCopyGrants(sdk.Bool(d.Get("copy_grants").(bool))) - // Set optionals if v, ok := d.GetOk("partition_by"); ok { - partitionBys := expandStringList(v.([]interface{})) - builder.WithPartitionBys(partitionBys) + req.WithPartitionBy(v.([]string)) } if v, ok := d.GetOk("pattern"); ok { - builder.WithPattern(v.(string)) + req.WithPattern(sdk.String(v.(string))) } if v, ok := d.GetOk("aws_sns_topic"); ok { - builder.WithAwsSNSTopic(v.(string)) + req.WithAwsSnsTopic(sdk.String(v.(string))) } if v, ok := d.GetOk("comment"); ok { - builder.WithComment(v.(string)) - } - - if v, ok := d.GetOk("tag"); ok { - tags := getTags(v) - builder.WithTags(tags.toSnowflakeTagValues()) + req.WithComment(sdk.String(v.(string))) } - stmt := builder.Create() - if err := snowflake.Exec(db, stmt); err != nil { - return fmt.Errorf("error creating externalTable %v err = %w", name, err) + if _, ok := d.GetOk("tag"); ok { + tagAssociations := getPropertyTags(d, "tag") + tagAssociationRequests := make([]*sdk.TagAssociationRequest, len(tagAssociations)) + for i, t := range tagAssociations { + tagAssociationRequests[i] = sdk.NewTagAssociationRequest(t.Name, t.Value) + } + req.WithTag(tagAssociationRequests) } - externalTableID := &externalTableID{ - DatabaseName: database, - SchemaName: dbSchema, - ExternalTableName: name, - } - dataIDInput, err := externalTableID.String() - if err != nil { + if err := client.ExternalTables.Create(ctx, req); err != nil { return err } - d.SetId(dataIDInput) + d.SetId(helpers.EncodeSnowflakeID(id)) return ReadExternalTable(d, meta) } // ReadExternalTable implements schema.ReadFunc. -func ReadExternalTable(d *schema.ResourceData, meta interface{}) error { +func ReadExternalTable(d *schema.ResourceData, meta any) error { db := meta.(*sql.DB) - externalTableID, err := externalTableIDFromString(d.Id()) - if err != nil { - return err - } + ctx := context.Background() + client := sdk.NewClientFromDB(db) + id := helpers.DecodeSnowflakeID(d.Id()).(sdk.SchemaObjectIdentifier) - dbName := externalTableID.DatabaseName - schema := externalTableID.SchemaName - name := externalTableID.ExternalTableName - - stmt := snowflake.NewExternalTableBuilder(name, dbName, schema).Show() - row := snowflake.QueryRow(db, stmt) - externalTable, err := snowflake.ScanExternalTable(row) + externalTable, err := client.ExternalTables.ShowByID(ctx, sdk.NewShowExternalTableByIDRequest(id)) if err != nil { - if err.Error() == snowflake.ErrNoRowInRS { - log.Printf("[DEBUG] external table (%s) not found", d.Id()) - d.SetId("") - return nil - } + log.Printf("[DEBUG] external table (%s) not found", d.Id()) + d.SetId("") return err } - if err := d.Set("name", externalTable.ExternalTableName.String); err != nil { + if err := d.Set("name", externalTable.Name); err != nil { return err } - if err := d.Set("owner", externalTable.Owner.String); err != nil { + if err := d.Set("owner", externalTable.Owner); err != nil { return err } + return nil } // UpdateExternalTable implements schema.UpdateFunc. -func UpdateExternalTable(d *schema.ResourceData, meta interface{}) error { +func UpdateExternalTable(d *schema.ResourceData, meta any) error { db := meta.(*sql.DB) - database := d.Get("database").(string) - dbSchema := d.Get("schema").(string) - name := d.Get("name").(string) - - builder := snowflake.NewExternalTableBuilder(name, database, dbSchema) + ctx := context.Background() + client := sdk.NewClientFromDB(db) + id := helpers.DecodeSnowflakeID(d.Id()).(sdk.SchemaObjectIdentifier) if d.HasChange("tag") { - v := d.Get("tag") - tags := getTags(v) - builder.WithTags(tags.toSnowflakeTagValues()) - } + unsetTags, setTags := GetTagsDiff(d, "tag") - stmt := builder.Update() - if err := snowflake.Exec(db, stmt); err != nil { - return fmt.Errorf("error updating externalTable %v err = %w", name, err) - } + err := client.ExternalTables.Alter(ctx, sdk.NewAlterExternalTableRequest(id).WithUnsetTag(unsetTags)) + if err != nil { + return fmt.Errorf("error setting tags on %v, err = %w", d.Id(), err) + } - externalTableID := &externalTableID{ - DatabaseName: database, - SchemaName: dbSchema, - ExternalTableName: name, - } - dataIDInput, err := externalTableID.String() - if err != nil { - return err + tagAssociationRequests := make([]*sdk.TagAssociationRequest, len(setTags)) + for i, t := range setTags { + tagAssociationRequests[i] = sdk.NewTagAssociationRequest(t.Name, t.Value) + } + err = client.ExternalTables.Alter(ctx, sdk.NewAlterExternalTableRequest(id).WithSetTag(tagAssociationRequests)) + if err != nil { + return fmt.Errorf("error setting tags on %v, err = %w", d.Id(), err) + } } - d.SetId(dataIDInput) return ReadExternalTable(d, meta) } // DeleteExternalTable implements schema.DeleteFunc. -func DeleteExternalTable(d *schema.ResourceData, meta interface{}) error { +func DeleteExternalTable(d *schema.ResourceData, meta any) error { db := meta.(*sql.DB) - externalTableID, err := externalTableIDFromString(d.Id()) + ctx := context.Background() + client := sdk.NewClientFromDB(db) + id := helpers.DecodeSnowflakeID(d.Id()).(sdk.SchemaObjectIdentifier) + + err := client.ExternalTables.Drop(ctx, sdk.NewDropExternalTableRequest(id)) if err != nil { return err } - dbName := externalTableID.DatabaseName - schema := externalTableID.SchemaName - externalTableName := externalTableID.ExternalTableName - - q := snowflake.NewExternalTableBuilder(externalTableName, dbName, schema).Drop() - if err := snowflake.Exec(db, q); err != nil { - return fmt.Errorf("error deleting pipe %v err = %w", d.Id(), err) - } - d.SetId("") return nil diff --git a/pkg/resources/external_table_acceptance_test.go b/pkg/resources/external_table_acceptance_test.go index d6ea4367db..1cb407b126 100644 --- a/pkg/resources/external_table_acceptance_test.go +++ b/pkg/resources/external_table_acceptance_test.go @@ -1,23 +1,29 @@ package resources_test import ( + "context" + "database/sql" "fmt" "os" "strings" "testing" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" + "github.com/hashicorp/terraform-plugin-testing/config" + "github.com/hashicorp/terraform-plugin-testing/terraform" + "github.com/hashicorp/terraform-plugin-testing/tfversion" + acc "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/acceptance" "github.com/hashicorp/terraform-plugin-testing/helper/acctest" "github.com/hashicorp/terraform-plugin-testing/helper/resource" ) -func TestAcc_ExternalTable(t *testing.T) { +func TestAcc_ExternalTable_basic(t *testing.T) { env := os.Getenv("SKIP_EXTERNAL_TABLE_TEST") if env != "" { t.Skip("Skipping TestAcc_ExternalTable") } - accName := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) - + name := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) bucketURL := os.Getenv("AWS_EXTERNAL_BUCKET_URL") if bucketURL == "" { t.Skip("Skipping TestAcc_ExternalTable") @@ -26,59 +32,60 @@ func TestAcc_ExternalTable(t *testing.T) { if roleName == "" { t.Skip("Skipping TestAcc_ExternalTable") } + resourceName := "snowflake_external_table.test_table" + + configVariables := map[string]config.Variable{ + "name": config.StringVariable(name), + "location": config.StringVariable(bucketURL), + "aws_arn": config.StringVariable(roleName), + "database": config.StringVariable(acc.TestDatabaseName), + "schema": config.StringVariable(acc.TestSchemaName), + } + resource.Test(t, resource.TestCase{ - Providers: acc.TestAccProviders(), - PreCheck: func() { acc.TestAccPreCheck(t) }, - CheckDestroy: nil, + ProtoV6ProviderFactories: acc.TestAccProtoV6ProviderFactories, + PreCheck: func() { acc.TestAccPreCheck(t) }, + TerraformVersionChecks: []tfversion.TerraformVersionCheck{ + tfversion.RequireAbove(tfversion.Version1_5_0), + }, + CheckDestroy: testAccCheckExternalTableDestroy, Steps: []resource.TestStep{ { - Config: externalTableConfig(accName, bucketURL, roleName, acc.TestDatabaseName, acc.TestSchemaName), + ConfigDirectory: config.TestNameDirectory(), + ConfigVariables: configVariables, Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("snowflake_external_table.test_table", "name", accName), - resource.TestCheckResourceAttr("snowflake_external_table.test_table", "database", accName), - resource.TestCheckResourceAttr("snowflake_external_table.test_table", "schema", accName), - resource.TestCheckResourceAttr("snowflake_external_table.test_table", "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttr(resourceName, "name", name), + resource.TestCheckResourceAttr(resourceName, "database", acc.TestDatabaseName), + resource.TestCheckResourceAttr(resourceName, "schema", acc.TestSchemaName), + resource.TestCheckResourceAttr(resourceName, "location", fmt.Sprintf(`@"%s"."%s"."%s"`, acc.TestDatabaseName, acc.TestSchemaName, name)), + resource.TestCheckResourceAttr(resourceName, "file_format", "TYPE = CSV"), + resource.TestCheckResourceAttr(resourceName, "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttr(resourceName, "column.#", "2"), + resource.TestCheckResourceAttr(resourceName, "column[0].name", "column1"), + resource.TestCheckResourceAttr(resourceName, "column[0].type", "STRING"), + resource.TestCheckResourceAttr(resourceName, "column[0].as", "TO_VARCHAR(TO_TIMESTAMP_NTZ(value:unix_timestamp_property::NUMBER, 3), 'yyyy-mm-dd-hh')"), + resource.TestCheckResourceAttr(resourceName, "column[1].name", "column2"), + resource.TestCheckResourceAttr(resourceName, "column[1].type", "TIMESTAMP_NTZ(9)"), + resource.TestCheckResourceAttr(resourceName, "column[1].as", "($1:\"CreatedDate\"::timestamp)"), ), }, }, }) } -func externalTableConfig(name string, bucketURL string, roleName string, databaseName string, schemaName string) string { - s := ` -resource "snowflake_storage_integration" "i" { - name = "%v" - storage_allowed_locations = ["%s"] - storage_provider = "S3" - storage_aws_role_arn = "%s" -} - -resource "snowflake_stage" "test" { - name = "%v" - url = "%s" - database = "%s" - schema = "%s" - storage_integration = snowflake_storage_integration.i.name -} - -resource "snowflake_external_table" "test_table" { - name = "%s" - database = "%s" - schema = "%s" - comment = "Terraform acceptance test" - column { - name = "column1" - type = "STRING" - as = "TO_VARCHAR(TO_TIMESTAMP_NTZ(value:unix_timestamp_property::NUMBER, 3), 'yyyy-mm-dd-hh')" - } - column { - name = "column2" - type = "TIMESTAMP_NTZ(9)" - as = "($1:\"CreatedDate\"::timestamp)" +func testAccCheckExternalTableDestroy(s *terraform.State) error { + db := acc.TestAccProvider.Meta().(*sql.DB) + client := sdk.NewClientFromDB(db) + for _, rs := range s.RootModule().Resources { + if rs.Type != "snowflake_external_table" { + continue + } + ctx := context.Background() + id := sdk.NewSchemaObjectIdentifier(rs.Primary.Attributes["database"], rs.Primary.Attributes["schema"], rs.Primary.Attributes["name"]) + dynamicTable, err := client.ExternalTables.ShowByID(ctx, sdk.NewShowExternalTableByIDRequest(id)) + if err == nil { + return fmt.Errorf("external table %v still exists", dynamicTable.Name) + } } - file_format = "TYPE = CSV" - location = "@\"%s\".\"%s\".\"${snowflake_stage.test.name}\"" -} -` - return fmt.Sprintf(s, name, bucketURL, roleName, name, bucketURL, databaseName, schemaName, name, databaseName, schemaName, databaseName, schemaName) + return nil } diff --git a/pkg/resources/external_table_test.go b/pkg/resources/external_table_test.go deleted file mode 100644 index 0cad818fc6..0000000000 --- a/pkg/resources/external_table_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package resources_test - -import ( - "database/sql" - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/provider" - "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/resources" - . "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/testhelpers" - "github.com/stretchr/testify/require" -) - -func TestExternalTable(t *testing.T) { - r := require.New(t) - err := resources.ExternalTable().InternalValidate(provider.Provider().Schema, true) - r.NoError(err) -} - -func TestExternalTableCreate(t *testing.T) { - r := require.New(t) - - in := map[string]interface{}{ - "name": "good_name", - "database": "database_name", - "schema": "schema_name", - "comment": "great comment", - "column": []interface{}{map[string]interface{}{"name": "column1", "type": "OBJECT", "as": "a"}, map[string]interface{}{"name": "column2", "type": "VARCHAR", "as": "b"}}, - "location": "location", - "file_format": "FORMAT_NAME = 'format'", - "pattern": "pattern", - } - d := externalTable(t, "database_name|schema_name|good_name", in) - - WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec(`CREATE EXTERNAL TABLE "database_name"."schema_name"."good_name" \("column1" OBJECT AS a, "column2" VARCHAR AS b\) WITH LOCATION = location REFRESH_ON_CREATE = true AUTO_REFRESH = true PATTERN = 'pattern' FILE_FORMAT = \( FORMAT_NAME = 'format' \) COMMENT = 'great comment'`).WillReturnResult(sqlmock.NewResult(1, 1)) - - expectExternalTableRead(mock) - err := resources.CreateExternalTable(d, db) - r.NoError(err) - r.Equal("good_name", d.Get("name").(string)) - }) -} - -func expectExternalTableRead(mock sqlmock.Sqlmock) { - rows := sqlmock.NewRows([]string{"name", "type", "kind", "null?", "default", "primary key", "unique key", "check", "expression", "comment"}).AddRow("good_name", "VARCHAR()", "COLUMN", "Y", "NULL", "NULL", "N", "N", "NULL", "mock comment") - mock.ExpectQuery(`SHOW EXTERNAL TABLES LIKE 'good_name' IN SCHEMA "database_name"."schema_name"`).WillReturnRows(rows) -} - -func TestExternalTableRead(t *testing.T) { - r := require.New(t) - - d := externalTable(t, "database_name|schema_name|good_name", map[string]interface{}{"name": "good_name", "comment": "mock comment"}) - - WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - expectExternalTableRead(mock) - - err := resources.ReadExternalTable(d, db) - r.NoError(err) - r.Equal("good_name", d.Get("name").(string)) - r.Equal("mock comment", d.Get("comment").(string)) - }) -} - -func TestExternalTableDelete(t *testing.T) { - r := require.New(t) - - d := externalTable(t, "database_name|schema_name|drop_it", map[string]interface{}{"name": "drop_it"}) - - WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec(`DROP EXTERNAL TABLE "database_name"."schema_name"."drop_it"`).WillReturnResult(sqlmock.NewResult(1, 1)) - err := resources.DeleteExternalTable(d, db) - r.NoError(err) - }) -} diff --git a/pkg/resources/helpers.go b/pkg/resources/helpers.go index 9691ae7106..f539f8499c 100644 --- a/pkg/resources/helpers.go +++ b/pkg/resources/helpers.go @@ -88,6 +88,32 @@ func getPropertyTags(d *schema.ResourceData, key string) []sdk.TagAssociation { return nil } +func GetTagsDiff(d *schema.ResourceData, key string) (unsetTags []sdk.ObjectIdentifier, setTags []sdk.TagAssociation) { + o, n := d.GetChange(key) + removed, added, changed := getTags(o).diffs(getTags(n)) + + unsetTags = make([]sdk.ObjectIdentifier, len(removed)) + for i, t := range removed { + unsetTags[i] = sdk.NewDatabaseObjectIdentifier(t.database, t.name) + } + + setTags = make([]sdk.TagAssociation, len(added)+len(changed)) + for i, t := range added { + setTags[i] = sdk.TagAssociation{ + Name: sdk.NewSchemaObjectIdentifier(t.database, t.schema, t.name), + Value: t.value, + } + } + for i, t := range changed { + setTags[len(added)+i] = sdk.TagAssociation{ + Name: sdk.NewSchemaObjectIdentifier(t.database, t.schema, t.name), + Value: t.value, + } + } + + return unsetTags, setTags +} + func GetPropertyAsPointer[T any](d *schema.ResourceData, property string) *T { value, ok := d.GetOk(property) if !ok { @@ -99,3 +125,20 @@ func GetPropertyAsPointer[T any](d *schema.ResourceData, property string) *T { } return &typedValue } + +func IsDataType() schema.SchemaValidateFunc { + return func(value any, key string) (warnings []string, errors []error) { + stringValue, ok := value.(string) + if !ok { + errors = append(errors, fmt.Errorf("expected type of %s to be string, got %T", key, value)) + return warnings, errors + } + + _, err := sdk.ToDataType(stringValue) + if err != nil { + errors = append(errors, fmt.Errorf("expected %s to be one of %T values, got %s", key, sdk.DataTypeString, stringValue)) + } + + return warnings, errors + } +} diff --git a/pkg/resources/helpers_test.go b/pkg/resources/helpers_test.go index 78c07827b6..87a8812274 100644 --- a/pkg/resources/helpers_test.go +++ b/pkg/resources/helpers_test.go @@ -470,3 +470,45 @@ func tagGrant(t *testing.T, id string, params map[string]interface{}) *schema.Re d.SetId(id) return d } + +func TestIsDataType(t *testing.T) { + isDataType := resources.IsDataType() + key := "tag" + + testCases := []struct { + Name string + Value any + Error string + }{ + { + Name: "validation: correct DataType value", + Value: "NUMBER", + }, + { + Name: "validation: correct DataType value in lowercase", + Value: "number", + }, + { + Name: "validation: incorrect DataType value", + Value: "invalid data type", + Error: "expected tag to be one of", + }, + { + Name: "validation: incorrect value type", + Value: 123, + Error: "expected type of tag to be string", + }, + } + + for _, tt := range testCases { + t.Run(tt.Name, func(t *testing.T) { + _, errors := isDataType(tt.Value, key) + if tt.Error != "" { + assert.Len(t, errors, 1) + assert.ErrorContains(t, errors[0], tt.Error) + } else { + assert.Len(t, errors, 0) + } + }) + } +} diff --git a/pkg/resources/schema.go b/pkg/resources/schema.go index 83b07d31ee..bdd30aa3ca 100644 --- a/pkg/resources/schema.go +++ b/pkg/resources/schema.go @@ -229,38 +229,20 @@ func UpdateSchema(d *schema.ResourceData, meta interface{}) error { } if d.HasChange("tag") { - o, n := d.GetChange("tag") - removed, added, changed := getTags(o).diffs(getTags(n)) + unsetTags, setTags := GetTagsDiff(d, "tag") - unsetTags := make([]sdk.ObjectIdentifier, len(removed)) - for i, t := range removed { - unsetTags[i] = sdk.NewDatabaseObjectIdentifier(t.database, t.name) - } err := client.Schemas.Alter(ctx, id, &sdk.AlterSchemaOptions{ UnsetTag: unsetTags, }) if err != nil { - return fmt.Errorf("error dropping tags on %v", d.Id()) + return fmt.Errorf("error occurred when dropping tags on %v, err = %w", d.Id(), err) } - setTags := make([]sdk.TagAssociation, len(added)+len(changed)) - for i, t := range added { - setTags[i] = sdk.TagAssociation{ - Name: sdk.NewSchemaObjectIdentifier(t.database, t.schema, t.name), - Value: t.value, - } - } - for i, t := range changed { - setTags[i] = sdk.TagAssociation{ - Name: sdk.NewSchemaObjectIdentifier(t.database, t.schema, t.name), - Value: t.value, - } - } err = client.Schemas.Alter(ctx, id, &sdk.AlterSchemaOptions{ SetTag: setTags, }) if err != nil { - return fmt.Errorf("error setting tags on %v", d.Id()) + return fmt.Errorf("error occurred when setting tags on %v, err = %w", d.Id(), err) } } diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_basic/test.tf b/pkg/resources/testdata/TestAcc_ExternalTable_basic/test.tf new file mode 100644 index 0000000000..aab230558f --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_basic/test.tf @@ -0,0 +1,33 @@ +resource "snowflake_storage_integration" "i" { + name = var.name + storage_allowed_locations = [var.location] + storage_provider = "S3" + storage_aws_role_arn = var.aws_arn +} + +resource "snowflake_stage" "test" { + name = var.name + url = var.location + database = var.database + schema = var.schema + storage_integration = snowflake_storage_integration.i.name +} + +resource "snowflake_external_table" "test_table" { + name = var.name + database = var.database + schema = var.schema + comment = "Terraform acceptance test" + column { + name = "column1" + type = "STRING" + as = "TO_VARCHAR(TO_TIMESTAMP_NTZ(value:unix_timestamp_property::NUMBER, 3), 'yyyy-mm-dd-hh')" + } + column { + name = "column2" + type = "TIMESTAMP_NTZ(9)" + as = "($1:\"CreatedDate\"::timestamp)" + } + file_format = "TYPE = CSV" + location = "@\"${var.database}\".\"${var.schema}\".\"${snowflake_stage.test.name}\"" +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_basic/variables.tf b/pkg/resources/testdata/TestAcc_ExternalTable_basic/variables.tf new file mode 100644 index 0000000000..a447ded368 --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_basic/variables.tf @@ -0,0 +1,19 @@ +variable "name" { + type = string +} + +variable "location" { + type = string +} + +variable "aws_arn" { + type = string +} + +variable "database" { + type = string +} + +variable "schema" { + type = string +} diff --git a/pkg/sdk/external_tables.go b/pkg/sdk/external_tables.go index 97752dea73..fdf5bc5378 100644 --- a/pkg/sdk/external_tables.go +++ b/pkg/sdk/external_tables.go @@ -57,6 +57,10 @@ func (v *ExternalTable) ObjectType() ObjectType { return ObjectTypeExternalTable } +type RawFileFormat struct { + Format string `ddl:"keyword"` +} + // CreateExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/create-external-table type CreateExternalTableOptions struct { create bool `ddl:"static" sql:"CREATE"` @@ -72,11 +76,15 @@ type CreateExternalTableOptions struct { AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` Pattern *string `ddl:"parameter,single_quotes" sql:"PATTERN"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` - AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"` - CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` - Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` - RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"` - Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` + // RawFileFormat was introduced, because of the decision taken during https://github.com/Snowflake-Labs/terraform-provider-snowflake/pull/2228 + // that for now the snowflake_external_table resource should continue on using raw file format, which wasn't previously supported by the new SDK. + // In the future it should most likely be replaced by a more structured version FileFormat + RawFileFormat *RawFileFormat `ddl:"list,parentheses" sql:"FILE_FORMAT ="` + AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"` + CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` + RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"` + Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` } type ExternalTableColumn struct { @@ -205,10 +213,14 @@ type CreateWithManualPartitioningExternalTableOptions struct { Location string `ddl:"parameter" sql:"LOCATION"` UserSpecifiedPartitionType *bool `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` - CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` - Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` - RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"` - Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` + // RawFileFormat was introduced, because of the decision taken during https://github.com/Snowflake-Labs/terraform-provider-snowflake/pull/2228 + // that for now the snowflake_external_table resource should continue on using raw file format, which wasn't previously supported by the new SDK. + // In the future it should most likely be replaced by a more structured version FileFormat + RawFileFormat *RawFileFormat `ddl:"list,parentheses" sql:"FILE_FORMAT ="` + CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` + RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"` + Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` } // CreateDeltaLakeExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/create-external-table @@ -226,11 +238,15 @@ type CreateDeltaLakeExternalTableOptions struct { AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` UserSpecifiedPartitionType *bool `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` - DeltaTableFormat *bool `ddl:"keyword" sql:"TABLE_FORMAT = DELTA"` - CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` - Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` - RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"` - Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` + // RawFileFormat was introduced, because of the decision taken during https://github.com/Snowflake-Labs/terraform-provider-snowflake/pull/2228 + // that for now the snowflake_external_table resource should continue on using raw file format, which wasn't previously supported by the new SDK. + // In the future it should most likely be replaced by a more structured version FileFormat + RawFileFormat *RawFileFormat `ddl:"list,parentheses" sql:"FILE_FORMAT ="` + DeltaTableFormat *bool `ddl:"keyword" sql:"TABLE_FORMAT = DELTA"` + CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` + RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"` + Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` } // CreateExternalTableUsingTemplateOptions based on https://docs.snowflake.com/en/sql-reference/sql/create-external-table#variant-syntax @@ -248,10 +264,14 @@ type CreateExternalTableUsingTemplateOptions struct { AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` Pattern *string `ddl:"parameter,single_quotes" sql:"PATTERN"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` - AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"` - Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` - RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"` - Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` + // RawFileFormat was introduced, because of the decision taken during https://github.com/Snowflake-Labs/terraform-provider-snowflake/pull/2228 + // that for now the snowflake_external_table resource should continue on using raw file format, which wasn't previously supported by the new SDK. + // In the future it should most likely be replaced by a more structured version FileFormat + RawFileFormat *RawFileFormat `ddl:"list,parentheses" sql:"FILE_FORMAT ="` + AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` + RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"` + Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"` } // AlterExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/alter-external-table diff --git a/pkg/sdk/external_tables_dto.go b/pkg/sdk/external_tables_dto.go index f266ce3c69..d27b0850d3 100644 --- a/pkg/sdk/external_tables_dto.go +++ b/pkg/sdk/external_tables_dto.go @@ -26,7 +26,8 @@ type CreateExternalTableRequest struct { refreshOnCreate *bool autoRefresh *bool pattern *string - fileFormat *ExternalTableFileFormatRequest // required + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest awsSnsTopic *string copyGrants *bool comment *string @@ -237,6 +238,13 @@ func (s *CreateExternalTableRequest) toOpts() *CreateExternalTableOptions { fileFormat = []ExternalTableFileFormat{s.fileFormat.toOpts()} } + var rawFileFormat *RawFileFormat + if s.rawFileFormat != nil { + rawFileFormat = &RawFileFormat{ + Format: *s.rawFileFormat, + } + } + var cloudProviderParams *CloudProviderParams if s.cloudProviderParams != nil { cloudProviderParams = s.cloudProviderParams.toOpts() @@ -264,6 +272,7 @@ func (s *CreateExternalTableRequest) toOpts() *CreateExternalTableOptions { RefreshOnCreate: s.refreshOnCreate, AutoRefresh: s.autoRefresh, Pattern: s.pattern, + RawFileFormat: rawFileFormat, FileFormat: fileFormat, AwsSnsTopic: s.awsSnsTopic, CopyGrants: s.copyGrants, @@ -282,7 +291,8 @@ type CreateWithManualPartitioningExternalTableRequest struct { partitionBy []string location string // required userSpecifiedPartitionType *bool - fileFormat *ExternalTableFileFormatRequest // required + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest copyGrants *bool comment *string rowAccessPolicy *RowAccessPolicyRequest @@ -307,6 +317,13 @@ func (v *CreateWithManualPartitioningExternalTableRequest) toOpts() *CreateWithM fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} } + var rawFileFormat *RawFileFormat + if v.rawFileFormat != nil { + rawFileFormat = &RawFileFormat{ + Format: *v.rawFileFormat, + } + } + var rowAccessPolicy *TableRowAccessPolicy if v.rowAccessPolicy != nil { rowAccessPolicy = v.rowAccessPolicy.toOpts() @@ -328,6 +345,7 @@ func (v *CreateWithManualPartitioningExternalTableRequest) toOpts() *CreateWithM PartitionBy: v.partitionBy, Location: v.location, UserSpecifiedPartitionType: v.userSpecifiedPartitionType, + RawFileFormat: rawFileFormat, FileFormat: fileFormat, CopyGrants: v.copyGrants, Comment: v.comment, @@ -347,7 +365,8 @@ type CreateDeltaLakeExternalTableRequest struct { userSpecifiedPartitionType *bool refreshOnCreate *bool autoRefresh *bool - fileFormat *ExternalTableFileFormatRequest // required + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest deltaTableFormat *bool copyGrants *bool comment *string @@ -373,6 +392,13 @@ func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalT fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} } + var rawFileFormat *RawFileFormat + if v.rawFileFormat != nil { + rawFileFormat = &RawFileFormat{ + Format: *v.rawFileFormat, + } + } + var rowAccessPolicy *TableRowAccessPolicy if v.rowAccessPolicy != nil { rowAccessPolicy = v.rowAccessPolicy.toOpts() @@ -396,6 +422,7 @@ func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalT UserSpecifiedPartitionType: v.userSpecifiedPartitionType, RefreshOnCreate: v.refreshOnCreate, AutoRefresh: v.autoRefresh, + RawFileFormat: rawFileFormat, FileFormat: fileFormat, DeltaTableFormat: v.deltaTableFormat, CopyGrants: v.copyGrants, @@ -416,7 +443,8 @@ type CreateExternalTableUsingTemplateRequest struct { refreshOnCreate *bool autoRefresh *bool pattern *string - fileFormat *ExternalTableFileFormatRequest // required + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest awsSnsTopic *string comment *string rowAccessPolicy *RowAccessPolicyRequest @@ -434,6 +462,13 @@ func (v *CreateExternalTableUsingTemplateRequest) toOpts() *CreateExternalTableU fileFormat = []ExternalTableFileFormat{v.fileFormat.toOpts()} } + var rawFileFormat *RawFileFormat + if v.rawFileFormat != nil { + rawFileFormat = &RawFileFormat{ + Format: *v.rawFileFormat, + } + } + var rowAccessPolicy *TableRowAccessPolicy if v.rowAccessPolicy != nil { rowAccessPolicy = v.rowAccessPolicy.toOpts() @@ -457,6 +492,7 @@ func (v *CreateExternalTableUsingTemplateRequest) toOpts() *CreateExternalTableU RefreshOnCreate: v.refreshOnCreate, AutoRefresh: v.autoRefresh, Pattern: v.pattern, + RawFileFormat: rawFileFormat, FileFormat: fileFormat, AwsSnsTopic: v.awsSnsTopic, Comment: v.comment, diff --git a/pkg/sdk/external_tables_dto_builders_gen.go b/pkg/sdk/external_tables_dto_builders_gen.go index 9a08c82078..c4a40af879 100644 --- a/pkg/sdk/external_tables_dto_builders_gen.go +++ b/pkg/sdk/external_tables_dto_builders_gen.go @@ -5,12 +5,10 @@ package sdk func NewCreateExternalTableRequest( name SchemaObjectIdentifier, location string, - fileFormat *ExternalTableFileFormatRequest, ) *CreateExternalTableRequest { s := CreateExternalTableRequest{} s.name = name s.location = location - s.fileFormat = fileFormat return &s } @@ -54,6 +52,16 @@ func (s *CreateExternalTableRequest) WithPattern(pattern *string) *CreateExterna return s } +func (s *CreateExternalTableRequest) WithRawFileFormat(rawFileFormat *string) *CreateExternalTableRequest { + s.rawFileFormat = rawFileFormat + return s +} + +func (s *CreateExternalTableRequest) WithFileFormat(fileFormat *ExternalTableFileFormatRequest) *CreateExternalTableRequest { + s.fileFormat = fileFormat + return s +} + func (s *CreateExternalTableRequest) WithAwsSnsTopic(awsSnsTopic *string) *CreateExternalTableRequest { s.awsSnsTopic = awsSnsTopic return s @@ -280,12 +288,10 @@ func NewRowAccessPolicyRequest( func NewCreateWithManualPartitioningExternalTableRequest( name SchemaObjectIdentifier, location string, - fileFormat *ExternalTableFileFormatRequest, ) *CreateWithManualPartitioningExternalTableRequest { s := CreateWithManualPartitioningExternalTableRequest{} s.name = name s.location = location - s.fileFormat = fileFormat return &s } @@ -319,6 +325,16 @@ func (s *CreateWithManualPartitioningExternalTableRequest) WithUserSpecifiedPart return s } +func (s *CreateWithManualPartitioningExternalTableRequest) WithRawFileFormat(rawFileFormat *string) *CreateWithManualPartitioningExternalTableRequest { + s.rawFileFormat = rawFileFormat + return s +} + +func (s *CreateWithManualPartitioningExternalTableRequest) WithFileFormat(fileFormat *ExternalTableFileFormatRequest) *CreateWithManualPartitioningExternalTableRequest { + s.fileFormat = fileFormat + return s +} + func (s *CreateWithManualPartitioningExternalTableRequest) WithCopyGrants(copyGrants *bool) *CreateWithManualPartitioningExternalTableRequest { s.copyGrants = copyGrants return s @@ -342,12 +358,10 @@ func (s *CreateWithManualPartitioningExternalTableRequest) WithTag(tag []*TagAss func NewCreateDeltaLakeExternalTableRequest( name SchemaObjectIdentifier, location string, - fileFormat *ExternalTableFileFormatRequest, ) *CreateDeltaLakeExternalTableRequest { s := CreateDeltaLakeExternalTableRequest{} s.name = name s.location = location - s.fileFormat = fileFormat return &s } @@ -391,6 +405,16 @@ func (s *CreateDeltaLakeExternalTableRequest) WithAutoRefresh(autoRefresh *bool) return s } +func (s *CreateDeltaLakeExternalTableRequest) WithRawFileFormat(rawFileFormat *string) *CreateDeltaLakeExternalTableRequest { + s.rawFileFormat = rawFileFormat + return s +} + +func (s *CreateDeltaLakeExternalTableRequest) WithFileFormat(fileFormat *ExternalTableFileFormatRequest) *CreateDeltaLakeExternalTableRequest { + s.fileFormat = fileFormat + return s +} + func (s *CreateDeltaLakeExternalTableRequest) WithDeltaTableFormat(deltaTableFormat *bool) *CreateDeltaLakeExternalTableRequest { s.deltaTableFormat = deltaTableFormat return s @@ -419,12 +443,10 @@ func (s *CreateDeltaLakeExternalTableRequest) WithTag(tag []*TagAssociationReque func NewCreateExternalTableUsingTemplateRequest( name SchemaObjectIdentifier, location string, - fileFormat *ExternalTableFileFormatRequest, ) *CreateExternalTableUsingTemplateRequest { s := CreateExternalTableUsingTemplateRequest{} s.name = name s.location = location - s.fileFormat = fileFormat return &s } @@ -468,6 +490,16 @@ func (s *CreateExternalTableUsingTemplateRequest) WithPattern(pattern *string) * return s } +func (s *CreateExternalTableUsingTemplateRequest) WithRawFileFormat(rawFileFormat *string) *CreateExternalTableUsingTemplateRequest { + s.rawFileFormat = rawFileFormat + return s +} + +func (s *CreateExternalTableUsingTemplateRequest) WithFileFormat(fileFormat *ExternalTableFileFormatRequest) *CreateExternalTableUsingTemplateRequest { + s.fileFormat = fileFormat + return s +} + func (s *CreateExternalTableUsingTemplateRequest) WithAwsSnsTopic(awsSnsTopic *string) *CreateExternalTableUsingTemplateRequest { s.awsSnsTopic = awsSnsTopic return s diff --git a/pkg/sdk/external_tables_impl.go b/pkg/sdk/external_tables_impl.go index b799040bf0..1afc0b6619 100644 --- a/pkg/sdk/external_tables_impl.go +++ b/pkg/sdk/external_tables_impl.go @@ -54,7 +54,9 @@ func (v *externalTables) ShowByID(ctx context.Context, req *ShowExternalTableByI return nil, ErrInvalidObjectIdentifier } - externalTables, err := v.client.ExternalTables.Show(ctx, NewShowExternalTableRequest().WithLike(String(req.id.Name()))) + externalTables, err := v.client.ExternalTables.Show(ctx, NewShowExternalTableRequest(). + WithIn(NewShowExternalTableInRequest().WithSchema(NewDatabaseObjectIdentifier(req.id.DatabaseName(), req.id.SchemaName()))). + WithLike(String(req.id.Name()))) if err != nil { return nil, err } diff --git a/pkg/sdk/external_tables_test.go b/pkg/sdk/external_tables_test.go index 3a3fccf57f..57eee6fa61 100644 --- a/pkg/sdk/external_tables_test.go +++ b/pkg/sdk/external_tables_test.go @@ -91,9 +91,50 @@ func TestExternalTablesCreate(t *testing.T) { errOneOf("CreateExternalTableOptions", "OrReplace", "IfNotExists"), ErrInvalidObjectIdentifier, errNotSet("CreateExternalTableOptions", "Location"), - errNotSet("CreateExternalTableOptions", "FileFormat"), + errExactlyOneOf("CreateExternalTableOptions", "RawFileFormat", "FileFormat"), ) }) + + t.Run("raw file format", func(t *testing.T) { + opts := &CreateExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + NotNull: Bool(true), + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + Type: ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + }) + + t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { + opts := &CreateExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + NotNull: Bool(true), + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + Type: ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + } + assertOptsInvalid(t, opts, errExactlyOneOf("CreateExternalTableOptions", "RawFileFormat", "FileFormat")) + }) } func TestExternalTablesCreateWithManualPartitioning(t *testing.T) { @@ -153,9 +194,50 @@ func TestExternalTablesCreateWithManualPartitioning(t *testing.T) { errOneOf("CreateWithManualPartitioningExternalTableOptions", "OrReplace", "IfNotExists"), ErrInvalidObjectIdentifier, errNotSet("CreateWithManualPartitioningExternalTableOptions", "Location"), - errNotSet("CreateWithManualPartitioningExternalTableOptions", "FileFormat"), + errExactlyOneOf("CreateWithManualPartitioningExternalTableOptions", "RawFileFormat", "FileFormat"), ) }) + + t.Run("raw file format", func(t *testing.T) { + opts := &CreateWithManualPartitioningExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + NotNull: Bool(true), + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + Type: ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + }) + + t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { + opts := &CreateWithManualPartitioningExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + NotNull: Bool(true), + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + Type: ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + } + assertOptsInvalid(t, opts, errExactlyOneOf("CreateWithManualPartitioningExternalTableOptions", "RawFileFormat", "FileFormat")) + }) } func TestExternalTablesCreateDeltaLake(t *testing.T) { @@ -213,9 +295,50 @@ func TestExternalTablesCreateDeltaLake(t *testing.T) { errOneOf("CreateDeltaLakeExternalTableOptions", "OrReplace", "IfNotExists"), ErrInvalidObjectIdentifier, errNotSet("CreateDeltaLakeExternalTableOptions", "Location"), - errNotSet("CreateDeltaLakeExternalTableOptions", "FileFormat"), + errExactlyOneOf("CreateDeltaLakeExternalTableOptions", "RawFileFormat", "FileFormat"), ) }) + + t.Run("raw file format", func(t *testing.T) { + opts := &CreateDeltaLakeExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + NotNull: Bool(true), + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + Type: ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + }) + + t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { + opts := &CreateDeltaLakeExternalTableOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Columns: []ExternalTableColumn{ + { + Name: "column", + Type: "varchar", + AsExpression: []string{"value::column::varchar"}, + NotNull: Bool(true), + InlineConstraint: &ColumnInlineConstraint{ + Name: String("my_constraint"), + Type: ColumnConstraintTypeUnique, + }, + }, + }, + Location: "@s1/logs/", + } + assertOptsInvalid(t, opts, errExactlyOneOf("CreateDeltaLakeExternalTableOptions", "RawFileFormat", "FileFormat")) + }) } func TestExternalTableUsingTemplateOpts(t *testing.T) { @@ -263,9 +386,32 @@ func TestExternalTableUsingTemplateOpts(t *testing.T) { ErrInvalidObjectIdentifier, errNotSet("CreateExternalTableUsingTemplateOptions", "Query"), errNotSet("CreateExternalTableUsingTemplateOptions", "Location"), - errNotSet("CreateExternalTableUsingTemplateOptions", "FileFormat"), + errExactlyOneOf("CreateExternalTableUsingTemplateOptions", "RawFileFormat", "FileFormat"), ) }) + + t.Run("raw file format", func(t *testing.T) { + opts := &CreateExternalTableUsingTemplateOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Location: "@s1/logs/", + Query: []string{ + "query statement", + }, + RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, + } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" USING TEMPLATE (query statement) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + }) + + t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { + opts := &CreateExternalTableUsingTemplateOptions{ + name: NewSchemaObjectIdentifier("db", "schema", "external_table"), + Location: "@s1/logs/", + Query: []string{ + "query statement", + }, + } + assertOptsInvalid(t, opts, errExactlyOneOf("CreateExternalTableUsingTemplateOptions", "RawFileFormat", "FileFormat")) + }) } func TestExternalTablesAlter(t *testing.T) { diff --git a/pkg/sdk/external_tables_validations.go b/pkg/sdk/external_tables_validations.go index a34264ce0c..4bdd96c50e 100644 --- a/pkg/sdk/external_tables_validations.go +++ b/pkg/sdk/external_tables_validations.go @@ -32,9 +32,10 @@ func (opts *CreateExternalTableOptions) validate() error { if !valueSet(opts.Location) { errs = append(errs, errNotSet("CreateExternalTableOptions", "Location")) } - if !valueSet(opts.FileFormat) { - errs = append(errs, errNotSet("CreateExternalTableOptions", "FileFormat")) - } else { + if !exactlyOneValueSet(opts.RawFileFormat, opts.FileFormat) { + errs = append(errs, errExactlyOneOf("CreateExternalTableOptions", "RawFileFormat", "FileFormat")) + } + if valueSet(opts.FileFormat) { for i, ff := range opts.FileFormat { if !valueSet(ff.Name) && !valueSet(ff.Type) { errs = append(errs, errNotSet(fmt.Sprintf("CreateExternalTableOptions.FileFormat[%d]", i), "Name or Type")) @@ -61,9 +62,10 @@ func (opts *CreateWithManualPartitioningExternalTableOptions) validate() error { if !valueSet(opts.Location) { errs = append(errs, errNotSet("CreateWithManualPartitioningExternalTableOptions", "Location")) } - if !valueSet(opts.FileFormat) { - errs = append(errs, errNotSet("CreateWithManualPartitioningExternalTableOptions", "FileFormat")) - } else { + if !exactlyOneValueSet(opts.RawFileFormat, opts.FileFormat) { + errs = append(errs, errExactlyOneOf("CreateWithManualPartitioningExternalTableOptions", "RawFileFormat", "FileFormat")) + } + if valueSet(opts.FileFormat) { for i, ff := range opts.FileFormat { if !valueSet(ff.Name) && !valueSet(ff.Type) { errs = append(errs, errNotSet(fmt.Sprintf("CreateWithManualPartitioningExternalTableOptions.FileFormat[%d]", i), "Name or Type")) @@ -90,9 +92,10 @@ func (opts *CreateDeltaLakeExternalTableOptions) validate() error { if !valueSet(opts.Location) { errs = append(errs, errNotSet("CreateDeltaLakeExternalTableOptions", "Location")) } - if !valueSet(opts.FileFormat) { - errs = append(errs, errNotSet("CreateDeltaLakeExternalTableOptions", "FileFormat")) - } else { + if !exactlyOneValueSet(opts.RawFileFormat, opts.FileFormat) { + errs = append(errs, errExactlyOneOf("CreateDeltaLakeExternalTableOptions", "RawFileFormat", "FileFormat")) + } + if valueSet(opts.FileFormat) { for i, ff := range opts.FileFormat { if !valueSet(ff.Name) && !valueSet(ff.Type) { errs = append(errs, errNotSet(fmt.Sprintf("CreateDeltaLakeExternalTableOptions.FileFormat[%d]", i), "Name or Type")) @@ -119,9 +122,10 @@ func (opts *CreateExternalTableUsingTemplateOptions) validate() error { if !valueSet(opts.Location) { errs = append(errs, errNotSet("CreateExternalTableUsingTemplateOptions", "Location")) } - if !valueSet(opts.FileFormat) { - errs = append(errs, errNotSet("CreateExternalTableUsingTemplateOptions", "FileFormat")) - } else { + if !exactlyOneValueSet(opts.RawFileFormat, opts.FileFormat) { + errs = append(errs, errExactlyOneOf("CreateExternalTableUsingTemplateOptions", "RawFileFormat", "FileFormat")) + } + if valueSet(opts.FileFormat) { for i, ff := range opts.FileFormat { if !valueSet(ff.Name) && !valueSet(ff.Type) { errs = append(errs, errNotSet(fmt.Sprintf("CreateExternalTableUsingTemplateOptions.FileFormat[%d]", i), "Name or Type")) diff --git a/pkg/sdk/testint/external_tables_integration_test.go b/pkg/sdk/testint/external_tables_integration_test.go index a3eb9e639b..cbe21326d4 100644 --- a/pkg/sdk/testint/external_tables_integration_test.go +++ b/pkg/sdk/testint/external_tables_integration_test.go @@ -40,16 +40,15 @@ func TestInt_ExternalTables(t *testing.T) { return sdk.NewCreateExternalTableRequest( sdk.NewSchemaObjectIdentifier(testDb(t).Name, testSchema(t).Name, name), stageLocation, - sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON), - ) + ).WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON)) } createExternalTableWithManualPartitioningReq := func(name string) *sdk.CreateWithManualPartitioningExternalTableRequest { return sdk.NewCreateWithManualPartitioningExternalTableRequest( sdk.NewSchemaObjectIdentifier(testDb(t).Name, testSchema(t).Name, name), stageLocation, - sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON), ). + WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON)). WithOrReplace(sdk.Bool(true)). WithColumns(columnsWithPartition). WithUserSpecifiedPartitionType(sdk.Bool(true)). @@ -70,6 +69,20 @@ func TestInt_ExternalTables(t *testing.T) { assert.Equal(t, name, externalTable.Name) }) + t.Run("Create: with raw file format", func(t *testing.T) { + name := random.AlphanumericN(32) + externalTableID := sdk.NewSchemaObjectIdentifier(testDb(t).Name, testSchema(t).Name, name) + err := client.ExternalTables.Create(ctx, minimalCreateExternalTableReq(name). + WithFileFormat(nil). + WithRawFileFormat(sdk.String("TYPE = JSON")), + ) + require.NoError(t, err) + + externalTable, err := client.ExternalTables.ShowByID(ctx, sdk.NewShowExternalTableByIDRequest(externalTableID)) + require.NoError(t, err) + assert.Equal(t, name, externalTable.Name) + }) + t.Run("Create: complete", func(t *testing.T) { name := random.AlphanumericN(32) externalTableID := sdk.NewSchemaObjectIdentifier(testDb(t).Name, testSchema(t).Name, name) @@ -78,8 +91,8 @@ func TestInt_ExternalTables(t *testing.T) { sdk.NewCreateExternalTableRequest( externalTableID, stageLocation, - sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON), ). + WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON)). WithOrReplace(sdk.Bool(true)). WithColumns(columns). WithPartitionBy([]string{"filename"}). @@ -111,8 +124,8 @@ func TestInt_ExternalTables(t *testing.T) { sdk.NewCreateExternalTableUsingTemplateRequest( id, stageLocation, - sdk.NewExternalTableFileFormatRequest().WithName(sdk.String(fileFormat.ID().FullyQualifiedName())), ). + WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithName(sdk.String(fileFormat.ID().FullyQualifiedName()))). WithQuery(query). WithAutoRefresh(sdk.Bool(false))) require.NoError(t, err) @@ -140,8 +153,8 @@ func TestInt_ExternalTables(t *testing.T) { sdk.NewCreateDeltaLakeExternalTableRequest( externalTableID, stageLocation, - sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeParquet), ). + WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeParquet)). WithOrReplace(sdk.Bool(true)). WithColumns(columnsWithPartition). WithPartitionBy([]string{"filename"}). diff --git a/pkg/sdk/testint/streams_gen_integration_test.go b/pkg/sdk/testint/streams_gen_integration_test.go index a6d1aba37f..93a03bd49a 100644 --- a/pkg/sdk/testint/streams_gen_integration_test.go +++ b/pkg/sdk/testint/streams_gen_integration_test.go @@ -60,7 +60,7 @@ func TestInt_Streams(t *testing.T) { _, _ = createStageWithURL(t, client, stageID, nycWeatherDataURL) externalTableId := sdk.NewSchemaObjectIdentifier(db.Name, schema.Name, random.AlphanumericN(32)) - err := client.ExternalTables.Create(ctx, sdk.NewCreateExternalTableRequest(externalTableId, stageLocation, sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON))) + err := client.ExternalTables.Create(ctx, sdk.NewCreateExternalTableRequest(externalTableId, stageLocation).WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON))) require.NoError(t, err) t.Cleanup(func() { err := client.ExternalTables.Drop(ctx, sdk.NewDropExternalTableRequest(externalTableId)) diff --git a/pkg/snowflake/external_table.go b/pkg/snowflake/external_table.go deleted file mode 100644 index 9eceebf25b..0000000000 --- a/pkg/snowflake/external_table.go +++ /dev/null @@ -1,242 +0,0 @@ -package snowflake - -import ( - "database/sql" - "errors" - "fmt" - "log" - "strings" - - "github.com/jmoiron/sqlx" -) - -// externalTableBuilder abstracts the creation of SQL queries for a Snowflake schema. -type ExternalTableBuilder struct { - name string - db string - schema string - columns []map[string]string - partitionBys []string - location string - refreshOnCreate bool - autoRefresh bool - pattern string - fileFormat string - copyGrants bool - awsSNSTopic string - comment string - tags []TagValue -} - -// QualifiedName prepends the db and schema if set and escapes everything nicely. -func (tb *ExternalTableBuilder) QualifiedName() string { - var n strings.Builder - - if tb.db != "" && tb.schema != "" { - n.WriteString(fmt.Sprintf(`"%v"."%v".`, tb.db, tb.schema)) - } - - if tb.db != "" && tb.schema == "" { - n.WriteString(fmt.Sprintf(`"%v"..`, tb.db)) - } - - if tb.db == "" && tb.schema != "" { - n.WriteString(fmt.Sprintf(`"%v".`, tb.schema)) - } - - n.WriteString(fmt.Sprintf(`"%v"`, tb.name)) - - return n.String() -} - -// WithComment adds a comment to the ExternalTableBuilder. -func (tb *ExternalTableBuilder) WithComment(c string) *ExternalTableBuilder { - tb.comment = c - return tb -} - -// WithColumns sets the column definitions on the ExternalTableBuilder. -func (tb *ExternalTableBuilder) WithColumns(c []map[string]string) *ExternalTableBuilder { - tb.columns = c - return tb -} - -func (tb *ExternalTableBuilder) WithPartitionBys(c []string) *ExternalTableBuilder { - tb.partitionBys = c - return tb -} - -func (tb *ExternalTableBuilder) WithLocation(c string) *ExternalTableBuilder { - tb.location = c - return tb -} - -func (tb *ExternalTableBuilder) WithRefreshOnCreate(c bool) *ExternalTableBuilder { - tb.refreshOnCreate = c - return tb -} - -func (tb *ExternalTableBuilder) WithAutoRefresh(c bool) *ExternalTableBuilder { - tb.autoRefresh = c - return tb -} - -func (tb *ExternalTableBuilder) WithPattern(c string) *ExternalTableBuilder { - tb.pattern = c - return tb -} - -func (tb *ExternalTableBuilder) WithFileFormat(c string) *ExternalTableBuilder { - tb.fileFormat = c - return tb -} - -func (tb *ExternalTableBuilder) WithCopyGrants(c bool) *ExternalTableBuilder { - tb.copyGrants = c - return tb -} - -func (tb *ExternalTableBuilder) WithAwsSNSTopic(c string) *ExternalTableBuilder { - tb.awsSNSTopic = c - return tb -} - -// WithTags sets the tags on the ExternalTableBuilder. -func (tb *ExternalTableBuilder) WithTags(tags []TagValue) *ExternalTableBuilder { - tb.tags = tags - return tb -} - -// ExternalexternalTable returns a pointer to a Builder that abstracts the DDL operations for a externalTable. -// -// Supported DDL operations are: -// - CREATE externalTable -// -// [Snowflake Reference](https://docs.snowflake.com/en/sql-reference/sql/create-external-table.html) - -func NewExternalTableBuilder(name, db, schema string) *ExternalTableBuilder { - return &ExternalTableBuilder{ - name: name, - db: db, - schema: schema, - } -} - -// Create returns the SQL statement required to create a externalTable. -func (tb *ExternalTableBuilder) Create() string { - q := strings.Builder{} - q.WriteString(fmt.Sprintf(`CREATE EXTERNAL TABLE %v`, tb.QualifiedName())) - - q.WriteString(` (`) - columnDefinitions := []string{} - for _, columnDefinition := range tb.columns { - columnDefinitions = append(columnDefinitions, fmt.Sprintf(`"%v" %v AS %v`, EscapeString(columnDefinition["name"]), EscapeString(columnDefinition["type"]), columnDefinition["as"])) - } - q.WriteString(strings.Join(columnDefinitions, ", ")) - q.WriteString(`)`) - - if len(tb.partitionBys) > 0 { - q.WriteString(` PARTITION BY ( `) - q.WriteString(EscapeString(strings.Join(tb.partitionBys, ", "))) - q.WriteString(` )`) - } - - q.WriteString(` WITH LOCATION = ` + EscapeString(tb.location)) - q.WriteString(fmt.Sprintf(` REFRESH_ON_CREATE = %t`, tb.refreshOnCreate)) - q.WriteString(fmt.Sprintf(` AUTO_REFRESH = %t`, tb.autoRefresh)) - - if tb.pattern != "" { - q.WriteString(fmt.Sprintf(` PATTERN = '%v'`, EscapeString(tb.pattern))) - } - - q.WriteString(fmt.Sprintf(` FILE_FORMAT = ( %v )`, tb.fileFormat)) - - if tb.awsSNSTopic != "" { - q.WriteString(fmt.Sprintf(` AWS_SNS_TOPIC = '%v'`, EscapeString(tb.awsSNSTopic))) - } - - if tb.copyGrants { - q.WriteString(" COPY GRANTS") - } - - if tb.comment != "" { - q.WriteString(fmt.Sprintf(` COMMENT = '%v'`, EscapeString(tb.comment))) - } - - if len(tb.tags) > 0 { - q.WriteString(fmt.Sprintf(` WITH TAG (%s)`, tb.GetTagValueString())) - } - - return q.String() -} - -// Update returns the SQL statement required to update an externalTable. -func (tb *ExternalTableBuilder) Update() string { - q := strings.Builder{} - q.WriteString(fmt.Sprintf(`ALTER EXTERNAL TABLE %v`, tb.QualifiedName())) - - if len(tb.tags) > 0 { - q.WriteString(fmt.Sprintf(` TAG %s`, tb.GetTagValueString())) - } - - return q.String() -} - -// Drop returns the SQL query that will drop a externalTable. -func (tb *ExternalTableBuilder) Drop() string { - return fmt.Sprintf(`DROP EXTERNAL TABLE %v`, tb.QualifiedName()) -} - -// Show returns the SQL query that will show a externalTable. -func (tb *ExternalTableBuilder) Show() string { - return fmt.Sprintf(`SHOW EXTERNAL TABLES LIKE '%v' IN SCHEMA "%v"."%v"`, tb.name, tb.db, tb.schema) -} - -func (tb *ExternalTableBuilder) GetTagValueString() string { - var q strings.Builder - for _, v := range tb.tags { - fmt.Println(v) - if v.Schema != "" { - if v.Database != "" { - q.WriteString(fmt.Sprintf(`"%v".`, v.Database)) - } - q.WriteString(fmt.Sprintf(`"%v".`, v.Schema)) - } - q.WriteString(fmt.Sprintf(`"%v" = "%v", `, v.Name, v.Value)) - } - return strings.TrimSuffix(q.String(), ", ") -} - -type ExternalTable struct { - CreatedOn sql.NullString `db:"created_on"` - ExternalTableName sql.NullString `db:"name"` - DatabaseName sql.NullString `db:"database_name"` - SchemaName sql.NullString `db:"schema_name"` - Comment sql.NullString `db:"comment"` - Owner sql.NullString `db:"owner"` -} - -func ScanExternalTable(row *sqlx.Row) (*ExternalTable, error) { - t := &ExternalTable{} - e := row.StructScan(t) - return t, e -} - -func ListExternalTables(databaseName string, schemaName string, db *sql.DB) ([]ExternalTable, error) { - stmt := fmt.Sprintf(`SHOW EXTERNAL TABLES IN SCHEMA "%s"."%v"`, databaseName, schemaName) - rows, err := Query(db, stmt) - if err != nil { - return nil, err - } - defer rows.Close() - - dbs := []ExternalTable{} - if err := sqlx.StructScan(rows, &dbs); err != nil { - if errors.Is(err, sql.ErrNoRows) { - log.Println("[DEBUG] no external tables found") - return nil, nil - } - return nil, fmt.Errorf("unable to scan row for %s err = %w", stmt, err) - } - return dbs, nil -} diff --git a/pkg/snowflake/external_table_test.go b/pkg/snowflake/external_table_test.go deleted file mode 100644 index f55ef97e30..0000000000 --- a/pkg/snowflake/external_table_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package snowflake - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestExternalTableCreate(t *testing.T) { - r := require.New(t) - s := NewExternalTableBuilder("test_table", "test_db", "test_schema") - s.WithColumns([]map[string]string{{"name": "column1", "type": "OBJECT", "as": "expression1"}, {"name": "column2", "type": "VARCHAR", "as": "expression2"}}) - s.WithLocation("location") - s.WithPattern("pattern") - s.WithFileFormat("TYPE = CSV FIELD_DELIMITER = '|'") - r.Equal(`"test_db"."test_schema"."test_table"`, s.QualifiedName()) - - r.Equal(`CREATE EXTERNAL TABLE "test_db"."test_schema"."test_table" ("column1" OBJECT AS expression1, "column2" VARCHAR AS expression2) WITH LOCATION = location REFRESH_ON_CREATE = false AUTO_REFRESH = false PATTERN = 'pattern' FILE_FORMAT = ( TYPE = CSV FIELD_DELIMITER = '|' )`, s.Create()) - - s.WithComment("Test Comment") - r.Equal(`CREATE EXTERNAL TABLE "test_db"."test_schema"."test_table" ("column1" OBJECT AS expression1, "column2" VARCHAR AS expression2) WITH LOCATION = location REFRESH_ON_CREATE = false AUTO_REFRESH = false PATTERN = 'pattern' FILE_FORMAT = ( TYPE = CSV FIELD_DELIMITER = '|' ) COMMENT = 'Test Comment'`, s.Create()) -} - -func TestExternalTableUpdate(t *testing.T) { - r := require.New(t) - s := NewExternalTableBuilder("test_table", "test_db", "test_schema") - s.WithTags([]TagValue{{Name: "tag1", Value: "value1", Schema: "test_schema", Database: "test_db"}}) - expected := `ALTER EXTERNAL TABLE "test_db"."test_schema"."test_table" TAG "test_db"."test_schema"."tag1" = "value1"` - r.Equal(expected, s.Update()) -} - -func TestExternalTableDrop(t *testing.T) { - r := require.New(t) - s := NewExternalTableBuilder("test_table", "test_db", "test_schema") - r.Equal(`DROP EXTERNAL TABLE "test_db"."test_schema"."test_table"`, s.Drop()) -} - -func TestExternalTableShow(t *testing.T) { - r := require.New(t) - s := NewExternalTableBuilder("test_table", "test_db", "test_schema") - r.Equal(`SHOW EXTERNAL TABLES LIKE 'test_table' IN SCHEMA "test_db"."test_schema"`, s.Show()) -}