Skip to content

Commit

Permalink
internal/yaml,internal/spanner: Support ARRAY types
Browse files Browse the repository at this point in the history 
issue: kauche#13

support types

- ARRAY<STRING(MAX)>
- ARRAY<Bool>
- ARRAY<INT64>
- ARRAY<FLOAT64>
- ARRAY<TIMESTAMP>
- ARRAY<DATE>
- ARRAY<BYTES(MAX)>
- ARRAY<NUMERIC>
- ARRAY<JSON>
  • Loading branch information
@uji
uji committed Jan 14, 2024
1 parent 57b7edc commit b8c07b6
Show file tree
Hide file tree
Showing 5 changed files with 146 additions and 24 deletions.
65 changes: 49 additions & 16 deletions internal/spanner/spanner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,25 @@ type baz struct {
}

type allTypes struct {
ID string `spanner:"ID"`
StringValue string `spanner:"StringValue"`
BoolValue bool `spanner:"BoolValue"`
Int64Value int64 `spanner:"Int64Value"`
Float64Value float64 `spanner:"Float64Value"`
JSONValue spanner.NullJSON `spanner:"JSONValue"`
BytesValue []byte `spanner:"BytesValue"`
TimestampValue time.Time `spanner:"TimestampValue"`
NumericValue *big.Rat `spanner:"NumericValue"`
DateValue civil.Date `spanner:"DateValue"`
StringArray []string `spanner:"StringArray"`
ID string `spanner:"ID"`
StringValue string `spanner:"StringValue"`
BoolValue bool `spanner:"BoolValue"`
Int64Value int64 `spanner:"Int64Value"`
Float64Value float64 `spanner:"Float64Value"`
JSONValue spanner.NullJSON `spanner:"JSONValue"`
BytesValue []byte `spanner:"BytesValue"`
TimestampValue time.Time `spanner:"TimestampValue"`
NumericValue *big.Rat `spanner:"NumericValue"`
DateValue civil.Date `spanner:"DateValue"`
StringArray []string `spanner:"StringArray"`
BoolArray []bool `spanner:"BoolArray"`
Int64Array []int64 `spanner:"Int64Array"`
Float64Array []float64 `spanner:"Float64Array"`
JSONArray []spanner.NullJSON `spanner:"JSONArray"`
BytesArray [][]byte `spanner:"BytesArray"`
TimestampArray []time.Time `spanner:"TimestampArray"`
NumericArray []*big.Rat `spanner:"NumericArray"`
DateArray []civil.Date `spanner:"DateArray"`
}

func TestSave(t *testing.T) {
Expand Down Expand Up @@ -143,10 +151,17 @@ func TestSave(t *testing.T) {
"NumericValue": "-12345678901234567890123456789.123456789",
"StringValue": "FooBar",
"TimestampValue": "2022-04-01T00:00:00Z",
// Commented out because it is not yet supported.
// "StringArray": []interface{}{"Foo", "Bar"},
"BoolValue": true,
"BytesValue": "aG9nZQ==",
"BoolValue": true,
"BytesValue": "aG9nZQ==",
"StringArray": []string{"Foo", "Bar"},
"BoolArray": []bool{true, false},
"Int64Array": []int64{12, 34},
"Float64Array": []float64{12.34, 56.789},
"JSONArray": []string{`{"test": 1}`, `{"test": 2}`},
"BytesArray": []string{"aG9nZQ==", "aG9nZQ=="},
"TimestampArray": []string{"2022-04-01T00:00:00Z", "2022-04-02T00:00:00Z"},
"NumericArray": []string{"-12345678901234567890123456789.123456789", "-12345678901234567890123456789.123456789"},
"DateArray": []string{"2022-04-01", "2022-04-02"},
},
},
},
Expand Down Expand Up @@ -284,7 +299,25 @@ func TestSave(t *testing.T) {
BytesValue: []byte{'h', 'o', 'g', 'e'},
TimestampValue: time.Date(2022, time.April, 1, 0, 0, 0, 0, time.UTC),
NumericValue: expectedNumeric,
DateValue: civil.Date{2022, time.April, 1},
DateValue: civil.Date{Year: 2022, Month: time.April, Day: 1},
StringArray: []string{"Foo", "Bar"},
BoolArray: []bool{true, false},
Float64Array: []float64{12.34, 56.789},
Int64Array: []int64{12, 34},
JSONArray: []spanner.NullJSON{
{Value: map[string]interface{}{"test": float64(1)}, Valid: true},
{Value: map[string]interface{}{"test": float64(2)}, Valid: true},
},
BytesArray: [][]byte{{'h', 'o', 'g', 'e'}, {'h', 'o', 'g', 'e'}},
TimestampArray: []time.Time{
time.Date(2022, time.April, 1, 0, 0, 0, 0, time.UTC),
time.Date(2022, time.April, 2, 0, 0, 0, 0, time.UTC),
},
NumericArray: []*big.Rat{expectedNumeric, expectedNumeric},
DateArray: []civil.Date{
{Year: 2022, Month: time.April, Day: 1},
{Year: 2022, Month: time.April, Day: 2},
},
},
}
if diff := cmp.Diff(actualAllTypes, expectedAllTypes, cmp.Comparer(func(x, y *big.Rat) bool {
Expand Down
8 changes: 8 additions & 0 deletions internal/spanner/testdata/schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,12 @@ CREATE TABLE AllTypes (
NumericValue NUMERIC,
JSONValue JSON,
StringArray ARRAY<STRING(MAX)>,
BoolArray ARRAY<Bool>,
Int64Array ARRAY<INT64>,
Float64Array ARRAY<FLOAT64>,
TimestampArray ARRAY<TIMESTAMP>,
DateArray ARRAY<DATE>,
BytesArray ARRAY<BYTES(MAX)>,
NumericArray ARRAY<NUMERIC>,
JSONArray ARRAY<JSON>,
) PRIMARY KEY(ID);
30 changes: 27 additions & 3 deletions internal/yaml/testdata/seeds/AllTypes.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,30 @@
NumericValue: "-12345678901234567890123456789.123456789"
StringValue: "FooBar"
TimestampValue: "2022-04-01T00:00:00Z"
# StringArray:
# - "Foo"
# - "Bar"
BoolArray:
- true
- false
BytesArray:
- "aG9nZQ==" # base64("hoge")
- "aG9nZQ==" # base64("hoge")
DateArray:
- "2022-04-01"
- "2022-04-02"
Fload64Array:
- 12.34
- 56.789
Int64Array:
- 12
- 34
JSONArray:
- '{"test": 1}'
- '{"test": 2}'
NumericArray:
- "-12345678901234567890123456789.123456789"
- "-12345678901234567890123456789.123456789"
StringArray:
- "Foo"
- "Bar"
TimestampArray:
- "2022-04-01T00:00:00Z"
- "2022-04-02T00:00:00Z"
53 changes: 51 additions & 2 deletions internal/yaml/yaml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,18 @@ func NewLoader() *Loader {
return &Loader{}
}

func assertTypedSlice[T any](slice []any) ([]T, error) {
tSlice := make([]T, 0, len(slice))
for _, v := range slice {
tVal, ok := v.(T)
if !ok {
return nil, fmt.Errorf("unsupported mixed types list: %v", slice)
}
tSlice = append(tSlice, tVal)
}
return tSlice, nil
}

func (l *Loader) Load(ctx context.Context, dir string) ([]*model.Table, error) {
var tables []*model.Table
err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error {
Expand Down Expand Up @@ -62,9 +74,46 @@ func (l *Loader) Load(ctx context.Context, dir string) ([]*model.Table, error) {
val, ok := p.Value.(uint64)
if ok {
records[i].Values[key] = int64(val)
} else {
records[i].Values[key] = p.Value
continue
}

// Spanner does not support the type []any
list, ok := p.Value.([]any)
if ok {
switch list[0].(type) {
case bool:
records[i].Values[key], err = assertTypedSlice[bool](list)
if err != nil {
return err
}
case string:
records[i].Values[key], err = assertTypedSlice[string](list)
if err != nil {
return err
}
case float64:
records[i].Values[key], err = assertTypedSlice[float64](list)
if err != nil {
return err
}
case uint64:
// Spanner does not support the type uint64 so assert to int64
int64List := make([]int64, 0, len(list))
for _, v := range list {
n, ok := v.(uint64)
if !ok {
return fmt.Errorf("unsupported mixed types list: %v", v)
}
int64List = append(int64List, int64(n))
}
records[i].Values[key] = int64List
default:
return fmt.Errorf("unsupported type in list: %v", list)
}
continue
}

records[i].Values[key] = p.Value
}

}
Expand Down
14 changes: 11 additions & 3 deletions internal/yaml/yaml_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ func TestLoad(t *testing.T) {
Records: []*model.Record{
{
Values: map[string]interface{}{
"BoolValue": true,
"BytesValue": "aG9nZQ==",
"DateValue": "2022-04-01",
"Float64Value": float64(3.14159),
"ID": "All_Type_Values",
Expand All @@ -34,9 +36,15 @@ func TestLoad(t *testing.T) {
"NumericValue": string("-12345678901234567890123456789.123456789"),
"StringValue": "FooBar",
"TimestampValue": "2022-04-01T00:00:00Z",
// "StringArray": []interface{}{"Foo", "Bar"},
"BoolValue": true,
"BytesValue": "aG9nZQ==",
"BoolArray": []bool{true, false},
"BytesArray": []string{"aG9nZQ==", "aG9nZQ=="},
"DateArray": []string{"2022-04-01", "2022-04-02"},
"Fload64Array": []float64{12.34, 56.789},
"Int64Array": []int64{12, 34},
"JSONArray": []string{`{"test": 1}`, `{"test": 2}`},
"NumericArray": []string{"-12345678901234567890123456789.123456789", "-12345678901234567890123456789.123456789"},
"StringArray": []string{"Foo", "Bar"},
"TimestampArray": []string{"2022-04-01T00:00:00Z", "2022-04-02T00:00:00Z"},
},
},
},
Expand Down

0 comments on commit b8c07b6

Please sign in to comment.