From cdbc205d4f25adc078abcac7fa5868e7cb343d74 Mon Sep 17 00:00:00 2001 From: chand1012 Date: Mon, 26 Feb 2024 09:53:55 -0500 Subject: [PATCH] Allow for nested JSON parsing and fix bool bug --- cmd/root.go | 22 +++++++++++----------- go.mod | 2 +- go.sum | 4 ++-- pkg/db/from_json.go | 11 ++++++++++- pkg/db/to_json.go | 21 +++++++++++++++++++++ pkg/db/utils.go | 13 +++---------- 6 files changed, 48 insertions(+), 25 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 59f491e..086a2fd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -122,6 +122,17 @@ func run(cmd *cobra.Command, args []string) { default: logger.HandlePanic(log, errors.New("unsupported file type"), verbose) } + + if outputFormat == "" { + outputFormat = fType.String() + } else { + switch outputFormat { + case "json", "jsonl", "csv", "sqlite": + // do nothing + default: + logger.HandlePanic(log, errors.New("unsupported output format"), verbose) + } + } } if err != nil { @@ -194,15 +205,4 @@ func prerun(cmd *cobra.Command, args []string) { if verbose { log = logger.VerboseLogger } - log.Debug("Resolving output format") - if outputFilePath != "" && outputFormat == "" { - outputFormat = file_types.ResolveByPath(outputFilePath).String() - } else if outputFormat == "" { - outputFormat = "csv" - } - // if outputFormat is not one of json, jsonl, or csv, panic with an error - if outputFormat != "json" && outputFormat != "jsonl" && outputFormat != "csv" && outputFormat != "sqlite" { - logger.HandlePanic(log, errors.New("unsupported output format"), verbose) - } - log.Debugf("Output format: %s", outputFormat) } diff --git a/go.mod b/go.mod index 9b632fe..539604c 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.17.0 // indirect modernc.org/libc v1.37.6 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.7.2 // indirect diff --git a/go.sum b/go.sum index 53f5311..3572672 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/db/from_json.go b/pkg/db/from_json.go index cb80bc3..0e20ea2 100644 --- a/pkg/db/from_json.go +++ b/pkg/db/from_json.go @@ -81,8 +81,17 @@ func FromJSON(b []byte, tableName string) (*sql.DB, string, error) { } values = append(values, string(jsonValue)) continue // Continue to the next column after appending + } else if valType.Kind() == reflect.Bool { + // cast to bool + boolVal := value.(bool) + if boolVal { + values = append(values, "true") + } else { + values = append(values, "false") + } + } else { + values = append(values, value) } - values = append(values, value) } _, err = stmt.Exec(values...) if err != nil { diff --git a/pkg/db/to_json.go b/pkg/db/to_json.go index 82c24cf..b5173d3 100644 --- a/pkg/db/to_json.go +++ b/pkg/db/to_json.go @@ -4,6 +4,7 @@ import ( "database/sql" "encoding/json" "fmt" + "strings" ) func ToJSON(db *sql.DB, tableName string) (string, error) { @@ -83,6 +84,26 @@ func RowsToJSON(rows *sql.Rows) (string, error) { for i, col := range cols { if scanValues[i] != nil { rowMap[col] = scanValues[i] + // try to cast it to a string + if s, ok := scanValues[i].(string); ok { + rowMap[col] = s + // if the string starts and ends with either [] or {} + // then we need to attempt to unmarshal it + if (strings.HasPrefix(rowMap[col].(string), "[") && strings.HasSuffix(rowMap[col].(string), "]")) || + (strings.HasPrefix(rowMap[col].(string), "{") && strings.HasSuffix(rowMap[col].(string), "}")) { + var v interface{} + err := json.Unmarshal([]byte(rowMap[col].(string)), &v) + if err == nil { + rowMap[col] = v + } + } + if rowMap[col] == "true" || rowMap[col] == "false" { + rowMap[col] = rowMap[col].(string) == "true" + } + if b, ok := rowMap[col].(bool); ok { + rowMap[col] = b + } + } } } diff --git a/pkg/db/utils.go b/pkg/db/utils.go index 596c530..a82a6ff 100644 --- a/pkg/db/utils.go +++ b/pkg/db/utils.go @@ -8,26 +8,19 @@ import ( ) func guessType(s string) string { - // First, try to parse it as an integer - if _, err := strconv.ParseInt(s, 10, 64); err == nil { - return "INTEGER" - } - - // If it's not an integer, try to parse it as a float + // assume all numbers are floats if _, err := strconv.ParseFloat(s, 64); err == nil { return "REAL" } - // If it's neither, then return "Text" + // Otherwise, it's a string return "TEXT" } // same as above, just takes generics and uses reflection func reflectType(v any) string { switch v.(type) { - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: - return "INTEGER" - case float32, float64: + case float32, float64, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: return "REAL" default: return "TEXT"