From d155b998bc8b2f50347efe66be54e82dce7cce4e Mon Sep 17 00:00:00 2001
From: Fritz Larco <flarco@live.com>
Date: Tue, 23 Jan 2024 11:46:13 -0300
Subject: [PATCH] add SetLengthPrecisionScale [nt]

---
 database/database.go                          |  3 ++
 database/database_bigquery.go                 |  2 +
 database/schemata.go                          |  1 +
 .../templates/types_native_to_general.tsv     |  2 +-
 go.mod                                        |  2 +-
 iop/datatype.go                               | 37 +++++++++++++++++++
 6 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/database/database.go b/database/database.go
index 3a46ec6..f959f98 100755
--- a/database/database.go
+++ b/database/database.go
@@ -1375,6 +1375,9 @@ func SQLColumns(colTypes []ColumnType, conn Connection) (columns iop.Columns) {
 			}
 		}
 
+		// parse length, precision, scale manually specified in mapping
+		col.SetLengthPrecisionScale()
+
 		// g.Trace("col %s (%s -> %s) has %d length, %d scale, sourced: %t", colType.Name(), colType.DatabaseTypeName(), Type, length, scale, ok)
 
 		columns[i] = col
diff --git a/database/database_bigquery.go b/database/database_bigquery.go
index b54964a..2724ada 100755
--- a/database/database_bigquery.go
+++ b/database/database_bigquery.go
@@ -324,6 +324,8 @@ func (conn *BigQueryConn) getItColumns(itSchema bigquery.Schema) (cols iop.Colum
 			Type:     NativeTypeToGeneral(field.Name, string(field.Type), conn),
 			DbType:   string(field.Type),
 		}
+		cols[i].SetLengthPrecisionScale()
+
 		if g.In(field.Type, bigquery.NumericFieldType, bigquery.FloatFieldType) {
 			bQTC.numericCols = append(bQTC.numericCols, i)
 		} else if field.Type == "DATETIME" || field.Type == bigquery.TimestampFieldType {
diff --git a/database/schemata.go b/database/schemata.go
index f4031d8..2d81dd7 100644
--- a/database/schemata.go
+++ b/database/schemata.go
@@ -250,6 +250,7 @@ func (s *Schemata) Columns() map[string]iop.Column {
 				for _, column := range table.Columns {
 					// get general type
 					column.Type = NativeTypeToGeneral(column.Name, column.DbType, s.conn)
+					column.SetLengthPrecisionScale()
 					key := strings.ToLower(g.F("%s.%s.%s.%s", db.Name, schema.Name, table.Name, column.Name))
 					columns[key] = column
 				}
diff --git a/database/templates/types_native_to_general.tsv b/database/templates/types_native_to_general.tsv
index 9b68242..8e17f6c 100755
--- a/database/templates/types_native_to_general.tsv
+++ b/database/templates/types_native_to_general.tsv
@@ -336,7 +336,7 @@ clickhouse	string	text	col_string string	TRUE	TRUE
 clickhouse	tuple	string	col_tuple tuple	TRUE	TRUE	
 clickhouse	uint16	integer	col_uint16 uint16	TRUE	TRUE	
 clickhouse	uint32	bigint	col_uint32 uint32	TRUE	TRUE	
-clickhouse	uint64	decimal	col_uint64 uint64	TRUE	TRUE	
+clickhouse	uint64	decimal(38,9)	col_uint64 uint64	TRUE	TRUE	
 clickhouse	uint8	integer	col_uint8 uint8	TRUE	TRUE	
 clickhouse	json	json	col_json json	FALSE	FALSE	
 clickhouse	map	json	col_map json	FALSE	FALSE	
diff --git a/go.mod b/go.mod
index 30c1d31..f760c4d 100644
--- a/go.mod
+++ b/go.mod
@@ -18,7 +18,7 @@ require (
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/dustin/go-humanize v1.0.1
 	github.com/flarco/bigquery v0.0.9
-	github.com/flarco/g v0.1.67
+	github.com/flarco/g v0.1.68
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/godror/godror v0.31.0
 	github.com/google/uuid v1.5.0
diff --git a/iop/datatype.go b/iop/datatype.go
index c4ff191..23c2055 100755
--- a/iop/datatype.go
+++ b/iop/datatype.go
@@ -600,6 +600,43 @@ func MakeRowsChan() chan []any {
 	return make(chan []any)
 }
 
+const regexExtractPrecisionScale = `[a-zA-Z]+ *\( *(\d+) *(, *\d+)* *\)`
+
+// SetLengthPrecisionScale parse length, precision, scale
+func (col *Column) SetLengthPrecisionScale() {
+	colType := strings.TrimSpace(string(col.Type))
+	if !strings.HasSuffix(colType, ")") {
+		return
+	}
+
+	// fix type value
+	parts := strings.Split(colType, "(")
+	col.Type = ColumnType(strings.TrimSpace(parts[0]))
+
+	matches := g.Matches(colType, regexExtractPrecisionScale)
+	if len(matches) == 1 {
+		vals := matches[0].Group
+
+		if len(vals) > 0 {
+			vals[0] = strings.TrimSpace(vals[0])
+			// grab length or precision
+			if col.Type.IsString() {
+				col.Stats.MaxLen = cast.ToInt(vals[0])
+			} else if col.Type.IsNumber() {
+				col.DbPrecision = cast.ToInt(vals[0])
+			}
+		}
+
+		if len(vals) > 1 {
+			vals[1] = strings.TrimSpace(strings.ReplaceAll(vals[1], ",", ""))
+			// grab scale
+			if col.Type.IsNumber() {
+				col.DbScale = cast.ToInt(vals[1])
+			}
+		}
+	}
+}
+
 func (col *Column) Key() string {
 	parts := []string{}
 	if col.Database != "" {