fix(staging model template)!: Use source() not ref()

I goodfed. This also adds a progress bar instead of a spinner for more visibility into progress when modeling large schemas.

forms.go

@@ -45,7 +45,8 @@ Generates:
 For each table in the designated schema/dataset.
 
 To prepare, make sure you have the following:
-✴︎ An existing dbt profiles.yml file to reference
+✴︎ The name of an existing dbt profile to reference
+(Can be found in the profiles.yml file)
 *_OR_*
 ✴︎ The necessary connection details for your warehouse
 

go.mod

@@ -73,6 +73,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mtibben/percent v0.2.1 // indirect
 	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
@@ -83,6 +84,7 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/schollz/progressbar/v3 v3.14.2 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opencensus.io v0.24.0 // indirect

go.sum

@@ -180,6 +180,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
 github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
 github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
@@ -206,6 +207,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
 github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
@@ -232,6 +235,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks=
+github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/snowflakedb/gosnowflake v1.9.0 h1:s2ZdwFxFfpqwa5CqlhnzRESnLmwU3fED6zyNOJHFBQA=
@@ -240,6 +245,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@@ -313,10 +319,12 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
 golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

main.go

@@ -6,7 +6,6 @@ import (
 	"log"
 	"time"
 
-	"github.com/charmbracelet/huh/spinner"
 	"github.com/gwenwindflower/tbd/sourcerer"
 )
 
@@ -28,59 +27,52 @@ func main() {
 	cd := SetConnectionDetails(formResponse)
 
 	e := Elapsed{}
-	s := spinner.New()
-	err := s.Action(func() {
-		e.DbStart = time.Now()
+	e.DbStart = time.Now()
 
-		bd := formResponse.BuildDir
-
-		dbc, err := sourcerer.GetConn(cd)
-		if err != nil {
-			log.Fatalf("Error getting database connection: %v\n", err)
-		}
-		err = dbc.ConnectToDb(ctx)
-		if err != nil {
-			log.Fatalf("Error connecting to database: %v\n", err)
-		}
-		ts, err := dbc.GetSourceTables(ctx)
-		if err != nil {
-			log.Fatalf("Error getting sources: %v\n", err)
-		}
-		err = sourcerer.PutColumnsOnTables(ctx, ts, dbc)
-		if err != nil {
-			log.Fatalf("Error putting columns on tables: %v\n", err)
-		}
+	bd := formResponse.BuildDir
+	err := PrepBuildDir(bd)
+	if err != nil {
+		log.Fatalf("Error preparing build directory: %v\n", err)
+	}
+	dbc, err := sourcerer.GetConn(cd)
+	if err != nil {
+		log.Fatalf("Error getting database connection: %v\n", err)
+	}
+	err = dbc.ConnectToDb(ctx)
+	if err != nil {
+		log.Fatalf("Error connecting to database: %v\n", err)
+	}
+	ts, err := dbc.GetSourceTables(ctx)
+	if err != nil {
+		log.Fatalf("Error getting sources: %v\n", err)
+	}
+	err = sourcerer.PutColumnsOnTables(ctx, ts, dbc)
+	if err != nil {
+		log.Fatalf("Error putting columns on tables: %v\n", err)
+	}
 
-		e.DbElapsed = time.Since(e.DbStart).Seconds()
-		// End of database interaction, start of processing
-		e.ProcessingStart = time.Now()
+	e.DbElapsed = time.Since(e.DbStart).Seconds()
+	// End of database interaction, start of processing
+	e.ProcessingStart = time.Now()
 
-		if formResponse.GenerateDescriptions {
-			GenerateColumnDescriptions(ts)
-		}
-		err = PrepBuildDir(bd)
-		if err != nil {
-			log.Fatalf("Error preparing build directory: %v\n", err)
-		}
-		if formResponse.CreateProfile {
-			WriteProfile(cd, bd)
-		}
-		if formResponse.ScaffoldProject {
-			s, err := WriteScaffoldProject(cd, bd, formResponse.ProjectName)
-			if err != nil {
-				log.Fatalf("Error scaffolding project: %v\n", err)
-			}
-			bd = s
-		}
-		err = WriteFiles(ts, bd, formResponse.Prefix)
+	if formResponse.GenerateDescriptions {
+		GenerateColumnDescriptions(ts)
+	}
+	if formResponse.CreateProfile {
+		WriteProfile(cd, bd)
+	}
+	if formResponse.ScaffoldProject {
+		s, err := WriteScaffoldProject(cd, bd, formResponse.ProjectName)
 		if err != nil {
-			log.Fatalf("Error writing files: %v\n", err)
+			log.Fatalf("Error scaffolding project: %v\n", err)
 		}
-	}).Title("🏎️✨ Generating YAML and SQL files...").Run()
+		bd = s
+	}
+	err = WriteFiles(ts, bd, formResponse.Prefix)
 	if err != nil {
-		log.Fatalf("Error running spinner action: %v\n", err)
+		log.Fatalf("Error writing files: %v\n", err)
 	}
 	e.ProcessingElapsed = time.Since(e.ProcessingStart).Seconds()
-	fmt.Printf("🏁 Done in %.1fs fetching data and %.1fs writing files! ", e.DbElapsed, e.ProcessingElapsed)
-	fmt.Println("Your YAML and SQL files are in the build directory.")
+	fmt.Printf("\n🏁 Done in %.1fs fetching data and %.1fs writing files! ", e.DbElapsed, e.ProcessingElapsed)
+	fmt.Printf("\nYour YAML and SQL files are in the %s directory.", formResponse.BuildDir)
 }

shared/types.go

@@ -11,6 +11,7 @@ type SourceTable struct {
 	DataTypeGroups map[string][]Column `yaml:"-"`
 	Name           string              `yaml:"name"`
 	Columns        []Column            `yaml:"columns"`
+	Schema         string              `yaml:"-"`
 }
 
 type SourceTables struct {

sourcerer/get_sources_tables.go

@@ -23,6 +23,7 @@ func (sfc *SfConn) GetSourceTables(ctx context.Context) (shared.SourceTables, er
 		if err := rows.Scan(&table.Name); err != nil {
 			log.Fatalf("Error scanning tables: %v\n", err)
 		}
+		table.Schema = sfc.Schema
 		ts.SourceTables = append(ts.SourceTables, table)
 	}
 	return ts, nil
@@ -41,7 +42,7 @@ func (bqc *BqConn) GetSourceTables(ctx context.Context) (shared.SourceTables, er
 		if err != nil {
 			log.Fatalf("Error fetching tables: %v\n", err)
 		}
-		ts.SourceTables = append(ts.SourceTables, shared.SourceTable{Name: table.TableID})
+		ts.SourceTables = append(ts.SourceTables, shared.SourceTable{Name: table.TableID, Schema: bqc.Dataset})
 	}
 	return ts, nil
 }
@@ -60,6 +61,7 @@ func (dc *DuckConn) GetSourceTables(ctx context.Context) (shared.SourceTables, e
 		if err := rows.Scan(&table.Name); err != nil {
 			log.Fatalf("Error scanning tables: %v\n", err)
 		}
+		table.Schema = dc.Schema
 		ts.SourceTables = append(ts.SourceTables, table)
 	}
 	return ts, nil

sourcerer/put_columns_on_tables.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 
 	"github.com/gwenwindflower/tbd/shared"
+	"github.com/schollz/progressbar/v3"
 )
 
 func PutColumnsOnTables(ctx context.Context, ts shared.SourceTables, dbc DbConn) error {
@@ -20,6 +21,13 @@ func PutColumnsOnTables(ctx context.Context, ts shared.SourceTables, dbc DbConn)
 	}
 	mutex := sync.Mutex{}
 
+	bar := progressbar.NewOptions(len(ts.SourceTables),
+		progressbar.OptionSetWidth(5),
+		progressbar.OptionShowCount(),
+		progressbar.OptionShowElapsedTimeOnFinish(),
+		progressbar.OptionEnableColorCodes(true),
+		progressbar.OptionSetDescription("[magenta]🏎️✨ Getting warehouse metadata...[reset]"),
+	)
 	var wg sync.WaitGroup
 	wg.Add(len(ts.SourceTables))
 	for i := range ts.SourceTables {
@@ -44,6 +52,7 @@ func PutColumnsOnTables(ctx context.Context, ts shared.SourceTables, dbc DbConn)
 					}
 				}
 			}
+			bar.Add(1)
 			mutex.Unlock()
 			return nil
 		}(i)

staging_template.sql

@@ -2,7 +2,7 @@ with
 
 source as (
 
-    select * from {{ "{{" }} ref('{{.Name}}') {{ "}}" }}
+    select * from {{ "{{" }} source('{{.Schema}}', '{{.Name}}') {{ "}}" }}
 
 ),
 

write_staging_models.go

@@ -15,12 +15,12 @@ import (
 //go:embed *.sql
 var stagingTemplate embed.FS
 
-func WriteStagingModels(tables shared.SourceTables, buildDir string, prefix string) {
+func WriteStagingModels(ts shared.SourceTables, buildDir string, prefix string) {
 	var wg sync.WaitGroup
 
-	for _, table := range tables.SourceTables {
+	for _, t := range ts.SourceTables {
 		wg.Add(1)
-		go func(table shared.SourceTable) {
+		go func(t shared.SourceTable) {
 			defer wg.Done()
 
 			tmpl := template.New("staging_template.sql").Funcs(template.FuncMap{"lower": strings.ToLower})
@@ -29,18 +29,18 @@ func WriteStagingModels(tables shared.SourceTables, buildDir string, prefix stri
 				log.Fatalf("Failed to parse template %v\n", err)
 			}
 
-			filename := fmt.Sprintf(buildDir + "/" + prefix + "_" + strings.ToLower(table.Name) + ".sql")
+			filename := fmt.Sprintf(buildDir + "/" + prefix + "_" + strings.ToLower(t.Name) + ".sql")
 			outputFile, err := os.Create(filename)
 			if err != nil {
 				log.Fatalf("Failed to create file %v\n", err)
 			}
 			defer outputFile.Close()
 
-			err = tmpl.Execute(outputFile, table)
+			err = tmpl.Execute(outputFile, t)
 			if err != nil {
 				log.Fatalf("Failed to execute template %v\n", err)
 			}
-		}(table)
+		}(t)
 	}
 	wg.Wait()
 }

write_staging_models_test.go

@@ -11,7 +11,8 @@ func TestWriteStagingModels(t *testing.T) {
 	ts := shared.SourceTables{
 		SourceTables: []shared.SourceTable{
 			{
-				Name: "table1",
+				Name:   "table1",
+				Schema: "raw",
 				Columns: []shared.Column{
 					{
 						Name:     "COLUMN3",
@@ -43,7 +44,7 @@ func TestWriteStagingModels(t *testing.T) {
 
 source as (
 
-    select * from {{ ref('table1') }}
+    select * from {{ source('raw', 'table1') }}
 
 ),