From 3c9587aa5f676e79c37632e1b6d8a2c831970d1a Mon Sep 17 00:00:00 2001 From: Tyson Gern Date: Tue, 30 Apr 2024 07:53:35 -0600 Subject: [PATCH] Move ai to pkg --- cmd/analyzer/main.go | 2 +- cmd/collector/main.go | 6 +++--- functions.go | 8 ++++---- internal/analyzer/analyze.go | 2 +- internal/app/handlers.go | 2 +- internal/app/index.go | 2 +- internal/collector/chunks_service.go | 2 +- {internal => pkg}/ai/chunker.go | 0 {internal => pkg}/ai/chunker_test.go | 6 +++--- {internal => pkg}/ai/client.go | 0 {internal => pkg}/ai/tokenizer.go | 0 {internal => pkg}/ai/tokenizer_test.go | 2 +- 12 files changed, 16 insertions(+), 16 deletions(-) rename {internal => pkg}/ai/chunker.go (100%) rename {internal => pkg}/ai/chunker_test.go (77%) rename {internal => pkg}/ai/client.go (100%) rename {internal => pkg}/ai/tokenizer.go (100%) rename {internal => pkg}/ai/tokenizer_test.go (85%) diff --git a/cmd/analyzer/main.go b/cmd/analyzer/main.go index e045c39..c4df8a3 100644 --- a/cmd/analyzer/main.go +++ b/cmd/analyzer/main.go @@ -2,9 +2,9 @@ package main import ( "context" - "github.com/initialcapacity/ai-starter/internal/ai" "github.com/initialcapacity/ai-starter/internal/analyzer" "github.com/initialcapacity/ai-starter/internal/collector" + "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/initialcapacity/ai-starter/pkg/dbsupport" "github.com/initialcapacity/ai-starter/pkg/websupport" "log/slog" diff --git a/cmd/collector/main.go b/cmd/collector/main.go index ff9ea3f..3704f44 100644 --- a/cmd/collector/main.go +++ b/cmd/collector/main.go @@ -1,8 +1,8 @@ package main import ( - "github.com/initialcapacity/ai-starter/internal/ai" "github.com/initialcapacity/ai-starter/internal/collector" + ai2 "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/initialcapacity/ai-starter/pkg/dbsupport" "github.com/initialcapacity/ai-starter/pkg/feedsupport" "github.com/initialcapacity/ai-starter/pkg/websupport" @@ -23,9 +23,9 @@ func main() { parser := feedsupport.NewParser(client) extractor := feedsupport.NewExtractor(client) dataGateway := collector.NewDataGateway(db) - t := ai.NewTokenizer(tokenizer.Cl100kBase) + t := ai2.NewTokenizer(tokenizer.Cl100kBase) chunksGateway := collector.NewChunksGateway(db) - chunker := ai.NewChunker(t, 6000) + chunker := ai2.NewChunker(t, 6000) chunksService := collector.NewChunksService(chunker, chunksGateway) c := collector.New(parser, extractor, dataGateway, chunksService) diff --git a/functions.go b/functions.go index 18b5908..e333232 100644 --- a/functions.go +++ b/functions.go @@ -4,9 +4,9 @@ import ( "context" "github.com/GoogleCloudPlatform/functions-framework-go/functions" "github.com/cloudevents/sdk-go/v2/event" - "github.com/initialcapacity/ai-starter/internal/ai" "github.com/initialcapacity/ai-starter/internal/analyzer" "github.com/initialcapacity/ai-starter/internal/collector" + ai2 "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/initialcapacity/ai-starter/pkg/dbsupport" "github.com/initialcapacity/ai-starter/pkg/feedsupport" "github.com/initialcapacity/ai-starter/pkg/websupport" @@ -31,9 +31,9 @@ func triggerCollect(ctx context.Context, e event.Event) error { parser := feedsupport.NewParser(client) extractor := feedsupport.NewExtractor(client) dataGateway := collector.NewDataGateway(db) - t := ai.NewTokenizer(tokenizer.Cl100kBase) + t := ai2.NewTokenizer(tokenizer.Cl100kBase) chunksGateway := collector.NewChunksGateway(db) - chunker := ai.NewChunker(t, 6000) + chunker := ai2.NewChunker(t, 6000) chunksService := collector.NewChunksService(chunker, chunksGateway) c := collector.New(parser, extractor, dataGateway, chunksService) @@ -48,7 +48,7 @@ func triggerAnalyze(ctx context.Context, e event.Event) error { db := dbsupport.CreateConnection(databaseUrl) chunksGateway := collector.NewChunksGateway(db) embeddingsGateway := analyzer.NewEmbeddingsGateway(db) - aiClient := ai.NewClient(openAiKey) + aiClient := ai2.NewClient(openAiKey) a := analyzer.NewAnalyzer(chunksGateway, embeddingsGateway, aiClient) diff --git a/internal/analyzer/analyze.go b/internal/analyzer/analyze.go index 5007f40..4efedb9 100644 --- a/internal/analyzer/analyze.go +++ b/internal/analyzer/analyze.go @@ -4,8 +4,8 @@ import ( "context" "errors" "fmt" - "github.com/initialcapacity/ai-starter/internal/ai" "github.com/initialcapacity/ai-starter/internal/collector" + "github.com/initialcapacity/ai-starter/pkg/ai" "log/slog" ) diff --git a/internal/app/handlers.go b/internal/app/handlers.go index 3d981fc..1df6fd1 100644 --- a/internal/app/handlers.go +++ b/internal/app/handlers.go @@ -1,8 +1,8 @@ package app import ( - "github.com/initialcapacity/ai-starter/internal/ai" "github.com/initialcapacity/ai-starter/internal/analyzer" + "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/initialcapacity/ai-starter/pkg/dbsupport" "io/fs" "net/http" diff --git a/internal/app/index.go b/internal/app/index.go index e94e265..2a15dac 100644 --- a/internal/app/index.go +++ b/internal/app/index.go @@ -2,8 +2,8 @@ package app import ( "fmt" - "github.com/initialcapacity/ai-starter/internal/ai" "github.com/initialcapacity/ai-starter/internal/analyzer" + "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/initialcapacity/ai-starter/pkg/deferrable" "github.com/initialcapacity/ai-starter/pkg/websupport" "log/slog" diff --git a/internal/collector/chunks_service.go b/internal/collector/chunks_service.go index e96ffb7..bde9c65 100644 --- a/internal/collector/chunks_service.go +++ b/internal/collector/chunks_service.go @@ -2,7 +2,7 @@ package collector import ( "errors" - "github.com/initialcapacity/ai-starter/internal/ai" + "github.com/initialcapacity/ai-starter/pkg/ai" ) type ChunksService struct { diff --git a/internal/ai/chunker.go b/pkg/ai/chunker.go similarity index 100% rename from internal/ai/chunker.go rename to pkg/ai/chunker.go diff --git a/internal/ai/chunker_test.go b/pkg/ai/chunker_test.go similarity index 77% rename from internal/ai/chunker_test.go rename to pkg/ai/chunker_test.go index 20103a4..264b63f 100644 --- a/internal/ai/chunker_test.go +++ b/pkg/ai/chunker_test.go @@ -1,15 +1,15 @@ package ai_test import ( - "github.com/initialcapacity/ai-starter/internal/ai" + ai2 "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/stretchr/testify/assert" "github.com/tiktoken-go/tokenizer" "testing" ) func TestChunker_Split(t *testing.T) { - token := ai.NewTokenizer(tokenizer.Cl100kBase) - chunker := ai.NewChunker(token, 30) + token := ai2.NewTokenizer(tokenizer.Cl100kBase) + chunker := ai2.NewChunker(token, 30) result := chunker.Split("I think that this string should have 31 tokens, but it's hard to say for sure. We'll have to count them manually, I guess.") diff --git a/internal/ai/client.go b/pkg/ai/client.go similarity index 100% rename from internal/ai/client.go rename to pkg/ai/client.go diff --git a/internal/ai/tokenizer.go b/pkg/ai/tokenizer.go similarity index 100% rename from internal/ai/tokenizer.go rename to pkg/ai/tokenizer.go diff --git a/internal/ai/tokenizer_test.go b/pkg/ai/tokenizer_test.go similarity index 85% rename from internal/ai/tokenizer_test.go rename to pkg/ai/tokenizer_test.go index 61343d6..c98fd0f 100644 --- a/internal/ai/tokenizer_test.go +++ b/pkg/ai/tokenizer_test.go @@ -1,7 +1,7 @@ package ai_test import ( - "github.com/initialcapacity/ai-starter/internal/ai" + "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/stretchr/testify/assert" tokenizer2 "github.com/tiktoken-go/tokenizer" "testing"