diff --git a/cmd/collector/main.go b/cmd/collector/main.go index 3704f44..6b11b98 100644 --- a/cmd/collector/main.go +++ b/cmd/collector/main.go @@ -2,7 +2,7 @@ package main import ( "github.com/initialcapacity/ai-starter/internal/collector" - ai2 "github.com/initialcapacity/ai-starter/pkg/ai" + "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/initialcapacity/ai-starter/pkg/dbsupport" "github.com/initialcapacity/ai-starter/pkg/feedsupport" "github.com/initialcapacity/ai-starter/pkg/websupport" @@ -23,9 +23,9 @@ func main() { parser := feedsupport.NewParser(client) extractor := feedsupport.NewExtractor(client) dataGateway := collector.NewDataGateway(db) - t := ai2.NewTokenizer(tokenizer.Cl100kBase) + t := ai.NewTokenizer(tokenizer.Cl100kBase) chunksGateway := collector.NewChunksGateway(db) - chunker := ai2.NewChunker(t, 6000) + chunker := ai.NewChunker(t, 6000) chunksService := collector.NewChunksService(chunker, chunksGateway) c := collector.New(parser, extractor, dataGateway, chunksService) diff --git a/functions.go b/functions.go index 3e3950a..e63353d 100644 --- a/functions.go +++ b/functions.go @@ -6,7 +6,7 @@ import ( "github.com/cloudevents/sdk-go/v2/event" "github.com/initialcapacity/ai-starter/internal/analyzer" "github.com/initialcapacity/ai-starter/internal/collector" - ai2 "github.com/initialcapacity/ai-starter/pkg/ai" + "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/initialcapacity/ai-starter/pkg/dbsupport" "github.com/initialcapacity/ai-starter/pkg/feedsupport" "github.com/initialcapacity/ai-starter/pkg/websupport" @@ -31,9 +31,9 @@ func triggerCollect(ctx context.Context, e event.Event) error { parser := feedsupport.NewParser(client) extractor := feedsupport.NewExtractor(client) dataGateway := collector.NewDataGateway(db) - t := ai2.NewTokenizer(tokenizer.Cl100kBase) + t := ai.NewTokenizer(tokenizer.Cl100kBase) chunksGateway := collector.NewChunksGateway(db) - chunker := ai2.NewChunker(t, 6000) + chunker := ai.NewChunker(t, 6000) chunksService := collector.NewChunksService(chunker, chunksGateway) c := collector.New(parser, extractor, dataGateway, chunksService) @@ -49,7 +49,7 @@ func triggerAnalyze(ctx context.Context, e event.Event) error { db := dbsupport.CreateConnection(databaseUrl) chunksGateway := collector.NewChunksGateway(db) embeddingsGateway := analyzer.NewEmbeddingsGateway(db) - aiClient := ai2.NewClient(openAiKey, openAiEndpoint) + aiClient := ai.NewClient(openAiKey, openAiEndpoint) a := analyzer.NewAnalyzer(chunksGateway, embeddingsGateway, aiClient) diff --git a/pkg/ai/chunker_test.go b/pkg/ai/chunker_test.go index 264b63f..cdd0586 100644 --- a/pkg/ai/chunker_test.go +++ b/pkg/ai/chunker_test.go @@ -1,15 +1,15 @@ package ai_test import ( - ai2 "github.com/initialcapacity/ai-starter/pkg/ai" + "github.com/initialcapacity/ai-starter/pkg/ai" "github.com/stretchr/testify/assert" "github.com/tiktoken-go/tokenizer" "testing" ) func TestChunker_Split(t *testing.T) { - token := ai2.NewTokenizer(tokenizer.Cl100kBase) - chunker := ai2.NewChunker(token, 30) + token := ai.NewTokenizer(tokenizer.Cl100kBase) + chunker := ai.NewChunker(token, 30) result := chunker.Split("I think that this string should have 31 tokens, but it's hard to say for sure. We'll have to count them manually, I guess.")