From 76872912e0c156acfb0eff8f10d5b8734f516a57 Mon Sep 17 00:00:00 2001 From: Valentino Stoll Date: Fri, 25 Aug 2023 14:24:17 -0400 Subject: [PATCH 1/7] Update README.md to also include copying of database.yml with initial setup. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index f085a82..f3ca119 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,11 @@ configure environment variables cp .env.example .env.local ``` +configure database +```sh +cp config/database.yml.example config/database.yml +``` + fire up the app ```sh docker-compose up --build From 696a9ebdb279d44c5745c97c13b7cf1e6404fb41 Mon Sep 17 00:00:00 2001 From: Valentino Stoll Date: Fri, 25 Aug 2023 14:25:02 -0400 Subject: [PATCH 2/7] Remove duplicate 'tags' method definition from the Conversation model --- app/models/conversation.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/models/conversation.rb b/app/models/conversation.rb index 7544a12..e3ee83a 100644 --- a/app/models/conversation.rb +++ b/app/models/conversation.rb @@ -116,10 +116,6 @@ def summary analysis[:summary] end - def tags - analysis[:tags] || [] - end - def total_token_count messages.sum(:tokens_count) end From 0e61e332019ee74778d84f8824987c5deacad84c Mon Sep 17 00:00:00 2001 From: Valentino Stoll Date: Fri, 25 Aug 2023 14:26:02 -0400 Subject: [PATCH 3/7] Update SessionsController to use Google's timezone from user info as it is formatted properly already --- app/controllers/sessions_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index 5d7ed60..9ab7c1f 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -9,7 +9,7 @@ def create # todo: should we do this everytime? calendar = service.get_calendar('primary') - user.update!(time_zone: calendar.time_zone.split('/').last.gsub('_', ' ')) + user.update!(time_zone: calendar.time_zone) redirect_to "/conversations", notice: "Signed in!" end From 4d3d2b407a966933d045d43d14b8052d0c88bbc6 Mon Sep 17 00:00:00 2001 From: Valentino Stoll Date: Fri, 25 Aug 2023 14:29:10 -0400 Subject: [PATCH 4/7] Update docker-compose.yml.example to mount an 'app' volume so that the 'css' service can use Rails commands --- docker-compose.yml.example | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml.example b/docker-compose.yml.example index 26fc047..e73f454 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -76,6 +76,7 @@ services: volumes: + app: null redis-data: null pg-data: null From d9fd29d51545f719716dcab6c533224486844919 Mon Sep 17 00:00:00 2001 From: Valentino Stoll Date: Fri, 25 Aug 2023 14:30:05 -0400 Subject: [PATCH 5/7] Add docker-compose.yml.marqo.example to showcase how to bundle marqo with the rest of the docker stack --- README.md | 2 + docker-compose.yml.marqo.example | 99 ++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 docker-compose.yml.marqo.example diff --git a/README.md b/README.md index f3ca119..bdbbd4c 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ Right now the only authentication method supported is Google Oauth. You'll need If you're using Marqo, make sure to set the `MARQO_URL` environment variable, otherwise the `MemoryAnnotator` will not run. +You can use the `docker-compose.yml.marqo.example` docker-compose configuration to get marqo propped up with everything else. If you do this, you can set `MARQO_URL=http://host.docker.internal:8882` + ### Admin User Admin privileges are granted simply with the `admin` boolean attribute on `User`. There is no admin UI at the moment, so if you want to give your user admin rights, do it via the console. diff --git a/docker-compose.yml.marqo.example b/docker-compose.yml.marqo.example new file mode 100644 index 0000000..8d5eb80 --- /dev/null +++ b/docker-compose.yml.marqo.example @@ -0,0 +1,99 @@ +version: "3" + +x-app: &default-app + build: + context: "." + dockerfile: Dockerfile + depends_on: + redis: + condition: service_healthy + db: + condition: service_healthy + env_file: + - .env.local + tty: true + volumes: + - .:/rails + +x-assets: &default-assets + build: + context: "." + dockerfile: Dockerfile + env_file: + - .env.local + tty: true + volumes: + - .:/rails + entrypoint: [] + ports: [] + +services: + marqoai: + image: marqoai/marqo:latest + privileged: true + ports: + - '8882:8882' + volumes: + - 'vector-data:/data' + networks: + default: + + redis: + image: redis + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 5 + command: redis-server --requirepass password + ports: + - '6380:6379' + volumes: + - 'redis-data:/data' + networks: + default: + + db: + image: postgres:15 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U rails"] + interval: 5s + timeout: 5s + retries: 5 + ports: + - "5433:5432" + restart: always + environment: + - POSTGRES_USER=rails + - POSTGRES_PASSWORD=password + - POSTGRES_HOST_AUTH_METHOD=trust + volumes: + - 'pg-data:/var/lib/postgresql/data' + - './db/development.sql:/docker-entrypoint-initdb.d/setup.sql' + networks: + default: + + web: + <<: *default-app + privileged: true + depends_on: + - marqoai + ports: + - "3000:3000" + networks: + default: + + css: + <<: *default-assets + command: bin/rails tailwindcss:watch + + +volumes: + app: null + redis-data: null + pg-data: null + vector-data: null + +networks: + default: + driver: bridge From 9c5b4dbc977e2befa673cd7385232fc8ce73b833 Mon Sep 17 00:00:00 2001 From: Valentino Stoll Date: Fri, 25 Aug 2023 14:31:29 -0400 Subject: [PATCH 6/7] Ensure new conversations for non-admins are hooked up with the proper botselect Stimulus controller --- app/views/conversations/new.html.erb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/views/conversations/new.html.erb b/app/views/conversations/new.html.erb index fddf308..57a1571 100644 --- a/app/views/conversations/new.html.erb +++ b/app/views/conversations/new.html.erb @@ -9,7 +9,8 @@
" data-placeholder="<%= it("Start a new conversation with #{bot.name.force_encoding("UTF-8")}") %>" - data-hello="<%= hello_in_user_language&.force_encoding("UTF-8") %>"> + data-hello="<%= hello_in_user_language&.force_encoding("UTF-8") %>" + data-controller="botselect" data-action="click->botselect#select">
<%= image_tag bot.image_url, class: "object-fit" %>
From 79cec9417a88c6914c5938866c758cba39aa5f38 Mon Sep 17 00:00:00 2001 From: Valentino Stoll Date: Fri, 25 Aug 2023 14:34:11 -0400 Subject: [PATCH 7/7] Automatically create marqo indexes when storing if it doesn't already exist --- app/services/marqo.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/app/services/marqo.rb b/app/services/marqo.rb index f2cad46..aebce89 100644 --- a/app/services/marqo.rb +++ b/app/services/marqo.rb @@ -15,6 +15,9 @@ class Marqo base_uri ENV.fetch('MARQO_URL') + class IndexNotFound < StandardError + end + class SearchResult < RecursiveOpenStruct end @@ -23,8 +26,25 @@ def initialize(auth = { username: 'admin', password: 'admin' }) @auth = auth end + def create(index:, model: "hf/all_datasets_v4_MiniLM-L6") + puts + puts "🧠🧠🧠 CREATING INDEX: #{index} 🧠🧠🧠" + puts + options = { + headers: { 'Content-Type' => 'application/json' }, + body: { index_defaults: { model: model } }.to_json + } + url = "/indexes/#{index.to_s.parameterize}" + self.class.post(url, options).then do |response| + puts response + end + end + def store(index:, doc:, id:, non_tensor_fields: []) return if ENV['MARQO_URL'].blank? + + create_index_attempts ||= 0 + puts puts "🧠🧠🧠 INDEX: #{index} 🧠🧠🧠" puts "🧠🧠🧠 DOC: #{doc} 🧠🧠🧠" @@ -42,8 +62,16 @@ def store(index:, doc:, id:, non_tensor_fields: []) end self.class.post(url, options).then do |response| puts response + raise IndexNotFound if response["type"].to_s == "invalid_request" && response["code"].to_s == "index_not_found" response.dig("items",0,"_id") end + rescue IndexNotFound + create_index_attempts += 1 + + if create_index_attempts < 2 + create(index: index) + retry + end end def search(index_name, query, filter: nil, limit: 5)