Support OpenAPI source

core/config/config.exs

@@ -60,6 +60,7 @@ config :canary, Oban,
     default: 10,
     github_processor: 2,
     webpage_processor: 10,
+    openapi_processor: 10,
     email: 10
   ],
   repo: Canary.Repo,

core/lib/canary/application.ex

@@ -10,6 +10,7 @@ defmodule Canary.Application do
     attach_oban_telemetry()
 
     :ok = Canary.Index.Collection.ensure(:webpage)
+    :ok = Canary.Index.Collection.ensure(:openapi)
     :ok = Canary.Index.Collection.ensure(:github_issue)
     :ok = Canary.Index.Collection.ensure(:github_discussion)
     :ok = Canary.Index.Stopword.ensure()

core/lib/canary/index/collection.ex

@@ -1,5 +1,5 @@
 defmodule Canary.Index.Collection do
-  def ensure(name) when name in [:webpage, :github_issue, :github_discussion] do
+  def ensure(name) when name in [:webpage, :openapi, :github_issue, :github_discussion] do
     with {:error, _} <- Canary.Index.Client.get_collection(name),
          {:error, _} <- Canary.Index.Client.create_collection(name, fields(name)) do
       :error
@@ -8,7 +8,7 @@ defmodule Canary.Index.Collection do
     end
   end
 
-  defp fields(name) when name in [:webpage, :github_issue, :github_discussion] do
+  defp fields(name) when name in [:webpage, :openapi, :github_issue, :github_discussion] do
     # https://typesense.org/docs/27.0/api/collections.html#indexing-all-but-some-fields
     shared = [
       %{name: "source_id", type: "string"},
@@ -26,6 +26,15 @@ defmodule Canary.Index.Collection do
             %{name: "content", type: "string", stem: true}
           ]
 
+        :openapi ->
+          [
+            %{name: "path", type: "string", stem: true},
+            %{name: "get", type: "string", stem: true, optional: true},
+            %{name: "post", type: "string", stem: true, optional: true},
+            %{name: "put", type: "string", stem: true, optional: true},
+            %{name: "delete", type: "string", stem: true, optional: true}
+          ]
+
         :github_issue ->
           [
             %{name: "title", type: "string", stem: true},

core/lib/canary/index/document.ex

@@ -33,6 +33,20 @@ defmodule Canary.Index.Document.Webpage.Meta do
   defstruct Shared.meta_fields()
 end
 
+defmodule Canary.Index.Document.OpenAPI do
+  alias Canary.Index.Document.Shared
+
+  @derive Jason.Encoder
+  defstruct Shared.top_level_fields() ++ [:path, :get, :post, :put, :delete]
+end
+
+defmodule Canary.Index.Document.OpenAPI.Meta do
+  alias Canary.Index.Document.Shared
+
+  @derive Jason.Encoder
+  defstruct Shared.meta_fields()
+end
+
 defmodule Canary.Index.Document.GithubIssue do
   alias Canary.Index.Document.Shared
 

core/lib/canary/index/index.ex

@@ -1,6 +1,7 @@
 defmodule Canary.Index do
   alias Canary.Sources.Source
   alias Canary.Sources.Webpage
+  alias Canary.Sources.OpenAPI
   alias Canary.Sources.GithubIssue
   alias Canary.Sources.GithubDiscussion
 
@@ -29,6 +30,29 @@ defmodule Canary.Index do
     Client.index_document(:webpage, doc)
   end
 
+  def insert_document(%OpenAPI.Chunk{} = chunk) do
+    meta = %Document.OpenAPI.Meta{
+      url: chunk.url,
+      document_id: chunk.document_id,
+      is_parent: false
+    }
+
+    doc = %Document.OpenAPI{
+      id: chunk.index_id,
+      source_id: chunk.source_id,
+      path: chunk.path,
+      get: chunk.get,
+      post: chunk.post,
+      put: chunk.put,
+      delete: chunk.delete,
+      tags: [],
+      is_empty_tags: true,
+      meta: meta
+    }
+
+    Client.index_document(:openapi, doc)
+  end
+
   def insert_document(%GithubIssue.Chunk{} = chunk) do
     meta = %Document.GithubIssue.Meta{
       url: chunk.url,
@@ -72,6 +96,7 @@ defmodule Canary.Index do
   def delete_document(source_type, id)
       when source_type in [
              :webpage,
+             :openapi,
              :github_issue,
              :github_discussion
            ] do
@@ -126,29 +151,41 @@ defmodule Canary.Index do
         |> Enum.reject(&is_nil/1)
         |> Enum.join(" && ")
 
-      query_by = ["title", "content"] |> Enum.join(",")
-      query_by_weights = [3, 1] |> Enum.join(",")
-
       %{
         collection: to_string(type),
         q: query,
         prefix: true,
-        query_by: query_by,
-        query_by_weights: query_by_weights,
         filter_by: filter_by,
         sort_by: "_text_match:desc",
-        highlight_fields: "content",
         stopwords: Canary.Index.Stopword.id(),
         prioritize_exact_match: true,
         prioritize_token_position: false,
         prioritize_num_matching_fields: false,
         max_candidates: 4 * 4
       }
-      |> add_embedding_args(opts)
+      |> handle_source_type(type)
+      |> handle_embedding(opts)
     end)
   end
 
-  defp add_embedding_args(args, opts) do
+  defp handle_source_type(args, type)
+       when type in [:webpage, :github_issue, :github_discussion] do
+    args
+    |> Map.put(:highlight_fields, "content")
+    |> Map.put(:query_by, Enum.join(["title", "content"], ","))
+    |> Map.put(:query_by_weights, Enum.join([3, 1], ","))
+  end
+
+  defp handle_source_type(args, type) when type in [:openapi] do
+    ops = ["get", "post", "put", "delete"]
+
+    args
+    |> Map.put(:highlight_fields, Enum.join(ops, ","))
+    |> Map.put(:query_by, Enum.join(["path"] ++ ops, ","))
+    |> Map.put(:query_by_weights, Enum.join([1, 2, 2, 2, 2], ","))
+  end
+
+  defp handle_embedding(args, opts) do
     embedding = opts[:embedding]
     embedding_alpha = opts[:embedding_alpha] || 0.3
 

core/lib/canary/searcher.ex

@@ -108,6 +108,9 @@ defmodule Canary.Searcher.Default do
               :webpage ->
                 %{}
 
+              :openapi ->
+                %{}
+
               :github_issue ->
                 %{closed: doc.meta.value.closed}
 

core/lib/canary/sources/document/create_openapi.ex

@@ -0,0 +1,90 @@
+defmodule Canary.Sources.Document.CreateOpenAPI do
+  use Ash.Resource.Change
+
+  alias Canary.Sources.Document
+  alias Canary.Sources.OpenAPI
+
+  @impl true
+  def init(opts) do
+    if [
+         :source_id_argument,
+         :fetcher_result_argument,
+         :chunks_attribute,
+         :meta_attribute
+       ]
+       |> Enum.any?(&is_nil(opts[&1])) do
+      :error
+    else
+      {:ok, opts}
+    end
+  end
+
+  @impl true
+  def change(changeset, opts, _context) do
+    source_id = Ash.Changeset.get_argument(changeset, opts[:source_id_argument])
+
+    %OpenAPI.FetcherResult{schema: %OpenApiSpex.OpenApi{} = schema, served_url: served_url} =
+      Ash.Changeset.get_argument(changeset, opts[:fetcher_result_argument])
+
+    changeset
+    |> Ash.Changeset.change_attribute(opts[:meta_attribute], wrap_union(%OpenAPI.DocumentMeta{}))
+    |> Ash.Changeset.change_attribute(opts[:chunks_attribute], [])
+    |> Ash.Changeset.after_action(fn _, record ->
+      hash =
+        schema
+        |> Jason.encode!()
+        |> then(&:crypto.hash(:sha256, &1))
+        |> Base.encode16(case: :lower)
+
+      chunks_create_result =
+        schema.paths
+        |> Enum.map(fn
+          {path, %OpenApiSpex.PathItem{get: get, post: post, put: put, delete: delete}} ->
+            %{
+              source_id: source_id,
+              document_id: record.id,
+              url: render_url(served_url, path),
+              path: path,
+              get: render_operation(get),
+              post: render_operation(post),
+              put: render_operation(put),
+              delete: render_operation(delete)
+            }
+        end)
+        |> Ash.bulk_create(OpenAPI.Chunk, :create,
+          return_errors?: true,
+          return_records?: true
+        )
+
+      meta = %OpenAPI.DocumentMeta{hash: hash}
+
+      case chunks_create_result do
+        %Ash.BulkResult{status: :success, records: records} ->
+          case Document.update(record, wrap_union(meta), Enum.map(records, &wrap_union/1)) do
+            {:ok, updated_record} -> {:ok, updated_record}
+            error -> error
+          end
+
+        %Ash.BulkResult{errors: errors} ->
+          {:error, errors}
+      end
+    end)
+  end
+
+  defp render_url(base_url, path) do
+    URI.parse(base_url)
+    |> Map.put(:fragment, ":~:text=#{path}")
+    |> URI.to_string()
+  end
+
+  defp render_operation(nil), do: nil
+
+  defp render_operation(%OpenApiSpex.Operation{} = op) do
+    [op.summary, op.description, op.operationId]
+    |> Enum.reject(&is_nil/1)
+    |> Enum.join(" / ")
+  end
+
+  defp wrap_union(%Ash.Union{} = v), do: v
+  defp wrap_union(v), do: %Ash.Union{type: :openapi, value: v}
+end

core/lib/canary/sources/document/document.ex

@@ -46,6 +46,21 @@ defmodule Canary.Sources.Document do
       }
     end
 
+    create :create_openapi do
+      argument :source_id, :uuid, allow_nil?: false
+      argument :fetcher_result, :map, allow_nil?: false
+
+      change manage_relationship(:source_id, :source, type: :append)
+
+      change {
+        Canary.Sources.Document.CreateOpenAPI,
+        source_id_argument: :source_id,
+        fetcher_result_argument: :fetcher_result,
+        chunks_attribute: :chunks,
+        meta_attribute: :meta
+      }
+    end
+
     create :create_github_issue do
       argument :source_id, :uuid, allow_nil?: false
       argument :fetcher_results, {:array, :map}, allow_nil?: false

core/lib/canary/sources/openapi_chunk.ex

@@ -0,0 +1,37 @@
+defmodule Canary.Sources.OpenAPI.Chunk do
+  use Ash.Resource, data_layer: :embedded
+
+  @ops [:get, :post, :put, :delete]
+
+  attributes do
+    attribute :index_id, :uuid, allow_nil?: false
+    attribute :source_id, :uuid, allow_nil?: false
+    attribute :document_id, :string, allow_nil?: false
+
+    attribute :url, :string, allow_nil?: false
+    attribute :path, :string, allow_nil?: false
+
+    Enum.each(@ops, fn op ->
+      attribute op, :string, allow_nil?: true
+    end)
+  end
+
+  actions do
+    defaults [:read]
+
+    create :create do
+      primary? true
+      accept [:index_id, :source_id, :document_id] ++ [:url, :path] ++ @ops
+      change {Canary.Change.AddToIndex, index_id_attribute: :index_id}
+    end
+
+    destroy :destroy do
+      primary? true
+
+      change {
+        Canary.Change.RemoveFromIndex,
+        source_type: :openapi, index_id_attribute: :index_id
+      }
+    end
+  end
+end

core/lib/canary/sources/openapi_config.ex

@@ -0,0 +1,26 @@
+defmodule Canary.Sources.OpenAPI.Config do
+  use Ash.Resource, data_layer: :embedded
+
+  attributes do
+    attribute :source_url, :string, allow_nil?: false
+    attribute :served_url, :string, allow_nil?: false
+
+    attribute :served_as, :atom,
+      constraints: [one_of: [:swagger, :redoc, :rapi]],
+      allow_nil?: true
+  end
+
+  actions do
+    defaults [:read]
+
+    create :create do
+      primary? true
+      accept [:source_url, :served_url, :served_as]
+    end
+
+    update :update do
+      primary? true
+      accept [:source_url, :served_url, :served_as]
+    end
+  end
+end

core/lib/canary/sources/openapi_document_meta.ex

@@ -0,0 +1,21 @@
+defmodule Canary.Sources.OpenAPI.DocumentMeta do
+  use Ash.Resource, data_layer: :embedded
+
+  attributes do
+    attribute :hash, :string, allow_nil?: false
+  end
+
+  actions do
+    defaults [:read, :destroy]
+
+    create :create do
+      primary? true
+      accept [:hash]
+    end
+
+    update :update do
+      primary? true
+      accept [:hash]
+    end
+  end
+end