feat: add analytics table for timeseries

Signed-off-by: Guillaume Hivert <[email protected]>

apps/backend/db/migrations/20240521204341_add_analytics_table.sql

@@ -0,0 +1,20 @@
+-- migrate:up
+create table analytics (
+  id integer primary key generated always as identity,
+  foreign_id int not null,
+  table_name text not null,
+  content jsonb not null,
+  day timestamptz not null,
+  created_at timestamptz default current_timestamp not null,
+  updated_at timestamptz default current_timestamp not null,
+  unique (foreign_id, table_name, day)
+);
+
+create trigger analytics_moddatetime
+  before update on analytics
+  for each row
+  execute procedure moddatetime (updated_at);
+
+-- migrate:down
+drop trigger analytics_moddatetime on analytics;
+drop table analytics;

apps/backend/db/schema.sql

@@ -53,6 +53,35 @@ SET default_tablespace = '';
 
 SET default_table_access_method = heap;
 
+--
+-- Name: analytics; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.analytics (
+    id integer NOT NULL,
+    foreign_id integer NOT NULL,
+    table_name text NOT NULL,
+    content jsonb NOT NULL,
+    day timestamp with time zone NOT NULL,
+    created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL
+);
+
+
+--
+-- Name: analytics_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+ALTER TABLE public.analytics ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY (
+    SEQUENCE NAME public.analytics_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1
+);
+
+
 --
 -- Name: hex_read; Type: TABLE; Schema: public; Owner: -
 --
@@ -255,6 +284,22 @@ CREATE TABLE public.schema_migrations (
 );
 
 
+--
+-- Name: analytics analytics_foreign_id_table_name_day_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.analytics
+    ADD CONSTRAINT analytics_foreign_id_table_name_day_key UNIQUE (foreign_id, table_name, day);
+
+
+--
+-- Name: analytics analytics_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.analytics
+    ADD CONSTRAINT analytics_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: hex_read hex_read_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -380,6 +425,13 @@ CREATE INDEX package_type_fun_signature_name ON public.package_type_fun_signatur
 CREATE INDEX package_type_fun_signature_signature ON public.package_type_fun_signature USING gin (to_tsvector('english'::regconfig, signature_));
 
 
+--
+-- Name: analytics analytics_moddatetime; Type: TRIGGER; Schema: public; Owner: -
+--
+
+CREATE TRIGGER analytics_moddatetime BEFORE UPDATE ON public.analytics FOR EACH ROW EXECUTE FUNCTION public.moddatetime('updated_at');
+
+
 --
 -- Name: hex_user hex_user_moddatetime; Type: TRIGGER; Schema: public; Owner: -
 --
@@ -486,4 +538,5 @@ INSERT INTO public.schema_migrations (version) VALUES
     ('20240514214138'),
     ('20240517083006'),
     ('20240518232212'),
-    ('20240521174525');
+    ('20240521174525'),
+    ('20240521204341');

apps/backend/src/backend/postgres/queries.gleam

@@ -9,7 +9,7 @@ import gleam/dynamic
 import gleam/hexpm
 import gleam/json
 import gleam/list
-import gleam/option.{type Option}
+import gleam/option.{type Option, None, Some}
 import gleam/package_interface
 import gleam/pair
 import gleam/pgo
@@ -623,8 +623,16 @@ pub fn update_package_rank(db: pgo.Connection, package: String, rank: Int) {
 }
 
 pub fn select_package_repository_address(db: pgo.Connection, offset: Int) {
-  let decoder = dynamic.element(0, dynamic.optional(dynamic.string))
-  "SELECT repository FROM package LIMIT 100 OFFSET $1"
+  let decoder = fn(dyn) {
+    dynamic.tuple2(dynamic.int, dynamic.optional(dynamic.string))(dyn)
+    |> result.map(fn(content) {
+      case content {
+        #(id, Some(repo)) -> Some(#(id, repo))
+        #(_, None) -> None
+      }
+    })
+  }
+  "SELECT id, repository FROM package LIMIT 100 OFFSET $1"
   |> pgo.execute(db, [pgo.int(offset)], decoder)
   |> result.map_error(error.DatabaseError)
   |> result.map(fn(r) { r.rows })
@@ -691,3 +699,31 @@ pub fn select_package_by_popularity(db: pgo.Connection, page: Int) {
   |> result.map(fn(r) { r.rows })
   |> result.map_error(error.DatabaseError)
 }
+
+pub fn insert_analytics(
+  db: pgo.Connection,
+  id: Int,
+  table_name: String,
+  content: Dict(String, Int),
+) {
+  let day =
+    birl.now()
+    |> birl.to_erlang_universal_datetime()
+    |> pair.map_second(fn(_) { #(0, 0, 0) })
+    |> dynamic.from()
+    |> dynamic.unsafe_coerce()
+  let content =
+    content
+    |> dict.to_list()
+    |> list.map(pair.map_second(_, json.int))
+    |> json.object()
+    |> json.to_string()
+    |> pgo.text()
+  let parameters = [pgo.int(id), pgo.text(table_name), content, day]
+  "INSERT INTO analytics (foreign_id, table_name, content, day)
+   VALUES ($1, $2, $3, $4)
+   ON CONFLICT (foreign_id, table_name, day) DO UPDATE
+     SET content = $3"
+  |> pgo.execute(db, parameters, dynamic.dynamic)
+  |> result.map_error(error.DatabaseError)
+}

apps/backend/src/tasks/popularity.gleam

@@ -34,11 +34,13 @@ fn do_compute_popularity(ctx: Context, offset offset: Int) {
   |> result.try(fn(_) { do_compute_popularity(ctx, offset: offset + 100) })
 }
 
-fn update_repo_popularity(ctx: Context, repo: String) {
-  wisp.log_debug("Syncing " <> repo)
-  use count <- result.try(github.get_stargazer_count(ctx.github_token, repo))
-  dict.from_list([#("github", count)])
-  |> queries.update_package_popularity(ctx.db, repo, _)
+fn update_repo_popularity(ctx: Context, repo: #(Int, String)) {
+  wisp.log_debug("Syncing " <> repo.1)
+  use count <- result.try(github.get_stargazer_count(ctx.github_token, repo.1))
+  let content = dict.from_list([#("github", count)])
+  let _ = queries.insert_analytics(ctx.db, repo.0, "package", content)
+  content
+  |> queries.update_package_popularity(ctx.db, repo.1, _)
   |> result.replace(Nil)
-  |> function.tap(fn(_) { wisp.log_debug("Synced " <> repo) })
+  |> function.tap(fn(_) { wisp.log_debug("Synced " <> repo.1) })
 }