feat: add github popularity fetching

Signed-off-by: Guillaume Hivert <[email protected]>

apps/backend/db/migrations/20240521174525_alter_table_package_add_popularity.sql

@@ -0,0 +1,9 @@
+-- migrate:up
+alter table package
+  add column popularity jsonb,
+  drop column favorites;
+
+-- migrate:down
+alter table package
+  drop column popularity,
+  add column favorites int not null default 0;

apps/backend/db/schema.sql

@@ -121,7 +121,7 @@ CREATE TABLE public.package (
     created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
     updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
     rank integer DEFAULT 0 NOT NULL,
-    favorites integer DEFAULT 0 NOT NULL
+    popularity jsonb
 );
 
 
@@ -485,4 +485,5 @@ INSERT INTO public.schema_migrations (version) VALUES
     ('20240512214036'),
     ('20240514214138'),
     ('20240517083006'),
-    ('20240518232212');
+    ('20240518232212'),
+    ('20240521174525');

apps/backend/gleam.toml

@@ -24,6 +24,7 @@ tom = ">= 1.0.0 and < 2.0.0"
 verl = ">= 1.1.1 and < 2.0.0"
 wisp = "~> 0.14"
 cors_builder = ">= 1.0.0 and < 2.0.0"
+decipher = ">= 1.2.0 and < 2.0.0"
 
 [dev-dependencies]
 gleeunit = "~> 1.0"

apps/backend/manifest.toml

@@ -6,6 +6,7 @@ packages = [
   { name = "backoff", version = "1.1.6", build_tools = ["rebar3"], requirements = [], otp_app = "backoff", source = "hex", outer_checksum = "CF0CFFF8995FB20562F822E5CC47D8CCF664C5ECDC26A684CBE85C225F9D7C39" },
   { name = "birl", version = "1.6.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "ranger"], otp_app = "birl", source = "hex", outer_checksum = "976CFF85D34D50F7775896615A71745FBE0C325E50399787088F941B539A0497" },
   { name = "cors_builder", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "mist", "wisp"], otp_app = "cors_builder", source = "hex", outer_checksum = "951B5B648E958BD6181A6EED98BCA4EEB302B83DC7DCE2954B3462114209EC43" },
+  { name = "decipher", version = "1.2.0", build_tools = ["gleam"], requirements = ["birl", "gleam_json", "gleam_stdlib", "stoiridh_version"], otp_app = "decipher", source = "hex", outer_checksum = "9F1B5C6FF0D798046E4E0EF87D09DD729324CB72BD7F0D4152B797324D51223E" },
   { name = "dot_env", version = "0.5.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "simplifile"], otp_app = "dot_env", source = "hex", outer_checksum = "AF5C972D6129F67AF3BB00134AB2808D37111A8D61686CFA86F3ADF652548982" },
   { name = "exception", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "exception", source = "hex", outer_checksum = "F5580D584F16A20B7FCDCABF9E9BE9A2C1F6AC4F9176FA6DD0B63E3B20D450AA" },
   { name = "filepath", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "EFB6FF65C98B2A16378ABC3EE2B14124168C0CE5201553DE652E2644DCFDB594" },
@@ -39,6 +40,7 @@ packages = [
   { name = "ranger", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "ranger", source = "hex", outer_checksum = "1566C272B1D141B3BBA38B25CB761EF56E312E79EC0E2DFD4D3C19FB0CC1F98C" },
   { name = "shellout", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "shellout", source = "hex", outer_checksum = "E2FCD18957F0E9F67E1F497FC9FF57393392F8A9BAEAEA4779541DE7A68DD7E0" },
   { name = "simplifile", version = "1.7.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "1D5DFA3A2F9319EC85825F6ED88B8E449F381B0D55A62F5E61424E748E7DDEB0" },
+  { name = "stoiridh_version", version = "0.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "stoiridh_version", source = "hex", outer_checksum = "298ABEA44DF37764A34C2E9190A84BF2770BC59DD9397C6DC7708040E5A0142B" },
   { name = "thoas", version = "1.2.1", build_tools = ["rebar3"], requirements = [], otp_app = "thoas", source = "hex", outer_checksum = "E38697EDFFD6E91BD12CEA41B155115282630075C2A727E7A6B2947F5408B86A" },
   { name = "tom", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "tom", source = "hex", outer_checksum = "A5364613E3DBF77F38EFF81DA9F99324086D029EC2B2D44348762FBE38602311" },
   { name = "verl", version = "1.1.1", build_tools = ["rebar3"], requirements = [], otp_app = "verl", source = "hex", outer_checksum = "0925E51CD92A0A8BE271765B02430B2E2CFF8AC30EF24D123BD0D58511E8FB18" },
@@ -48,7 +50,8 @@ packages = [
 [requirements]
 aws4_request = { version = ">= 0.1.1 and < 1.0.0" }
 birl = { version = "~> 1.6" }
-cors_builder = { version = ">= 1.0.0 and < 2.0.0"}
+cors_builder = { version = ">= 1.0.0 and < 2.0.0" }
+decipher = { version = ">= 1.2.0 and < 2.0.0"}
 dot_env = { version = "~> 0.5" }
 gleam_erlang = { version = "~> 0.25" }
 gleam_hexpm = { version = "~> 1.0" }

apps/backend/src/api/github.gleam

@@ -0,0 +1,60 @@
+import api/github/stargazer_count
+import backend/error
+import gleam/dynamic
+import gleam/function
+import gleam/http
+import gleam/http/request
+import gleam/httpc
+import gleam/json
+import gleam/list
+import gleam/option.{type Option, Some}
+import gleam/regex
+import gleam/result
+
+fn query(
+  token: String,
+  query: String,
+  variables: Option(json.Json),
+  decoder: dynamic.Decoder(a),
+) {
+  let body =
+    json.object([
+      #("query", json.string(query)),
+      #("variables", json.nullable(variables, function.identity)),
+    ])
+  use response <- result.try(
+    request.new()
+    |> request.set_header("authorization", "Bearer " <> token)
+    |> request.set_header("user-agent", "gloogle / 0.0.0")
+    |> request.set_method(http.Post)
+    |> request.set_scheme(http.Https)
+    |> request.set_host("api.github.com")
+    |> request.set_path("/graphql")
+    |> request.set_body(json.to_string(body))
+    |> httpc.send()
+    |> result.map_error(error.FetchError),
+  )
+
+  response.body
+  |> json.decode(using: decoder)
+  |> result.map_error(error.JsonError)
+}
+
+fn match_repository_name(repo_url: String) {
+  let assert Ok(owner_name) = regex.from_string("https://github.com/(.+)/(.+)")
+  regex.scan(with: owner_name, content: repo_url)
+  |> list.first()
+  |> result.replace_error(error.UnknownError(
+    "No repository match for " <> repo_url,
+  ))
+}
+
+pub fn get_stargazer_count(token: String, repo_url: String) {
+  use match <- result.try(match_repository_name(repo_url))
+  case match.submatches {
+    [Some(owner), Some(name)] ->
+      stargazer_count.variables(name, owner)
+      |> query(token, stargazer_count.query, _, stargazer_count.decoder)
+    _ -> Error(error.UnknownError(""))
+  }
+}

apps/backend/src/api/github/stargazer_count.gleam

@@ -0,0 +1,21 @@
+import decipher
+import gleam/dynamic
+import gleam/json
+import gleam/option.{Some}
+
+pub const query = "
+query getStargazers($name: String!, $owner: String!) {
+  repository(owner: $owner, name: $name) {
+    stargazerCount
+  }
+}"
+
+pub fn decoder(dyn) {
+  decipher.at(["data", "repository", "stargazerCount"], dynamic.int)(dyn)
+}
+
+pub fn variables(name: String, owner: String) {
+  Some(
+    json.object([#("name", json.string(name)), #("owner", json.string(owner))]),
+  )
+}

apps/backend/src/backend.gleam

@@ -9,6 +9,7 @@ import mist
 import periodic
 import setup
 import tasks/hex
+import tasks/popularity
 import tasks/ranking
 import wisp
 import wisp/logger
@@ -42,6 +43,9 @@ pub fn main() {
           add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() {
             ranking.compute_ranking(ctx)
           })
+          add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() {
+            popularity.compute_popularity(ctx)
+          })
         })
     })
 

apps/backend/src/backend/config.gleam

@@ -6,7 +6,7 @@ import wisp
 import wisp/logger
 
 pub type Context {
-  Context(db: pgo.Connection, hex_api_key: String)
+  Context(db: pgo.Connection, hex_api_key: String, github_token: String)
 }
 
 pub type Config {
@@ -15,20 +15,22 @@ pub type Config {
     hex_api_key: String,
     port: Int,
     level: logger.Level,
+    github_token: String,
   )
 }
 
 pub fn read_config() {
   let assert Ok(database_url) = os.get_env("DATABASE_URL")
   let assert Ok(hex_api_key) = os.get_env("HEX_API_KEY")
+  let assert Ok(github_token) = os.get_env("GITHUB_TOKEN")
   let assert Ok(port) =
     os.get_env("PORT")
     |> result.try(int.parse)
   let level =
     os.get_env("LOG_LEVEL")
     |> result.try(logger.parse)
     |> result.unwrap(logger.Info)
-  Config(database_url, hex_api_key, port, level)
+  Config(database_url, hex_api_key, port, level, github_token)
 }
 
 pub fn get_secret_key_base() {

apps/backend/src/backend/postgres/postgres.gleam

@@ -13,7 +13,13 @@ pub fn connect(cnf: Config) {
   let assert Ok(config) = parse_database_url(cnf.database_url)
   config
   |> pgo.connect()
-  |> fn(db) { Context(db: db, hex_api_key: cnf.hex_api_key) }
+  |> fn(db) {
+    Context(
+      db: db,
+      hex_api_key: cnf.hex_api_key,
+      github_token: cnf.github_token,
+    )
+  }
 }
 
 fn parse_database_url(database_url: String) {

apps/backend/src/backend/postgres/queries.gleam

@@ -4,13 +4,14 @@ import backend/error
 import backend/gleam/context
 import birl.{type Time}
 import gleam/bool
-import gleam/dict
+import gleam/dict.{type Dict}
 import gleam/dynamic
 import gleam/hexpm
 import gleam/json
 import gleam/list
 import gleam/option.{type Option}
 import gleam/package_interface
+import gleam/pair
 import gleam/pgo
 import gleam/result
 import gleam/string
@@ -592,3 +593,27 @@ pub fn update_package_rank(db: pgo.Connection, package: String, rank: Int) {
   |> pgo.execute(db, [pgo.text(package), pgo.int(rank)], dynamic.dynamic)
   |> result.map_error(error.DatabaseError)
 }
+
+pub fn select_package_repository_address(db: pgo.Connection, offset: Int) {
+  let decoder = dynamic.element(0, dynamic.optional(dynamic.string))
+  "SELECT repository FROM package LIMIT 100 OFFSET $1"
+  |> pgo.execute(db, [pgo.int(offset)], decoder)
+  |> result.map_error(error.DatabaseError)
+  |> result.map(fn(r) { r.rows })
+}
+
+pub fn update_package_popularity(
+  db: pgo.Connection,
+  url: String,
+  popularity: Dict(String, Int),
+) {
+  let popularity =
+    dict.to_list(popularity)
+    |> list.map(pair.map_second(_, json.int))
+    |> json.object()
+    |> json.to_string()
+    |> pgo.text()
+  "UPDATE package SET popularity = $2 WHERE repository = $1"
+  |> pgo.execute(db, [pgo.text(url), popularity], dynamic.dynamic)
+  |> result.map_error(error.DatabaseError)
+}

apps/backend/src/tasks/popularity.gleam

@@ -0,0 +1,39 @@
+import api/github
+import backend/config.{type Context}
+import backend/postgres/queries
+import gleam/bool
+import gleam/dict
+import gleam/function
+import gleam/list
+import gleam/option
+import gleam/result
+import wisp
+
+pub fn compute_popularity(ctx: Context) {
+  wisp.log_info("Syncing popularity")
+  do_compute_popularity(ctx, offset: 0)
+  |> function.tap(fn(_) { wisp.log_info("Syncing package ranks finished!") })
+}
+
+fn do_compute_popularity(ctx: Context, offset offset: Int) {
+  let db = ctx.db
+  use repos <- result.try(queries.select_package_repository_address(db, offset))
+  use <- bool.guard(when: list.is_empty(repos), return: Ok(Nil))
+  list.map(repos, fn(repo) {
+    repo
+    |> option.map(update_repo_popularity(ctx, _))
+    |> option.unwrap(Ok(Nil))
+    |> result.try_recover(fn(_) { Ok(Nil) })
+  })
+  |> result.all()
+  |> result.try(fn(_) { do_compute_popularity(ctx, offset: offset + 100) })
+}
+
+fn update_repo_popularity(ctx: Context, repo: String) {
+  wisp.log_debug("Syncing " <> repo)
+  use count <- result.try(github.get_stargazer_count(ctx.github_token, repo))
+  dict.from_list([#("github", count)])
+  |> queries.update_package_popularity(ctx.db, repo, _)
+  |> result.replace(Nil)
+  |> function.tap(fn(_) { wisp.log_debug("Synced " <> repo) })
+}

apps/frontend/manifest.toml

@@ -27,7 +27,7 @@ gleam_javascript = { version = "~> 0.8" }
 gleam_json = { version = ">= 1.0.1 and < 2.0.0" }
 gleam_stdlib = { version = "~> 0.34 or ~> 1.0" }
 gleeunit = { version = "~> 1.0" }
-grille_pain = { version = ">= 1.0.0 and < 2.0.0"}
+grille_pain = { version = ">= 1.0.0 and < 2.0.0" }
 lustre = { version = ">= 4.2.0 and < 5.0.0" }
 lustre_http = { version = "~> 0.5" }
 modem = { version = ">= 1.1.0 and < 2.0.0" }