From 1a46e28198f2275d1716196987129f6fa437b381 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Wed, 4 Sep 2024 03:13:27 +0200 Subject: [PATCH] feat: add analytics --- .../src/backend/postgres/queries.gleam | 23 +++++- apps/backend/src/backend/router.gleam | 32 ++++++-- apps/frontend/package.json | 1 + apps/frontend/src/chart.mjs | 69 ++++++++++++++++++ apps/frontend/src/data/model.gleam | 30 ++++++-- apps/frontend/src/data/msg.gleam | 2 +- apps/frontend/src/frontend.gleam | 29 +++++--- apps/frontend/src/frontend.ts | 3 + .../src/frontend/view/body/body.gleam | 73 ++++++++++++++++++- apps/frontend/src/line_chart.gleam | 16 ++++ apps/frontend/src/stylesheets/all.css | 25 ++++++- yarn.lock | 17 +++++ 12 files changed, 293 insertions(+), 27 deletions(-) create mode 100644 apps/frontend/src/chart.mjs create mode 100644 apps/frontend/src/line_chart.gleam diff --git a/apps/backend/src/backend/postgres/queries.gleam b/apps/backend/src/backend/postgres/queries.gleam index 6340d9b..b2dc20f 100644 --- a/apps/backend/src/backend/postgres/queries.gleam +++ b/apps/backend/src/backend/postgres/queries.gleam @@ -3,12 +3,12 @@ import backend/data/hex_user.{type HexUser} import backend/error import backend/gleam/context import birl.{type Time} -import birl/duration import gleam/bool import gleam/dict.{type Dict} import gleam/dynamic import gleam/hexpm import gleam/int +import gleam/io import gleam/json import gleam/list import gleam/option.{type Option, None, Some} @@ -157,6 +157,27 @@ fn get_current_package_owners(db: pgo.Connection, package_id: Int) { |> result.map_error(error.DatabaseError) } +pub fn get_total_searches(db: pgo.Connection) { + "SELECT SUM(occurences) FROM search_analytics" + |> pgo.execute(db, [], dynamic.element(0, dynamic.int)) + |> result.map(fn(r) { r.rows }) + |> result.map_error(error.DatabaseError) +} + +pub fn get_total_signatures(db: pgo.Connection) { + "SELECT COUNT(*) FROM package_type_fun_signature" + |> pgo.execute(db, [], dynamic.element(0, dynamic.int)) + |> result.map(fn(r) { r.rows }) + |> result.map_error(error.DatabaseError) +} + +pub fn get_total_packages(db: pgo.Connection) { + "SELECT COUNT(*) FROM package" + |> pgo.execute(db, [], dynamic.element(0, dynamic.int)) + |> result.map(fn(r) { r.rows }) + |> result.map_error(error.DatabaseError) +} + fn add_new_package_owners( db: pgo.Connection, owners: List(HexUser), diff --git a/apps/backend/src/backend/router.gleam b/apps/backend/src/backend/router.gleam index 73576f9..652054c 100644 --- a/apps/backend/src/backend/router.gleam +++ b/apps/backend/src/backend/router.gleam @@ -12,6 +12,7 @@ import gleam/int import gleam/json import gleam/list import gleam/option +import gleam/pair import gleam/result import gleam/string_builder import tasks/hex as syncing @@ -143,14 +144,31 @@ pub fn handle_get(req: Request, ctx: Context) { }) |> result.unwrap(wisp.internal_server_error()) ["analytics"] -> { - queries.get_timeseries_count(ctx.db) + { + use timeseries <- result.try(queries.get_timeseries_count(ctx.db)) + use total <- result.try(queries.get_total_searches(ctx.db)) + use signatures <- result.try(queries.get_total_signatures(ctx.db)) + use packages <- result.try(queries.get_total_packages(ctx.db)) + let total = list.first(total) |> result.unwrap(0) + let signatures = list.first(signatures) |> result.unwrap(0) + let packages = list.first(packages) |> result.unwrap(0) + Ok(#(timeseries, total, signatures, packages)) + } |> result.map(fn(content) { - json.array(content, fn(row) { - json.object([ - #("count", json.int(row.0)), - #("date", json.string(birl.to_iso8601(row.1))), - ]) - }) + let #(timeseries, total, signatures, packages) = content + json.object([ + #("total", json.int(total)), + #("signatures", json.int(signatures)), + #("packages", json.int(packages)), + #("timeseries", { + json.array(timeseries, fn(row) { + json.object([ + #("count", json.int(row.0)), + #("date", json.string(birl.to_iso8601(row.1))), + ]) + }) + }), + ]) |> json.to_string_builder |> wisp.json_response(200) }) diff --git a/apps/frontend/package.json b/apps/frontend/package.json index 6071282..e9c6a5f 100644 --- a/apps/frontend/package.json +++ b/apps/frontend/package.json @@ -12,6 +12,7 @@ "@chouqueth/gleam": "^1.4.1", "@gleam-lang/highlight.js-gleam": "^1.5.0", "@sentry/browser": "^8.0.0", + "chart.js": "^4.4.4", "dompurify": "^3.1.4", "highlight.js": "^11.9.0", "marked": "^12.0.2", diff --git a/apps/frontend/src/chart.mjs b/apps/frontend/src/chart.mjs new file mode 100644 index 0000000..01d1f6d --- /dev/null +++ b/apps/frontend/src/chart.mjs @@ -0,0 +1,69 @@ +import { Chart } from 'chart.js/auto' + +export class LineChart extends HTMLElement { + static observedAttributes = ['datasets'] + + #shadow + #canvas + dataset + + constructor() { + super() + this.#shadow = this.attachShadow({ mode: 'open' }) + } + + connectedCallback() { + this.#render() + } + + #render() { + const labels = this.datasets.labels.toArray() + const data = this.datasets.data.toArray() + const wrapper = document.createElement('div') + wrapper.style.position = 'relative' + // wrapper.style.maxWidth = '400px' + // wrapper.style.maxHeight = '150px' + this.#canvas = document.createElement('canvas') + wrapper.appendChild(this.#canvas) + this.#shadow.appendChild(wrapper) + + new Chart(this.#canvas, { + type: 'line', + data: { + labels, + datasets: [ + { + data, + borderColor: '#8c3a96', + fill: false, + tension: 0.4, + }, + ], + }, + options: { + responsive: true, + animation: false, + events: [], + plugins: { + legend: { + display: false, + }, + }, + scales: { + x: { display: true, title: { display: true } }, + y: { display: true, title: { display: true, text: 'Value' } }, + }, + }, + }) + } + + // Lifecycle functions. + disconnectedCallback() {} + adoptedCallback() {} + + attributeChangedCallback() {} + + static register() { + customElements.define('line-chart', LineChart) + } +} diff --git a/apps/frontend/src/data/model.gleam b/apps/frontend/src/data/model.gleam index 76656a7..65cbdda 100644 --- a/apps/frontend/src/data/model.gleam +++ b/apps/frontend/src/data/model.gleam @@ -37,7 +37,10 @@ pub type Model { show_old_packages: Bool, show_documentation_search: Bool, show_vector_search: Bool, - analytics: List(#(Int, birl.Time)), + total_searches: Int, + total_signatures: Int, + total_packages: Int, + timeseries: List(#(Int, birl.Time)), ) } @@ -64,7 +67,10 @@ pub fn init() { show_old_packages: False, show_documentation_search: False, show_vector_search: False, - analytics: [], + total_searches: 0, + total_signatures: 0, + total_packages: 0, + timeseries: [], ) } @@ -96,8 +102,19 @@ pub fn update_input(model: Model, content: String) { Model(..model, input: content) } -pub fn update_analytics(model: Model, analytics: List(#(Int, birl.Time))) { - Model(..model, analytics: analytics) +pub fn update_analytics( + model: Model, + analytics: #(Int, Int, Int, List(#(Int, birl.Time))), +) { + let #(total_searches, total_signatures, total_packages, timeseries) = + analytics + Model( + ..model, + timeseries:, + total_searches:, + total_signatures:, + total_packages:, + ) } pub fn search_key(key key: String, model model: Model) { @@ -331,7 +348,10 @@ pub fn reset(model: Model) { show_old_packages: False, show_documentation_search: False, show_vector_search: False, - analytics: [], + timeseries: [], + total_searches: 0, + total_signatures: 0, + total_packages: 0, ) } diff --git a/apps/frontend/src/data/msg.gleam b/apps/frontend/src/data/msg.gleam index 1a2c93f..0918e00 100644 --- a/apps/frontend/src/data/msg.gleam +++ b/apps/frontend/src/data/msg.gleam @@ -26,7 +26,7 @@ pub type Msg { Reset ScrollTo(String) OnEscape - Analytics(Result(List(#(Int, birl.Time)), http.HttpError)) + Analytics(Result(#(Int, Int, Int, List(#(Int, birl.Time))), http.HttpError)) OnRouteChange(router.Route) OnCheckFilter(Filter, Bool) } diff --git a/apps/frontend/src/frontend.gleam b/apps/frontend/src/frontend.gleam index d67d7dc..2e1e5cd 100644 --- a/apps/frontend/src/frontend.gleam +++ b/apps/frontend/src/frontend.gleam @@ -12,7 +12,6 @@ import frontend/view/body/search_result as sr import gleam/bool import gleam/dict import gleam/dynamic -import gleam/io import gleam/option.{None, Some} import gleam/pair import gleam/result @@ -93,17 +92,25 @@ fn init(_) { |> update.add_effect( msg.Analytics |> http.expect_json( - dynamic.list(dynamic.decode2( - pair.new, - dynamic.field("count", dynamic.int), - dynamic.field("date", fn(dyn) { - dynamic.string(dyn) - |> result.then(fn(t) { - birl.parse(t) - |> result.replace_error([]) - }) + dynamic.decode4( + fn(a, b, c, d) { #(a, b, c, d) }, + dynamic.field("total", dynamic.int), + dynamic.field("signatures", dynamic.int), + dynamic.field("packages", dynamic.int), + dynamic.field("timeseries", { + dynamic.list(dynamic.decode2( + pair.new, + dynamic.field("count", dynamic.int), + dynamic.field("date", fn(dyn) { + dynamic.string(dyn) + |> result.then(fn(t) { + birl.parse(t) + |> result.replace_error([]) + }) + }), + )) }), - )), + ), _, ) |> http.get(config.api_endpoint() <> "/analytics", _), diff --git a/apps/frontend/src/frontend.ts b/apps/frontend/src/frontend.ts index 4501e9f..0145d33 100644 --- a/apps/frontend/src/frontend.ts +++ b/apps/frontend/src/frontend.ts @@ -3,12 +3,15 @@ import gleamHljs from '@gleam-lang/highlight.js-gleam' import hljs from 'highlight.js/lib/core' import plaintext from 'highlight.js/lib/languages/plaintext' // @ts-ignore +import { LineChart } from './chart.mjs' +// @ts-ignore import { main } from './frontend.gleam' import './stylesheets/all.css' import './stylesheets/hljs-theme.css' import './stylesheets/main.css' import './stylesheets/normalize.css' +LineChart.register() // @ts-ignore Element.prototype._attachShadow = Element.prototype.attachShadow Element.prototype.attachShadow = function () { diff --git a/apps/frontend/src/frontend/view/body/body.gleam b/apps/frontend/src/frontend/view/body/body.gleam index c971d43..4dd38a1 100644 --- a/apps/frontend/src/frontend/view/body/body.gleam +++ b/apps/frontend/src/frontend/view/body/body.gleam @@ -1,3 +1,4 @@ +import birl import data/model.{type Model} import data/msg import data/search_result @@ -8,8 +9,13 @@ import frontend/strings as frontend_strings import frontend/view/search_input/search_input import gleam/bool import gleam/dict +import gleam/float +import gleam/int +import gleam/io +import gleam/list import gleam/result import gleam/string +import line_chart import lustre/attribute as a import lustre/element as el import lustre/element/html as h @@ -220,12 +226,77 @@ fn sidebar_link(href href: String, title title: String, icon icon) { ]) } +fn format_huge_number(number: Int) { + let number = int.to_float(number) + let g = number /. 1_000_000_000.0 + let m = number /. 1_000_000.0 + let k = number /. 1000.0 + case number { + _ if g >. 1.0 -> float.to_string(g) |> string.slice(0, 5) <> " G" + _ if m >. 1.0 -> float.to_string(m) |> string.slice(0, 5) <> " M" + _ if k >. 1.0 -> float.to_string(k) |> string.slice(0, 5) <> " K" + _ -> float.round(number) |> int.to_string + } +} + pub fn body(model: Model) { case model.route { router.Home -> h.main([a.class("main")], [view_search_input(model)]) router.Trending -> h.main([a.class("main")], [view_trending(model)]) router.Analytics -> - el.fragment([sidebar(model), h.main([a.class("main")], [])]) + el.fragment([ + sidebar(model), + h.main([a.class("main")], [ + h.div( + [a.class("items-wrapper"), a.style([#("padding-left", "24px")])], + [ + h.div([a.class("matches-titles")], [ + h.div([a.class("matches-title")], [h.text("Global analytics")]), + ]), + h.div([a.class("analytics-box-wrapper")], [ + h.div([a.class("analytics-box")], [ + h.div([a.class("analytics-title")], [ + h.text("Number of searches"), + ]), + h.text(format_huge_number(model.total_searches)), + ]), + h.div([a.class("analytics-box")], [ + h.div([a.class("analytics-title")], [ + h.text("Number of signatures indexed"), + ]), + h.text(format_huge_number(model.total_signatures)), + ]), + h.div([a.class("analytics-box")], [ + h.div([a.class("analytics-title")], [ + h.text("Number of packages indexed"), + ]), + h.text(format_huge_number(model.total_packages)), + ]), + ]), + h.div([a.class("matches-titles")], [ + h.div([a.class("matches-title")], [h.text("Last 30 days")]), + ]), + h.div([a.style([#("width", "auto"), #("height", "500px")])], [ + case model.timeseries { + [] -> el.none() + data -> { + line_chart.line_chart({ + use line_chart.Dataset(dates, value), #(count, date) <- list.fold( + data, + line_chart.Dataset([], []), + ) + line_chart.Dataset([birl.to_iso8601(date), ..dates], [ + count, + ..value + ]) + }) + } + }, + ]), + ], + ), + ]), + ]) router.Search(_) -> { let key = model.search_key(model.submitted_input, model) el.fragment([ diff --git a/apps/frontend/src/line_chart.gleam b/apps/frontend/src/line_chart.gleam new file mode 100644 index 0000000..b2943b3 --- /dev/null +++ b/apps/frontend/src/line_chart.gleam @@ -0,0 +1,16 @@ +import gleam/string +import lustre/attribute +import lustre/element + +pub type Dataset { + Dataset(labels: List(String), data: List(Int)) +} + +pub fn line_chart(datasets: Dataset) { + let datasets = attribute.property("datasets", datasets) + element.element( + "line-chart", + [attribute.style([#("display", "block")]), datasets], + [], + ) +} diff --git a/apps/frontend/src/stylesheets/all.css b/apps/frontend/src/stylesheets/all.css index 1fd3415..10d2d80 100644 --- a/apps/frontend/src/stylesheets/all.css +++ b/apps/frontend/src/stylesheets/all.css @@ -447,7 +447,7 @@ lazy-node:has(:not(:defined)) { .sidebar-links { display: flex; flex-direction: column; - gap: 12px; + gap: 24px; padding: 12px; } @@ -553,3 +553,26 @@ lazy-node:has(:not(:defined)) { line-height: 1.6; font-weight: 400; } + +.analytics-box { + display: flex; + flex-direction: column; + gap: 24px; + padding: 24px; + border: 1px solid var(--border-color); + border-radius: 10px; + font-size: 2.5rem; + line-height: 1.75; + width: 200px; + /* height: 200px; */ + justify-content: space-between; +} + +.analytics-title { + font-size: 1rem; +} + +.analytics-box-wrapper { + display: flex; + gap: 24px; +} diff --git a/yarn.lock b/yarn.lock index 6950440..7c816ea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -880,6 +880,13 @@ __metadata: languageName: node linkType: hard +"@kurkle/color@npm:^0.3.0": + version: 0.3.2 + resolution: "@kurkle/color@npm:0.3.2" + checksum: 10c0/a9e8e3e35dcd59dec4dd4f0105919c05e24823a96347bcf152965c29e48d6290b66d5fb96c071875db752e10930724c48ce6d338fefbd65e0ce5082d5c78970e + languageName: node + linkType: hard + "@npmcli/agent@npm:^2.0.0": version: 2.2.2 resolution: "@npmcli/agent@npm:2.2.2" @@ -1643,6 +1650,15 @@ __metadata: languageName: node linkType: hard +"chart.js@npm:^4.4.4": + version: 4.4.4 + resolution: "chart.js@npm:4.4.4" + dependencies: + "@kurkle/color": "npm:^0.3.0" + checksum: 10c0/9fa3206403a6103916f7762c2665d322c42b0cc07fba91526b1d033ddb887c1ba74b3ebc0bd0748a9e55abd1017f25fdb2292cdd6579d8c2d3bcb1c58f71281c + languageName: node + linkType: hard + "check-error@npm:^1.0.3": version: 1.0.3 resolution: "check-error@npm:1.0.3" @@ -2471,6 +2487,7 @@ __metadata: "@sentry/vite-plugin": "npm:^2.16.1" "@trivago/prettier-plugin-sort-imports": "npm:^4.3.0" "@types/dompurify": "npm:^3.0.5" + chart.js: "npm:^4.4.4" dompurify: "npm:^3.1.4" dotenv: "npm:^16.4.5" highlight.js: "npm:^11.9.0"