feat: track links in a json column

Author: DanielFGray

app/db.d.ts

@@ -17,6 +17,7 @@ export interface MessageStats {
   char_count: number;
   code_stats: Generated<string>;
   guild_id: string;
+  link_stats: Generated<string>;
   message_id: string | null;
   react_count: Generated<number>;
   recipient_id: string | null;

app/discord/activityTracker.ts

@@ -88,12 +88,24 @@ async function getMessageStats(msg: Message | PartialMessage) {
 
   const blocks = parseMarkdownBlocks(content);
 
-  const [textblocks, codeblocks] = partition(blocks, (b) => b.type === "text");
+  // TODO: groupBy would be better here, but this was easier to keep typesafe
+  const [textblocks, nontextblocks] = partition(
+    blocks,
+    (b) => b.type === "text",
+  );
+  const [links, codeblocks] = partition(
+    nontextblocks,
+    (b) => b.type === "link",
+  );
+
+  const linkStats = links.map((link) => ({ url: link.url }));
 
-  const { wordCount, charCount } = textblocks.reduce(
+  const { wordCount, charCount } = [...links, ...textblocks].reduce(
     (acc, block) => {
-      const words = getWords(block.content).length;
-      const chars = getChars(block.content).length;
+      const content =
+        block.type === "link" ? (block.label ?? "") : block.content;
+      const words = getWords(content).length;
+      const chars = getChars(content).length;
       return {
         wordCount: acc.wordCount + words,
         charCount: acc.charCount + chars,
@@ -128,6 +140,7 @@ async function getMessageStats(msg: Message | PartialMessage) {
     char_count: charCount,
     word_count: wordCount,
     code_stats: JSON.stringify(codeStats),
+    link_stats: JSON.stringify(linkStats),
     react_count: msg.reactions.cache.size,
     sent_at: msg.createdTimestamp,
   };

app/helpers/messageParsing.test.ts

@@ -1,6 +1,39 @@
 import { parseMarkdownBlocks } from "./messageParsing";
 
 describe("markdown parser", () => {
+  test("matches bare link", () => {
+    const message = `bold claim (source https://trustme.bro)`;
+    const result = parseMarkdownBlocks(message);
+    expect(result[1]).toEqual({ type: "link", url: "https://trustme.bro" });
+  });
+
+  test("matches link with label", () => {
+    const message = `check out this [link](<https://example.com>)`;
+    const result = parseMarkdownBlocks(message);
+    expect(result[1]).toEqual({
+      type: "link",
+      url: "https://example.com",
+      label: "link",
+    });
+  });
+
+  test("matches many links", () => {
+    const message = `bare link https://asdf.com
+(see also https://links.com)
+* [*bold*](<https://foo.xyz>)
+* [*bold*](<https://bar.xyz>)
+words and things [\`foo()\`](<https://links.xom>) asdfasdf`;
+    const result = parseMarkdownBlocks(message);
+    const links = result.filter((x) => x.type === "link").map((x) => x.url);
+    expect(links).toEqual([
+      "https://asdf.com",
+      "https://links.com",
+      "https://foo.xyz",
+      "https://bar.xyz",
+      "https://links.xom",
+    ]);
+  });
+
   test("matches fenced code blocks and inline text", () => {
     const message = `here is some text
 

app/helpers/messageParsing.ts

@@ -1,5 +1,6 @@
 export type MarkdownBlock =
   | { type: "text"; content: string }
+  | { type: "link"; url: string; label?: string }
   | { type: "fencedcode"; lang: undefined | string; code: string[] }
   | { type: "inlinecode"; code: string };
 
@@ -11,6 +12,8 @@ export function parseMarkdownBlocks(markdown: string): MarkdownBlock[] {
   const matchers = {
     fencedCode: /```[\s\S]+?\n^```/,
     inlineCode: /`.+?`/,
+    mdlink: /\[([^\]]+)\]\(([^)]+)\)/,
+    link: /https?:\/\/[^\s>)]+/,
   };
 
   // replaceAll gives easy access to the position of the match
@@ -37,6 +40,13 @@ export function parseMarkdownBlocks(markdown: string): MarkdownBlock[] {
         // match is inline code, return a string without backticks
         const code = match.slice(1, -1);
         blocks.push({ type: "inlinecode", code });
+      } else if (match.startsWith("[") || match.startsWith("http")) {
+        // match is a link
+        const label = captured[0] || undefined;
+        let url = captured[1] || match;
+        // links in discord may have angle brackets around them to suppress previews
+        if (url.startsWith("<") && url.endsWith(">")) url = url.slice(1, -1);
+        blocks.push({ type: "link", url, label });
       } else {
         console.error("unknown match", match);
         throw new Error("Unexpected match in markdown parsing");

migrations/20250612220730_message_links.ts

@@ -0,0 +1,14 @@
+import type { Kysely } from "kysely";
+
+const column = "link_stats";
+
+export async function up(db: Kysely<any>) {
+  return await db.schema
+    .alterTable("message_stats")
+    .addColumn(column, "text", (c) => c.notNull().defaultTo("[]"))
+    .execute();
+}
+
+export async function down(db: Kysely<any>) {
+  return db.schema.alterTable("message_stats").dropColumn(column).execute();
+}