tweak(docs,troubleshooting,db): add validation

.github/workflows/docs-sync-troubleshooting.yml

@@ -0,0 +1,51 @@
+name: '[Docs] Sync troubleshooting guides to GitHub Discussions'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'apps/docs/content/troubleshooting/**'
+  workflow_dispatch:
+  # testing only, delete later
+  # pull_request:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-troubleshooting:
+    runs-on: ubuntu-latest
+
+    env:
+      NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.SEARCH_SUPABASE_URL }}
+      SUPABASE_SECRET_KEY: ${{ secrets.SEARCH_SUPABASE_SERVICE_ROLE_KEY }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            apps/docs
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run Troubleshooting script
+        run: npm --prefix=apps/docs run troubleshooting:sync
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
+        with:
+          # for testing on pull_request only
+          # base: ${{ github.head_ref }}
+          commit-message: '[bot] sync troubleshooting guides to db'
+          title: '[bot] sync troubleshooting guides to db'
+          author: '[github-docs-sync-bot]'
+          branch: 'bot/docs-sync-troubleshooting'

apps/docs/features/docs/Troubleshooting.page.tsx

@@ -4,18 +4,22 @@ import Breadcrumbs from '~/components/Breadcrumbs'
 import { Feedback } from '~/components/Feedback'
 import { SidebarSkeleton } from '~/layouts/MainSkeleton'
 import { MDXRemoteBase } from './MdxBase'
-import { type ITroubleshootingEntry } from './Troubleshooting.utils'
+import { getTroubleshootingUpdatedDates, type ITroubleshootingEntry } from './Troubleshooting.utils'
 import Link from 'next/link'
 import { formatError, serializeTroubleshootingSearchParams } from './Troubleshooting.utils.shared'
 
-export default function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) {
+export default async function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) {
+  const dateUpdated = entry.data.database_id.startsWith('pseudo-')
+    ? new Date()
+    : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id)
+
   return (
     <SidebarSkeleton className="@container/troubleshooting-entry-layout w-full max-w-screen-lg mx-auto lg:py-8 lg:px-5">
       <div className="px-5 py-8 lg:px-0 lg:py-0">
         <Breadcrumbs minLength={1} forceDisplayOnMobile />
         <article className="prose max-w-none mt-4">
           <h1>{entry.data.title}</h1>
-          {entry.data.updated_at && <p>Last edited: {entry.data.updated_at.toLocaleString()}</p>}
+          {dateUpdated && <p>Last edited: {dateUpdated.toLocaleString()}</p>}
           <hr className="my-7" aria-hidden />
           <div className="grid gap-10 @3xl/troubleshooting-entry-layout:grid-cols-[1fr,250px]">
             <div>

apps/docs/features/docs/Troubleshooting.script.ts

@@ -0,0 +1,165 @@
+/**
+ * Sync new troubleshooting entries from the GitHub repo with GitHub
+ * Discussions.
+ */
+
+import { createHash } from 'crypto'
+import matter from 'gray-matter'
+import { fromMarkdown } from 'mdast-util-from-markdown'
+import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm'
+import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx'
+import { toMarkdown } from 'mdast-util-to-markdown'
+import { gfm } from 'micromark-extension-gfm'
+import { mdxjs } from 'micromark-extension-mdxjs'
+import { readFile, writeFile } from 'node:fs/promises'
+import { stringify } from 'smol-toml'
+import toml from 'toml'
+
+import { octokit } from '~/lib/octokit'
+import { supabaseAdmin } from '~/lib/supabaseAdmin'
+import {
+  getAllTroubleshootingEntries,
+  getArticleSlug,
+  type ITroubleshootingEntry,
+} from './Troubleshooting.utils'
+
+async function syncTroubleshootingEntries() {
+  const troubleshootingEntries = await getAllTroubleshootingEntries()
+
+  const tasks = troubleshootingEntries.map(async (entry) => {
+    const databaseId = entry.data.database_id
+    if (databaseId.startsWith('pseudo-')) {
+      // The database entry is faked, so we need to insert a new entry.
+      const githubUrl = entry.data.github_url ?? (await createGithubDiscussion(entry))
+      const id = await insertNewTroubleshootingEntry(entry, githubUrl)
+      await updateFileId(entry, id)
+    } else {
+      // The database entry already exists, so check for updates.
+      await updateChecksumIfNeeded(entry)
+    }
+  })
+
+  const results = await Promise.allSettled(tasks)
+  let hasErrors = false
+  results.forEach((result, index) => {
+    if (result.status === 'rejected') {
+      console.error(`Failed to insert GitHub discussion for ${index}:`, result.reason)
+      hasErrors = true
+    }
+  })
+
+  return hasErrors
+}
+
+function calculateChecksum(content: string) {
+  // Normalize to ignore changes that don't affect the final displayed content.
+  const mdast = fromMarkdown(content, {
+    extensions: [gfm(), mdxjs()],
+    mdastExtensions: [gfmFromMarkdown(), mdxFromMarkdown()],
+  })
+  const normalized = toMarkdown(mdast, { extensions: [gfmToMarkdown(), mdxToMarkdown()] })
+
+  return createHash('sha256').update(normalized).digest('base64')
+}
+
+async function insertNewTroubleshootingEntry(entry: ITroubleshootingEntry, githubUrl: string) {
+  const timestamp = Date.now()
+  const checksum = calculateChecksum(entry.content)
+
+  const { data, error } = await supabaseAdmin()
+    .from('troubleshooting_entries')
+    .insert({
+      ...entry.data,
+      // @ts-ignore
+      checksum,
+      github_url: githubUrl,
+      date_created: entry.data.date_created ?? timestamp,
+      date_updated: timestamp,
+    })
+    .select('id')
+    .single()
+  if (error) {
+    throw error
+  }
+
+  return data.id
+}
+
+async function updateChecksumIfNeeded(entry: ITroubleshootingEntry) {
+  const { data, error } = await supabaseAdmin()
+    .from('troubleshooting_entries')
+    .select('checksum')
+    .eq('id', entry.data.database_id)
+    .single()
+  if (error) {
+    throw error
+  }
+
+  if (data.checksum !== calculateChecksum(entry.content)) {
+    const timestamp = new Date().toISOString()
+    const { error } = await supabaseAdmin()
+      .from('troubleshooting_entries')
+      .update({
+        checksum: calculateChecksum(entry.content),
+        date_updated: timestamp,
+      })
+      .eq('id', entry.data.database_id)
+
+    if (error) {
+      throw error
+    }
+  }
+}
+
+async function createGithubDiscussion(entry: ITroubleshootingEntry) {
+  const docsUrl = 'https://supabase.com/docs/guides/troubleshooting/' + getArticleSlug(entry.data)
+  const content =
+    entry.content +
+    `\n\n_This is a copy of a troubleshooting article on Supabase's docs site. You can find the original [here](${docsUrl})._`
+
+  const mutation = `
+    mutation {
+      createDiscussion(input: {
+        repositoryId: "MDEwOlJlcG9zaXRvcnkyMTQ1ODcxOTM=",
+        categoryId: "DIC_kwDODMpXOc4CUvEr",
+        body: "${content}",
+        title: "${entry.data.title}"
+      }) {
+        discussion {
+          url
+        }
+      }
+    }
+    `
+
+  const { discussion } = await octokit().graphql<{ discussion: { url: string } }>(mutation)
+  return discussion.url
+}
+
+async function updateFileId(entry: ITroubleshootingEntry, id: string) {
+  const fileContents = await readFile(entry.filePath, 'utf-8')
+  const { data, content } = matter(fileContents, {
+    language: 'toml',
+    engine: toml.parse.bind(toml),
+  })
+  data.database_id = id
+
+  const newFrontmatter = stringify(data)
+  const newContent = `---\n${newFrontmatter}\n---\n\n${content}`
+
+  await writeFile(entry.filePath, newContent)
+}
+
+async function main() {
+  try {
+    const hasErrors = await syncTroubleshootingEntries()
+    if (hasErrors) {
+      process.exit(1)
+    }
+  } catch (error) {
+    console.error(error)
+    process.exit(1)
+  }
+}
+
+main()

apps/docs/features/docs/Troubleshooting.ui.tsx

@@ -2,12 +2,20 @@ import { Wrench } from 'lucide-react'
 import Link from 'next/link'
 import { type PropsWithChildren, useCallback } from 'react'
 
-import { type ITroubleshootingEntry, getArticleSlug } from './Troubleshooting.utils'
+import {
+  type ITroubleshootingEntry,
+  getArticleSlug,
+  getTroubleshootingUpdatedDates,
+} from './Troubleshooting.utils'
 import { TroubleshootingFilter } from './Troubleshooting.ui.client'
 import { formatError, TROUBLESHOOTING_DATA_ATTRIBUTES } from './Troubleshooting.utils.shared'
 import { cn } from 'ui'
 
-export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) {
+export async function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) {
+  const dateUpdated = entry.data.database_id.startsWith('pseudo-')
+    ? new Date()
+    : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id)
+
   const keywords = [...entry.data.topics, ...(entry.data.keywords ?? [])]
   const attributes = {
     [TROUBLESHOOTING_DATA_ATTRIBUTES.QUERY_ATTRIBUTE]:
@@ -58,14 +66,13 @@ export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry
             ))}
           </div>
           <div className="basis-l8 flex-shrink-0 flex-grow-0 truncate text-sm text-foreground-lighter">
-            {entry.data.updated_at &&
+            {dateUpdated &&
               (() => {
-                const date = new Date(entry.data.updated_at)
                 const options = { month: 'short', day: 'numeric' } as Intl.DateTimeFormatOptions
-                if (date.getFullYear() !== new Date().getFullYear()) {
+                if (dateUpdated.getFullYear() !== new Date().getFullYear()) {
                   options.year = 'numeric'
                 }
-                return date.toLocaleDateString(undefined, options)
+                return dateUpdated.toLocaleDateString(undefined, options)
               })()}
           </div>
         </div>

apps/docs/features/docs/Troubleshooting.utils.ts

@@ -11,6 +11,7 @@ import { v4 as uuidv4 } from 'uuid'
 import { z } from 'zod'
 
 import { DOCS_DIRECTORY } from 'lib/docs'
+import { supabaseAdmin } from '~/lib/supabaseAdmin'
 import { cache_fullProcess_withDevCacheBust } from '~/features/helpers.fs'
 import { formatError } from './Troubleshooting.utils.shared'
 
@@ -56,9 +57,9 @@ const TroubleshootingSchema = z
           .strict()
       )
       .optional(),
-    database_id: z.string().uuid().default(uuidv4()),
-    created_at: z.date({ coerce: true }).optional(),
-    updated_at: z.date({ coerce: true }).optional(),
+    database_id: z.string().default(`pseudo-${uuidv4()}`),
+    github_url: z.string().url().optional(),
+    date_created: z.date({ coerce: true }).optional(),
   })
   .strict()
 
@@ -76,10 +77,12 @@ async function getAllTroubleshootingEntriesInternal() {
     const isHidden = entry.startsWith('_')
     if (isHidden) return null
 
-    const isFile = (await stat(join(TROUBLESHOOTING_DIRECTORY, entry))).isFile()
+    const filePath = join(TROUBLESHOOTING_DIRECTORY, entry)
+
+    const isFile = (await stat(filePath)).isFile()
     if (!isFile) return null
 
-    const fileContents = await readFile(join(TROUBLESHOOTING_DIRECTORY, entry), 'utf-8')
+    const fileContents = await readFile(filePath, 'utf-8')
     const { content, data: frontmatter } = matter(fileContents, {
       language: 'toml',
       engines: { toml: toml.parse.bind(toml) },
@@ -117,6 +120,7 @@ async function getAllTroubleshootingEntriesInternal() {
     const contentWithoutJsx = toMarkdown(mdxTree)
 
     return {
+      filePath,
       content,
       contentWithoutJsx,
       data: parseResult.data,
@@ -190,3 +194,27 @@ export function getArticleSlug(entry: ITroubleshootingMetadata) {
   const escapedTitle = encodeURIComponent(slugifiedTitle)
   return escapedTitle
 }
+
+async function getTroubleshootingUpdatedDatesInternal() {
+  const databaseIds = (await getAllTroubleshootingEntries())
+    .map((entry) => entry.data.database_id)
+    .filter((id) => !id.startsWith('pseudo-'))
+
+  const { data, error } = await supabaseAdmin()
+    .from('troubleshooting_entries')
+    .select('id, date_updated')
+    .in('id', databaseIds)
+  if (error) {
+    console.error(error)
+  }
+
+  return (data ?? []).reduce((acc, entry) => {
+    acc.set(entry.id, new Date(entry.date_updated))
+    return acc
+  }, new Map<string, Date>())
+}
+export const getTroubleshootingUpdatedDates = cache_fullProcess_withDevCacheBust(
+  getTroubleshootingUpdatedDatesInternal,
+  TROUBLESHOOTING_DIRECTORY,
+  () => JSON.stringify([])
+)

apps/docs/lib/octokit.ts

@@ -8,7 +8,7 @@ import { fetchRevalidatePerDay } from '~/features/helpers.fetch'
 
 let octokitInstance: Octokit
 
-function octokit() {
+export function octokit() {
   if (!octokitInstance) {
     const privateKeyPkcs8 = crypto
       .createPrivateKey(process.env.DOCS_GITHUB_APP_PRIVATE_KEY)

apps/docs/lib/supabaseAdmin.ts

@@ -0,0 +1,18 @@
+import 'server-only'
+
+import { createClient, type SupabaseClient } from '@supabase/supabase-js'
+
+import { type Database } from 'common'
+
+let supabaseAdminClient: SupabaseClient<Database> | null = null
+
+export function supabaseAdmin() {
+  if (!supabaseAdminClient) {
+    supabaseAdminClient = createClient(
+      process.env.NEXT_PUBLIC_SUPABASE_URL,
+      process.env.SUPABASE_SECRET_KEY
+    )
+  }
+
+  return supabaseAdminClient
+}