From aaf5894b642773365abf43e73bb20f5982f68cde Mon Sep 17 00:00:00 2001 From: Charis Lam <26616127+charislam@users.noreply.github.com> Date: Thu, 12 Sep 2024 18:09:17 -0400 Subject: [PATCH 1/3] feat(docs,db): sync troubleshooting entries Sync new troubleshooting entries created in the GitHub repo by creating a GitHub discussion for them and persisting the content and metadata to the database. Add index on github_url column, and validation on the shapes of the api and errors columns. Uses pg_jsonschema extension. --- .../features/docs/Troubleshooting.page.tsx | 10 +- .../features/docs/Troubleshooting.script.ts | 165 ++++++++++++++++++ .../docs/features/docs/Troubleshooting.ui.tsx | 19 +- .../features/docs/Troubleshooting.utils.ts | 37 +++- apps/docs/lib/octokit.ts | 2 +- apps/docs/lib/supabaseAdmin.ts | 18 ++ apps/docs/package.json | 1 + package-lock.json | 13 ++ packages/common/database-types.ts | 36 ++++ ...41002215612_troubleshooting_validation.sql | 71 ++++++++ 10 files changed, 357 insertions(+), 15 deletions(-) create mode 100644 apps/docs/features/docs/Troubleshooting.script.ts create mode 100644 apps/docs/lib/supabaseAdmin.ts create mode 100644 supabase/migrations/20241002215612_troubleshooting_validation.sql diff --git a/apps/docs/features/docs/Troubleshooting.page.tsx b/apps/docs/features/docs/Troubleshooting.page.tsx index a7032716e1a23..1b5be5ed02be7 100644 --- a/apps/docs/features/docs/Troubleshooting.page.tsx +++ b/apps/docs/features/docs/Troubleshooting.page.tsx @@ -4,18 +4,22 @@ import Breadcrumbs from '~/components/Breadcrumbs' import { Feedback } from '~/components/Feedback' import { SidebarSkeleton } from '~/layouts/MainSkeleton' import { MDXRemoteBase } from './MdxBase' -import { type ITroubleshootingEntry } from './Troubleshooting.utils' +import { getTroubleshootingUpdatedDates, type ITroubleshootingEntry } from './Troubleshooting.utils' import Link from 'next/link' import { formatError, serializeTroubleshootingSearchParams } from './Troubleshooting.utils.shared' -export default function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) { +export default async function TroubleshootingPage({ entry }: { entry: ITroubleshootingEntry }) { + const dateUpdated = entry.data.database_id.startsWith('pseudo-') + ? new Date() + : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id) + return (

{entry.data.title}

- {entry.data.updated_at &&

Last edited: {entry.data.updated_at.toLocaleString()}

} + {dateUpdated &&

Last edited: {dateUpdated.toLocaleString()}

}
diff --git a/apps/docs/features/docs/Troubleshooting.script.ts b/apps/docs/features/docs/Troubleshooting.script.ts new file mode 100644 index 0000000000000..f1b5162af598b --- /dev/null +++ b/apps/docs/features/docs/Troubleshooting.script.ts @@ -0,0 +1,165 @@ +/** + * Sync new troubleshooting entries from the GitHub repo with GitHub + * Discussions. + */ + +import { createHash } from 'crypto' +import matter from 'gray-matter' +import { fromMarkdown } from 'mdast-util-from-markdown' +import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm' +import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx' +import { toMarkdown } from 'mdast-util-to-markdown' +import { gfm } from 'micromark-extension-gfm' +import { mdxjs } from 'micromark-extension-mdxjs' +import { readFile, writeFile } from 'node:fs/promises' +import { stringify } from 'smol-toml' +import toml from 'toml' + +import { octokit } from '~/lib/octokit' +import { supabaseAdmin } from '~/lib/supabaseAdmin' +import { + getAllTroubleshootingEntries, + getArticleSlug, + type ITroubleshootingEntry, +} from './Troubleshooting.utils' + +async function syncTroubleshootingEntries() { + const troubleshootingEntries = await getAllTroubleshootingEntries() + + const tasks = troubleshootingEntries.map(async (entry) => { + const databaseId = entry.data.database_id + if (databaseId.startsWith('pseudo-')) { + // The database entry is faked, so we need to insert a new entry. + const githubUrl = entry.data.github_url ?? (await createGithubDiscussion(entry)) + const id = await insertNewTroubleshootingEntry(entry, githubUrl) + await updateFileId(entry, id) + } else { + // The database entry already exists, so check for updates. + await updateChecksumIfNeeded(entry) + } + }) + + const results = await Promise.allSettled(tasks) + let hasErrors = false + results.forEach((result, index) => { + if (result.status === 'rejected') { + console.error(`Failed to insert GitHub discussion for ${index}:`, result.reason) + hasErrors = true + } + }) + + return hasErrors +} + +function calculateChecksum(content: string) { + // Normalize to ignore changes that don't affect the final displayed content. + const mdast = fromMarkdown(content, { + extensions: [gfm(), mdxjs()], + mdastExtensions: [gfmFromMarkdown(), mdxFromMarkdown()], + }) + const normalized = toMarkdown(mdast, { extensions: [gfmToMarkdown(), mdxToMarkdown()] }) + + return createHash('sha256').update(normalized).digest('base64') +} + +async function insertNewTroubleshootingEntry(entry: ITroubleshootingEntry, githubUrl: string) { + const timestamp = Date.now() + const checksum = calculateChecksum(entry.content) + + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .insert({ + ...entry.data, + // @ts-ignore + checksum, + github_url: githubUrl, + date_created: timestamp, + date_updated: timestamp, + }) + .select('id') + .single() + if (error) { + throw error + } + + return data.id +} + +async function updateChecksumIfNeeded(entry: ITroubleshootingEntry) { + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('checksum') + .eq('id', entry.data.database_id) + .single() + if (error) { + throw error + } + + if (data.checksum !== calculateChecksum(entry.content)) { + const timestamp = new Date().toISOString() + const { error } = await supabaseAdmin() + .from('troubleshooting_entries') + .update({ + checksum: calculateChecksum(entry.content), + date_updated: timestamp, + }) + .eq('id', entry.data.database_id) + + if (error) { + throw error + } + } +} + +async function createGithubDiscussion(entry: ITroubleshootingEntry) { + const docsUrl = 'https://supabase.com/docs/guides/troubleshooting/' + getArticleSlug(entry.data) + const content = + entry.content + + `\n\n_This is a copy of a troubleshooting article on Supabase's docs site. You can find the original [here](${docsUrl})._` + + const mutation = ` + mutation { + createDiscussion(input: { + repositoryId: "MDEwOlJlcG9zaXRvcnkyMTQ1ODcxOTM=", + categoryId: "DIC_kwDODMpXOc4CUvEr", + body: "${content}", + title: "${entry.data.title}" + }) { + discussion { + url + } + } + } + ` + + const { discussion } = await octokit().graphql<{ discussion: { url: string } }>(mutation) + return discussion.url +} + +async function updateFileId(entry: ITroubleshootingEntry, id: string) { + const fileContents = await readFile(entry.filePath, 'utf-8') + const { data, content } = matter(fileContents, { + language: 'toml', + engine: toml.parse.bind(toml), + }) + data.database_id = id + + const newFrontmatter = stringify(data) + const newContent = `---\n${newFrontmatter}\n---\n\n${content}` + + await writeFile(entry.filePath, newContent) +} + +async function main() { + try { + const hasErrors = await syncTroubleshootingEntries() + if (hasErrors) { + process.exit(1) + } + } catch (error) { + console.error(error) + process.exit(1) + } +} + +main() diff --git a/apps/docs/features/docs/Troubleshooting.ui.tsx b/apps/docs/features/docs/Troubleshooting.ui.tsx index 9c7502f74f630..20b7875b9e109 100644 --- a/apps/docs/features/docs/Troubleshooting.ui.tsx +++ b/apps/docs/features/docs/Troubleshooting.ui.tsx @@ -2,12 +2,20 @@ import { Wrench } from 'lucide-react' import Link from 'next/link' import { type PropsWithChildren, useCallback } from 'react' -import { type ITroubleshootingEntry, getArticleSlug } from './Troubleshooting.utils' +import { + type ITroubleshootingEntry, + getArticleSlug, + getTroubleshootingUpdatedDates, +} from './Troubleshooting.utils' import { TroubleshootingFilter } from './Troubleshooting.ui.client' import { formatError, TROUBLESHOOTING_DATA_ATTRIBUTES } from './Troubleshooting.utils.shared' import { cn } from 'ui' -export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) { +export async function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry }) { + const dateUpdated = entry.data.database_id.startsWith('pseudo-') + ? new Date() + : (await getTroubleshootingUpdatedDates()).get(entry.data.database_id) + const keywords = [...entry.data.topics, ...(entry.data.keywords ?? [])] const attributes = { [TROUBLESHOOTING_DATA_ATTRIBUTES.QUERY_ATTRIBUTE]: @@ -58,14 +66,13 @@ export function TroubleshootingPreview({ entry }: { entry: ITroubleshootingEntry ))}
- {entry.data.updated_at && + {dateUpdated && (() => { - const date = new Date(entry.data.updated_at) const options = { month: 'short', day: 'numeric' } as Intl.DateTimeFormatOptions - if (date.getFullYear() !== new Date().getFullYear()) { + if (dateUpdated.getFullYear() !== new Date().getFullYear()) { options.year = 'numeric' } - return date.toLocaleDateString(undefined, options) + return dateUpdated.toLocaleDateString(undefined, options) })()}
diff --git a/apps/docs/features/docs/Troubleshooting.utils.ts b/apps/docs/features/docs/Troubleshooting.utils.ts index 0b2c755303a5d..03607a1ec4d40 100644 --- a/apps/docs/features/docs/Troubleshooting.utils.ts +++ b/apps/docs/features/docs/Troubleshooting.utils.ts @@ -11,6 +11,7 @@ import { v4 as uuidv4 } from 'uuid' import { z } from 'zod' import { DOCS_DIRECTORY } from 'lib/docs' +import { supabaseAdmin } from '~/lib/supabaseAdmin' import { cache_fullProcess_withDevCacheBust } from '~/features/helpers.fs' import { formatError } from './Troubleshooting.utils.shared' @@ -56,9 +57,8 @@ const TroubleshootingSchema = z .strict() ) .optional(), - database_id: z.string().uuid().default(uuidv4()), - created_at: z.date({ coerce: true }).optional(), - updated_at: z.date({ coerce: true }).optional(), + database_id: z.string().default(`pseudo-${uuidv4()}`), + github_url: z.string().url().optional(), }) .strict() @@ -76,10 +76,12 @@ async function getAllTroubleshootingEntriesInternal() { const isHidden = entry.startsWith('_') if (isHidden) return null - const isFile = (await stat(join(TROUBLESHOOTING_DIRECTORY, entry))).isFile() + const filePath = join(TROUBLESHOOTING_DIRECTORY, entry) + + const isFile = (await stat(filePath)).isFile() if (!isFile) return null - const fileContents = await readFile(join(TROUBLESHOOTING_DIRECTORY, entry), 'utf-8') + const fileContents = await readFile(filePath, 'utf-8') const { content, data: frontmatter } = matter(fileContents, { language: 'toml', engines: { toml: toml.parse.bind(toml) }, @@ -117,6 +119,7 @@ async function getAllTroubleshootingEntriesInternal() { const contentWithoutJsx = toMarkdown(mdxTree) return { + filePath, content, contentWithoutJsx, data: parseResult.data, @@ -190,3 +193,27 @@ export function getArticleSlug(entry: ITroubleshootingMetadata) { const escapedTitle = encodeURIComponent(slugifiedTitle) return escapedTitle } + +async function getTroubleshootingUpdatedDatesInternal() { + const databaseIds = (await getAllTroubleshootingEntries()) + .map((entry) => entry.data.database_id) + .filter((id) => !id.startsWith('pseudo-')) + + const { data, error } = await supabaseAdmin() + .from('troubleshooting_entries') + .select('id, date_updated') + .in('id', databaseIds) + if (error) { + console.error(error) + } + + return (data ?? []).reduce((acc, entry) => { + acc.set(entry.id, new Date(entry.date_updated)) + return acc + }, new Map()) +} +export const getTroubleshootingUpdatedDates = cache_fullProcess_withDevCacheBust( + getTroubleshootingUpdatedDatesInternal, + TROUBLESHOOTING_DIRECTORY, + () => JSON.stringify([]) +) diff --git a/apps/docs/lib/octokit.ts b/apps/docs/lib/octokit.ts index c1ccf9073cba9..1ad390e4ea16b 100644 --- a/apps/docs/lib/octokit.ts +++ b/apps/docs/lib/octokit.ts @@ -8,7 +8,7 @@ import { fetchRevalidatePerDay } from '~/features/helpers.fetch' let octokitInstance: Octokit -function octokit() { +export function octokit() { if (!octokitInstance) { const privateKeyPkcs8 = crypto .createPrivateKey(process.env.DOCS_GITHUB_APP_PRIVATE_KEY) diff --git a/apps/docs/lib/supabaseAdmin.ts b/apps/docs/lib/supabaseAdmin.ts new file mode 100644 index 0000000000000..2c58ea6539592 --- /dev/null +++ b/apps/docs/lib/supabaseAdmin.ts @@ -0,0 +1,18 @@ +import 'server-only' + +import { createClient, type SupabaseClient } from '@supabase/supabase-js' + +import { type Database } from 'common' + +let supabaseAdminClient: SupabaseClient | null = null + +export function supabaseAdmin() { + if (!supabaseAdminClient) { + supabaseAdminClient = createClient( + process.env.NEXT_PUBLIC_SUPABASE_URL, + process.env.SUPABASE_SECRET_KEY + ) + } + + return supabaseAdminClient +} diff --git a/apps/docs/package.json b/apps/docs/package.json index 1950af7746123..98599880b0c34 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -127,6 +127,7 @@ "openapi-types": "^12.1.3", "simple-git": "^3.24.0", "slugify": "^1.6.6", + "smol-toml": "^1.3.0", "tsconfig": "*", "tsx": "^4.6.2", "typescript": "~5.5.0", diff --git a/package-lock.json b/package-lock.json index 0bbdd849c3e64..09583bcc37789 100644 --- a/package-lock.json +++ b/package-lock.json @@ -958,6 +958,7 @@ "openapi-types": "^12.1.3", "simple-git": "^3.24.0", "slugify": "^1.6.6", + "smol-toml": "^1.3.0", "tsconfig": "*", "tsx": "^4.6.2", "typescript": "~5.5.0", @@ -38218,6 +38219,18 @@ "npm": ">= 3.0.0" } }, + "node_modules/smol-toml": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.3.0.tgz", + "integrity": "sha512-tWpi2TsODPScmi48b/OQZGi2lgUmBCHy6SZrhi/FdnnHiU1GwebbCfuQuxsC3nHaLwtYeJGPrDZDIeodDOc4pA==", + "dev": true, + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, "node_modules/snake-case": { "version": "3.0.4", "dev": true, diff --git a/packages/common/database-types.ts b/packages/common/database-types.ts index c13e9a2de8363..2da741988edc4 100644 --- a/packages/common/database-types.ts +++ b/packages/common/database-types.ts @@ -318,6 +318,7 @@ export type Database = { troubleshooting_entries: { Row: { api: Json | null + checksum: string date_created: string date_updated: string errors: Json[] | null @@ -329,6 +330,7 @@ export type Database = { } Insert: { api?: Json | null + checksum: string date_created?: string date_updated?: string errors?: Json[] | null @@ -340,6 +342,7 @@ export type Database = { } Update: { api?: Json | null + checksum?: string date_created?: string date_updated?: string errors?: Json[] | null @@ -464,6 +467,33 @@ export type Database = { } Returns: unknown } + json_matches_schema: { + Args: { + schema: Json + instance: Json + } + Returns: boolean + } + jsonb_matches_schema: { + Args: { + schema: Json + instance: Json + } + Returns: boolean + } + jsonschema_is_valid: { + Args: { + schema: Json + } + Returns: boolean + } + jsonschema_validation_errors: { + Args: { + schema: Json + instance: Json + } + Returns: string[] + } match_page_sections_v2: { Args: { embedding: string @@ -491,6 +521,12 @@ export type Database = { } Returns: string } + validate_troubleshooting_errors: { + Args: { + errors: Json[] + } + Returns: boolean + } vector_avg: { Args: { "": number[] diff --git a/supabase/migrations/20241002215612_troubleshooting_validation.sql b/supabase/migrations/20241002215612_troubleshooting_validation.sql new file mode 100644 index 0000000000000..6401f36175cb3 --- /dev/null +++ b/supabase/migrations/20241002215612_troubleshooting_validation.sql @@ -0,0 +1,71 @@ +create index idx_troubleshooting_github_url +on troubleshooting_entries (github_url); + +alter table troubleshooting_entries +add column checksum varchar(44) not null; + +create extension pg_jsonschema; + +alter table troubleshooting_entries +add constraint troubleshooting_api_check +check ( + api is null or + jsonb_matches_schema( + schema := '{ + "type": "object", + "properties": { + "sdk": { + "type": "array", + "items": { "type": "string" } + }, + "management_api": { + "type": "array", + "items": { "type": "string" } + }, + "cli": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": false + }', + instance := api + ) +); + +create or replace function validate_troubleshooting_errors(errors jsonb[]) +returns boolean as $$ +declare + error jsonb; +begin + if errors is null then + return true; + end if; + + foreach error in array errors + loop + if not jsonb_matches_schema( + schema := '{ + "type": "object", + "properties": { + "http_status_code": { "type": "number" }, + "code": { "type": "string" }, + "message": { "type": "string" } + }, + "additionalProperties": false + }', + error + ) then + return false; + end if; + end loop; + + return true; +end; +$$ language plpgsql; + +alter table troubleshooting_entries +add constraint troubleshooting_errors_check +check ( + validate_troubleshooting_errors(errors) +); From d3fa8c7b37af789fae59717f1f529d0e3013e5ed Mon Sep 17 00:00:00 2001 From: Charis Lam <26616127+charislam@users.noreply.github.com> Date: Thu, 3 Oct 2024 16:21:03 -0400 Subject: [PATCH 2/3] ci(docs): sync troubleshooting guides Sync new troubleshooting entries: - Create a new GitHub discussion if one doesn't exist - Create a DB entry for the troubleshooting guide - Sync the file with the DB entry by writing database_id to frontmatter --- .../workflows/docs-sync-troubleshooting.yml | 51 +++++++++++++++++++ apps/docs/package.json | 1 + 2 files changed, 52 insertions(+) create mode 100644 .github/workflows/docs-sync-troubleshooting.yml diff --git a/.github/workflows/docs-sync-troubleshooting.yml b/.github/workflows/docs-sync-troubleshooting.yml new file mode 100644 index 0000000000000..bf396e8977bdb --- /dev/null +++ b/.github/workflows/docs-sync-troubleshooting.yml @@ -0,0 +1,51 @@ +name: '[Docs] Sync troubleshooting guides to GitHub Discussions' + +on: + push: + branches: + - master + paths: + - 'apps/docs/content/troubleshooting/**' + workflow_dispatch: + # testing only, delete later + # pull_request: + +permissions: + contents: write + pull-requests: write + +jobs: + update-troubleshooting: + runs-on: ubuntu-latest + + env: + NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.SEARCH_SUPABASE_URL }} + SUPABASE_SECRET_KEY: ${{ secrets.SEARCH_SUPABASE_SERVICE_ROLE_KEY }} + + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + apps/docs + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run Troubleshooting script + run: npm --prefix=apps/docs run troubleshooting:sync + + - name: Create Pull Request + uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5 + with: + # for testing on pull_request only + # base: ${{ github.head_ref }} + commit-message: '[bot] sync troubleshooting guides to db' + title: '[bot] sync troubleshooting guides to db' + author: '[github-docs-sync-bot]' + branch: 'bot/docs-sync-troubleshooting' diff --git a/apps/docs/package.json b/apps/docs/package.json index 98599880b0c34..63565e2bfcb94 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -18,6 +18,7 @@ "test:smoke": "npm run codegen:references && vitest -t \"prod smoke test\"", "embeddings": "tsx scripts/search/generate-embeddings.ts", "embeddings:refresh": "npm run embeddings -- --refresh", + "troubleshooting:sync": "tsx features/docs/Troubleshooting.script.ts", "last-changed": "tsx scripts/last-changed.ts", "last-changed:reset": "npm run last-changed -- --reset", "codegen:references": "tsx features/docs/Reference.generated.script.ts", From 4018d36bac1e303c10b745f313ef7566a450a477 Mon Sep 17 00:00:00 2001 From: Charis Lam <26616127+charislam@users.noreply.github.com> Date: Thu, 3 Oct 2024 16:23:02 -0400 Subject: [PATCH 3/3] tweak(docs): troubleshooting created date Allow overriding troubleshooting created date in the frontmatter, defaulting to the sync time if it is not provided. --- apps/docs/features/docs/Troubleshooting.script.ts | 2 +- apps/docs/features/docs/Troubleshooting.utils.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/docs/features/docs/Troubleshooting.script.ts b/apps/docs/features/docs/Troubleshooting.script.ts index f1b5162af598b..cd58d8a16cc68 100644 --- a/apps/docs/features/docs/Troubleshooting.script.ts +++ b/apps/docs/features/docs/Troubleshooting.script.ts @@ -73,7 +73,7 @@ async function insertNewTroubleshootingEntry(entry: ITroubleshootingEntry, githu // @ts-ignore checksum, github_url: githubUrl, - date_created: timestamp, + date_created: entry.data.date_created ?? timestamp, date_updated: timestamp, }) .select('id') diff --git a/apps/docs/features/docs/Troubleshooting.utils.ts b/apps/docs/features/docs/Troubleshooting.utils.ts index 03607a1ec4d40..69daf4cb7e3dd 100644 --- a/apps/docs/features/docs/Troubleshooting.utils.ts +++ b/apps/docs/features/docs/Troubleshooting.utils.ts @@ -59,6 +59,7 @@ const TroubleshootingSchema = z .optional(), database_id: z.string().default(`pseudo-${uuidv4()}`), github_url: z.string().url().optional(), + date_created: z.date({ coerce: true }).optional(), }) .strict()