From a71663132025a80c7de04ebc5709e382348d594b Mon Sep 17 00:00:00 2001 From: Zicklag Date: Sat, 16 Nov 2024 15:33:27 -0600 Subject: [PATCH] chore: add database migration script for latest username update. --- .env.local | 4 +- leaf/ts/index.ts | 12 +- src/lib/usernames.ts | 36 +++-- .../__internal__/admin/+page.svelte | 2 +- .../admin/migration-508f757/+page.server.ts | 152 ++++++++++++++++++ .../admin/migration-508f757/+page.svelte | 63 ++++++++ .../admin/migration/+page.server.ts | 49 ------ .../__internal__/admin/migration/+page.svelte | 37 ----- 8 files changed, 244 insertions(+), 111 deletions(-) create mode 100644 src/routes/(internal)/__internal__/admin/migration-508f757/+page.server.ts create mode 100644 src/routes/(internal)/__internal__/admin/migration-508f757/+page.svelte delete mode 100644 src/routes/(internal)/__internal__/admin/migration/+page.server.ts delete mode 100644 src/routes/(internal)/__internal__/admin/migration/+page.svelte diff --git a/.env.local b/.env.local index e27de982..18c7b6bd 100644 --- a/.env.local +++ b/.env.local @@ -1,6 +1,6 @@ BACKEND_URL="http://127.0.0.1:7431" BACKEND_SECRET="temporarydevelopmentkey" -INSTANCE_SUBSPACE_SECRET=afkknpungyscs5gszd4ywzaq4jghwt2k7e2liejajaqjjps2pqsq +INSTANCE_SUBSPACE_SECRET=gqvkucapoouch6xhhnn3pnqu7zpwpd3a4nctz2vkm2qrsnfbf6ha RAUTHY_URL="http://localhost:8921" SMTP_HOST="localhost" SMTP_PORT="2525" @@ -22,7 +22,7 @@ FEEDBACK_WEBHOOK="" PUBLIC_URL=http://localhost:9523 PUBLIC_DOMAIN=localhost:9523 DNS_PORT=7753 -PUBLIC_USER_DOMAIN_PARENT=user.localhost:9523 +PUBLIC_USER_DOMAIN_PARENT=weird.one REDIS_URL="redis://localhost:7634" # DNS diff --git a/leaf/ts/index.ts b/leaf/ts/index.ts index 44955c43..ecfbad15 100644 --- a/leaf/ts/index.ts +++ b/leaf/ts/index.ts @@ -666,14 +666,16 @@ export class RpcClient { async add_components( link: ExactLink, - components: C[], + components: (C | { schema: Digest; data: Uint8Array })[], replaceExisting = true ): Promise { let componentData = components.map((component) => { - return { - schema: Object.getPrototypeOf(component).constructor.schemaId(), - data: component.serialize() - }; + return component instanceof Component + ? { + schema: Object.getPrototypeOf(component).constructor.schemaId(), + data: component.serialize() + } + : component; }); const resp = await this.#send_req({ AddComponents: { diff --git a/src/lib/usernames.ts b/src/lib/usernames.ts index f61e0d54..55c199b6 100644 --- a/src/lib/usernames.ts +++ b/src/lib/usernames.ts @@ -18,7 +18,7 @@ export async function setUserSubspace(rauthyId: string, subspace: SubspaceId) { } export async function claimUsername( - input: { username: string } | { domain: string }, + input: { username: string } | { domain: string; skipDomainCheck?: boolean }, rauthyId: string ) { const rauthyIdKey = USER_RAUTHY_IDS_PREFIX + rauthyId; @@ -31,7 +31,7 @@ export async function claimUsername( // Claiming a local username if (!input.username.match(validUsernameRegex)) { - throw 'Username does not pass valid username check.'; + throw `Username does not pass valid username check: '${input.username}'`; } else { username = input.username + '.' + env.PUBLIC_USER_DOMAIN_PARENT; } @@ -39,28 +39,30 @@ export async function claimUsername( // Claim a custom domain const isApex = input.domain.split('.').length == 2; - if (isApex) { - const ips = await resolveAuthoritative(input.domain, 'A'); - let matches = 0; - for (const ip of ips) { - if (ip in APP_IPS) { - matches += 1; - } else { - throw `DNS validation failed: ${input.domain} resolves to \ + if (!input.skipDomainCheck) { + if (isApex) { + const ips = await resolveAuthoritative(input.domain, 'A'); + let matches = 0; + for (const ip of ips) { + if (ip in APP_IPS) { + matches += 1; + } else { + throw `DNS validation failed: ${input.domain} resolves to \ an IP address ${ip} that is not the Weird server.`; + } } - } - if (matches == 0) { - throw `DNS validation failed: ${input.domain} does not resolve \ + if (matches == 0) { + throw `DNS validation failed: ${input.domain} does not resolve \ to the weird server.`; + } + } else { } - } else { - } - throw "Can't use custom domains yet"; + throw "Can't use custom domains yet"; + } - // username = input.domain; + username = input.domain; } const usernameKey = USER_NAMES_PREFIX + username; diff --git a/src/routes/(internal)/__internal__/admin/+page.svelte b/src/routes/(internal)/__internal__/admin/+page.svelte index d499c255..2df483b2 100644 --- a/src/routes/(internal)/__internal__/admin/+page.svelte +++ b/src/routes/(internal)/__internal__/admin/+page.svelte @@ -6,5 +6,5 @@
  • DNS Resolver
  • DNS Record Set
  • Username Manager
    • -
  • Migration ( Temp )
    • +
  • Migration ( 508f757 )
    • diff --git a/src/routes/(internal)/__internal__/admin/migration-508f757/+page.server.ts b/src/routes/(internal)/__internal__/admin/migration-508f757/+page.server.ts new file mode 100644 index 00000000..cbdf6f0d --- /dev/null +++ b/src/routes/(internal)/__internal__/admin/migration-508f757/+page.server.ts @@ -0,0 +1,152 @@ +import { claimUsername, unsetUsername, listUsers, userSubspaceByRauthyId } from '$lib/usernames'; +import { fail, type ServerLoad } from '@sveltejs/kit'; +import type { Actions } from './$types'; +import { + DatabaseDumpSchema, + base32Decode, + borshDeserialize, + type DatabaseDump, + type SubspaceId, + type DatabaseDumpDocument, + type DatabaseDumpSubspace, + formatEntityPath, + type ExactLink, + Component, + BorshSchema +} from 'leaf-proto'; +import { WEIRD_NAMESPACE, leafClient, subspace_link } from '$lib/leaf'; +import _ from 'underscore'; +import { CommonMark, RawImage } from 'leaf-proto/components'; +import { WeirdCustomDomain, setAvatar } from '$lib/leaf/profile'; + +import { createAvatar } from '@dicebear/core'; +import { glass } from '@dicebear/collection'; + +export const load: ServerLoad = async ({}) => { + const users = []; + for await (const user of listUsers()) users.push(user); + return { users }; +}; + +class Username extends Component { + value: string = ''; + constructor(s: string) { + super(); + this.value = s; + } + static componentName(): string { + return 'Username'; + } + static borshSchema(): BorshSchema { + return BorshSchema.String; + } + static specification(): Component[] { + return [new CommonMark('The username of the user represented by this entity.')]; + } +} + +export const actions = { + migrate: async ({ request }) => { + const formData = await request.formData(); + const subspaceIdStr = formData.get('subspaceId'); + if (!subspaceIdStr) + return fail(400, { + error: 'You must provide the weird instance subspace ID to import from.' + }); + let subspaceId: SubspaceId; + try { + subspaceId = base32Decode(subspaceIdStr.toString()); + } catch (_) { + return fail(400, { error: 'Could not parse subspace ID.' }); + } + const dumpFormData = formData.get('dump'); + if (!dumpFormData) return fail(400, { error: 'You must provide database dump file.' }); + const dumpData = new Uint8Array(await (dumpFormData as File).arrayBuffer()); + const dump: DatabaseDump = borshDeserialize(DatabaseDumpSchema, dumpData); + + let doc: DatabaseDumpDocument | undefined; + if (dump.documents.size != 1) return fail(400, { error: 'Dump has multiple namespaces' }); + for (const [namespace, document] of dump.documents) { + const n = new Uint8Array(namespace); + if (_.isEqual(n, WEIRD_NAMESPACE)) { + doc = document; + break; + } + } + if (doc === undefined) { + return fail(400, { error: 'Dump is missing weird namespace' }); + } + + let subspace: DatabaseDumpSubspace | undefined; + for (const [id, ss] of doc.subspaces) { + const i = new Uint8Array(id); + if (_.isEqual(subspaceId, i)) { + subspace = ss; + break; + } + } + if (subspace === undefined) { + return fail(400, { error: 'could not find specified subspace in dump' }); + } + + for (const [path, entity] of subspace) { + if (_.isEqual(path[0], { String: 'profiles' })) { + if ('String' in path[1]) { + const rauthyId = path[1].String; + const subspace = await userSubspaceByRauthyId(rauthyId); + + // User profile + let newLink: ExactLink; + const isProfile = path.length == 2; + if (isProfile) { + newLink = subspace_link(subspace, null); + } else { + newLink = subspace_link(subspace, ...path.slice(2)); + } + + const components = []; + let username: string | undefined; + let customDomain: string | undefined; + let hasAvatar = false; + for (const [schema, componentDatas] of entity.components) { + const s = new Uint8Array(schema); + if (_.isEqual(Username.schemaId(), s)) { + username = Username.deserialize(new Uint8Array(componentDatas[0])) as any; + username = username?.split('@')[0].toLowerCase().replace('.', '-'); + continue; + } + if (_.isEqual(WeirdCustomDomain.schemaId(), s)) { + customDomain = WeirdCustomDomain.deserialize( + new Uint8Array(componentDatas[0]) + ) as any; + continue; + } + if (_.isEqual(s, RawImage.schemaId())) { + hasAvatar = true; + } + for (const data of componentDatas) { + console; + components.push({ schema: new Uint8Array(schema), data: new Uint8Array(data) }); + } + } + if (customDomain) { + await claimUsername({ domain: customDomain, skipDomainCheck: true }, rauthyId); + } else if (username) { + await claimUsername({ username }, rauthyId); + } + await leafClient.add_components(newLink, components); + + if (isProfile && !hasAvatar) { + const avatar = createAvatar(glass, { seed: rauthyId, radius: 50 }); + await setAvatar( + newLink, + new RawImage('image/svg+xml', new TextEncoder().encode(avatar.toString())) + ); + } + } + } + } + + return { dump: `${JSON.stringify(dump, null, ' ')}` }; + } +} satisfies Actions; diff --git a/src/routes/(internal)/__internal__/admin/migration-508f757/+page.svelte b/src/routes/(internal)/__internal__/admin/migration-508f757/+page.svelte new file mode 100644 index 00000000..7ea8afbc --- /dev/null +++ b/src/routes/(internal)/__internal__/admin/migration-508f757/+page.svelte @@ -0,0 +1,63 @@ + + +{#if form?.error} +
    + {form.error} +
    +{:else if form?.dump} +
    + 
    +
+		{form.dump}
+
    +
    +{/if} + +

    Migrate Database

    + +

    + Upload a dump of the previous version of the database and it will be imported and migrated to the + new version. +

    + +

    + Below you can see the commits for the previous version of Weird that we are migrating from, and + the commit after that which includes the newer data model that we are migrating to. +

    + + + + + + + + + + + + + + +
    Previous Version CommitCurrent Version Commit
    + e291612d36be303eecbe136967c39fa5e2b6af5a + + + 508f757686d3ca20fa5b8b4a29b09c2fe992b4a3 + +
    + +
    + + + + +
    diff --git a/src/routes/(internal)/__internal__/admin/migration/+page.server.ts b/src/routes/(internal)/__internal__/admin/migration/+page.server.ts deleted file mode 100644 index d02bcef8..00000000 --- a/src/routes/(internal)/__internal__/admin/migration/+page.server.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { claimUsername, unsetUsername, listUsers } from '$lib/usernames'; -import type { ServerLoad } from '@sveltejs/kit'; -import type { Actions } from './$types'; -import { base32Decode } from 'leaf-proto'; -import { leafClient, subspace_link } from '$lib/leaf'; - -export const load: ServerLoad = async ({}) => { - const users = []; - for await (const user of listUsers()) users.push(user); - return { users }; -}; - -export const actions = { - dump: async ({ request }) => { - if (false) { - return { error: 'err' }; - } - - const subspace = base32Decode('xbicffkhd5jkcz4bzwwcmtfxtyttvfagipnxcq25et7pzuj4euta'); - const oldUsers = subspace_link(subspace); - - const entities = await leafClient.list_entities(oldUsers); - - const usernames = entities.flatMap((x) => { - if ( - x.path.length == 2 && - 'String' in x.path[0] && - x.path[0].String == 'profiles' && - 'String' in x.path[1] - ) { - return [x.path[1].String]; - } else { - return []; - } - }); - - return { - dump: JSON.stringify( - { length: usernames.length, usernames }, - // (key, value) => (typeof value === 'bigint' ? value.toString() + 'n' : value), - null, - ' ' - ) - }; - }, - do: async ({ request }) => { - // return { success: `Username ${username} deleted.` }; - } -} satisfies Actions; diff --git a/src/routes/(internal)/__internal__/admin/migration/+page.svelte b/src/routes/(internal)/__internal__/admin/migration/+page.svelte deleted file mode 100644 index c3edcbb6..00000000 --- a/src/routes/(internal)/__internal__/admin/migration/+page.svelte +++ /dev/null @@ -1,37 +0,0 @@ - - -{#if form?.error} -
    - {form.error} -
    -{:else if form?.dump} -
    - 
    -
-		{form.dump}
-
    -
    -{/if} - -

    Dump Migration ( Dry Run )

    - -
    - -
    - -
    -
    - -

    Do Migration

    - -
    - -
    - -
    -