From 16b0ecf7b1347002592bdad1467ac47216dfc7e9 Mon Sep 17 00:00:00 2001 From: Danny Browning Date: Wed, 15 Nov 2023 14:53:34 -0700 Subject: [PATCH 1/4] feat: add flag to deploy composite on creation Adds a flag to deploy composite when creating, which is defaulted to false. Additionally adds a flag to disable or enable indexing when deploying a composite. --- packages/cli/src/commands/composite/create.ts | 13 +++++- packages/cli/src/commands/composite/deploy.ts | 6 ++- packages/cli/test/composites.test.ts | 42 +++++++++++++++---- packages/devtools-node/src/fs.ts | 8 +++- website/docs/api/commands/cli.composite.md | 13 ++++++ 5 files changed, 71 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/commands/composite/create.ts b/packages/cli/src/commands/composite/create.ts index a6375f0a9..13a351cc1 100644 --- a/packages/cli/src/commands/composite/create.ts +++ b/packages/cli/src/commands/composite/create.ts @@ -4,6 +4,7 @@ import { createComposite, writeEncodedComposite } from '@composedb/devtools-node type Flags = CommandFlags & { output?: string + deploy: boolean } export default class CreateComposite extends Command { @@ -23,12 +24,22 @@ export default class CreateComposite extends Command { try { this.spinner.start('Creating the composite...') - const composite = await createComposite(this.ceramic, this.args.schemaFilePath) + const composite = await createComposite( + this.ceramic, + this.args.schemaFilePath, + this.flags.deploy, + ) if (this.flags.output != null) { const output = this.flags.output await writeEncodedComposite(composite, output) diff --git a/packages/cli/src/commands/composite/deploy.ts b/packages/cli/src/commands/composite/deploy.ts index c549442fc..fe65c7920 100644 --- a/packages/cli/src/commands/composite/deploy.ts +++ b/packages/cli/src/commands/composite/deploy.ts @@ -23,7 +23,11 @@ export default class CompositeDeploy extends Command< let composite: Composite | undefined = undefined if (this.stdin !== undefined) { const definition = JSON.parse(this.stdin) as EncodedCompositeDefinition - composite = await Composite.fromJSON({ ceramic: this.ceramic, definition, index: true }) + composite = await Composite.fromJSON({ + ceramic: this.ceramic, + definition, + index: true, + }) } else if (this.args.compositePath !== undefined) { composite = await readEncodedComposite(this.ceramic, this.args.compositePath, true) } else { diff --git a/packages/cli/test/composites.test.ts b/packages/cli/test/composites.test.ts index f0596656e..732d05d0a 100644 --- a/packages/cli/test/composites.test.ts +++ b/packages/cli/test/composites.test.ts @@ -8,7 +8,8 @@ import fs from 'fs-extra' import stripAnsi from 'strip-ansi' // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore -import { TEST_OUTPUT_DIR_PATH } from '../globalConsts.js' // not a module +import { TEST_OUTPUT_DIR_PATH } from '../globalConsts.js' +import { StreamID } from '@ceramicnetwork/streamid' const { readFile, readJSON } = fs @@ -18,6 +19,11 @@ const MODEL1_JSON = const MODEL2_JSON = '{"version": "1.0","name":"Model2","accountRelation":{"type":"list"},"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"stringPropName":{"type":"string","maxLength":80}},"additionalProperties":false,"required":["stringPropName"]}}' +async function checkIfModelIndexed(ceramic: CeramicClient, streamId: string): Promise { + const models = await ceramic.admin.getIndexedModels() + return models.includes(StreamID.fromString(streamId)) +} + describe('composites', () => { const seed = '3a6de55a5ef33d110a5a37438704b0f0cb77ca5977131775a70ffd1c23779c8c' @@ -56,17 +62,37 @@ describe('composites', () => { ).toBe(true) }, 60000) - test('composite creation succeeds', async () => { + test('composite creation succeeds but model is not deployed', async () => { + const ceramic = new CeramicClient() + const create = await execa('bin/run.js', [ + 'composite:create', + 'test/mocks/composite.profile.post.schema', + `--did-private-key=${seed}`, + ]) + const output = create.stdout.toString() + const def = JSON.parse(output) as EncodedCompositeDefinition + expect(def.version).toBe('1.1') + expect(Object.keys(def.version).length).not.toBe(0) + expect(def.aliases).toBeNull() + expect(def.views).toBeNull() + await expect(checkIfModelIndexed(ceramic, Object.keys(def.models)[0])).resolves.toBeFalsy() + }, 60000) + + test('composite creation succeeds and model is deployed', async () => { + const ceramic = new CeramicClient() const create = await execa('bin/run.js', [ 'composite:create', 'test/mocks/composite.profile.post.schema', `--did-private-key=${seed}`, + '--deploy', ]) const output = create.stdout.toString() - expect(output.includes('"version":"1.1"')).toBe(true) - expect(output.includes('"indices":{"')).toBe(true) - expect(output.includes('"aliases":')).toBe(true) - expect(output.includes('"views":')).toBe(true) + const def = JSON.parse(output) as EncodedCompositeDefinition + expect(def.version).toBe('1.1') + expect(Object.keys(def.version).length).not.toBe(0) + expect(def.aliases).toBeNull() + expect(def.views).toBeNull() + await expect(checkIfModelIndexed(ceramic, Object.keys(def.models)[0])).resolves.toBeTruthy() }, 60000) }) @@ -91,6 +117,7 @@ describe('composites', () => { return model }), ]) + return wasModelLoaded } @@ -105,7 +132,7 @@ describe('composites', () => { ).toBe(true) }, 60000) - test('composite deployment succeeds', async () => { + test('composite deployment succeeds and is indexed by default', async () => { const nonExistentModelStreamID = Object.keys( (undeployedComposite as EncodedCompositeDefinition).models, )[0] @@ -125,6 +152,7 @@ describe('composites', () => { const doesModelExistNow = await checkIfModelExist(ceramic, nonExistentModelStreamID) expect(doesModelExistNow).toBeTruthy() + await expect(checkIfModelIndexed(ceramic, nonExistentModelStreamID)).resolves.toBeTruthy() }, 60000) }) diff --git a/packages/devtools-node/src/fs.ts b/packages/devtools-node/src/fs.ts index 3af823e9f..25b1bae26 100644 --- a/packages/devtools-node/src/fs.ts +++ b/packages/devtools-node/src/fs.ts @@ -25,9 +25,13 @@ export function getDirPath(path: PathInput): string { /** * Create a Composite from a GraphQL schema path. */ -export async function createComposite(ceramic: CeramicClient, path: PathInput): Promise { +export async function createComposite( + ceramic: CeramicClient, + path: PathInput, + deploy: boolean, +): Promise { const file = await readFile(getFilePath(path)) - return await Composite.create({ ceramic, schema: file.toString() }) + return await Composite.create({ ceramic, schema: file.toString(), index: !deploy }) } /** diff --git a/website/docs/api/commands/cli.composite.md b/website/docs/api/commands/cli.composite.md index 9710a35b0..305705a99 100644 --- a/website/docs/api/commands/cli.composite.md +++ b/website/docs/api/commands/cli.composite.md @@ -46,6 +46,10 @@ Create an encoded composite definition from GraphQL [Composite Schema](https://d You can find a detailed guide on the creation of Composites [here](https://developers.ceramic.network/docs/composedb/guides/data-modeling/composites) +If updating your composite, run this command with `--no-deploy`. Your GraphQL +definition will still be updated, but Ceramic will not attempt to re-index +your composite. + ``` USAGE $ composedb composite:create INPUT @@ -57,6 +61,8 @@ OPTIONS -c, --ceramic-url Ceramic API URL -k, --did-private-key DID Private Key (you can generate a fresh private key using composedb did:generate-private-key) -o, --output a path to file where the resulting encoded composite definition should be saved + -d, --deploy Deploy the composite to the ceramic node, which will start indexing on the composite + --no-deploy Do not deploy the composite to the ceramic node ``` ### `composedb composite:models` @@ -117,6 +123,12 @@ available on the Ceramic Node that yor DApp connects to. You can find a detailed guide on Composites' deployment [here](https://developers.ceramic.network/docs/composedb/guides/data-modeling/composites#deploying-composites) +If updating your composite to add additional query fields, do not run this command. +It should only be run _the first time_ you add your composite to the Ceramic node. + +If you are reusing a model multiple times in different composites, you can also +skip this command. + ``` USAGE $ composedb composite:deploy PATH @@ -126,6 +138,7 @@ ARGUMENTS OPTIONS -c, --ceramic-url Ceramic API URL + -k, --did-private-key DID Private Key (you can generate a fresh private key using composedb did:generate-private-key) ``` ### `composedb composite:compile` From 2e41e5e58a5a7118fc337e88c0637a22f3927092 Mon Sep 17 00:00:00 2001 From: Danny Browning Date: Tue, 5 Dec 2023 10:11:25 -0700 Subject: [PATCH 2/4] review changes --- packages/cli/src/commands/composite/compile.ts | 2 +- packages/cli/src/commands/composite/create.ts | 1 + packages/cli/src/commands/composite/extract-model.ts | 2 +- packages/cli/src/commands/composite/from-model.ts | 8 ++++++++ packages/cli/src/commands/composite/merge.ts | 2 +- packages/cli/src/commands/composite/models.ts | 2 +- packages/cli/test/composites.test.ts | 1 + packages/devtools-node/src/fs.ts | 10 +++++----- packages/devtools-node/src/server.ts | 2 +- packages/devtools/src/composite.ts | 2 +- website/docs/api/commands/cli.composite.md | 9 +++++---- 11 files changed, 26 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/commands/composite/compile.ts b/packages/cli/src/commands/composite/compile.ts index 3ce50180c..14b42767c 100644 --- a/packages/cli/src/commands/composite/compile.ts +++ b/packages/cli/src/commands/composite/compile.ts @@ -31,7 +31,7 @@ export default class CompositeCompile extends Command { composite = await Composite.fromJSON({ ceramic: this.ceramic, definition }) outputPaths = allArgs } else if (this.stdin === undefined && allArgs.length >= 2) { - composite = await readEncodedComposite(this.ceramic, allArgs[0]) + composite = await readEncodedComposite(this.ceramic, allArgs[0], false) outputPaths = allArgs.splice(1) } else if (this.stdin !== undefined && allArgs.length < 1) { this.spinner.fail( diff --git a/packages/cli/src/commands/composite/create.ts b/packages/cli/src/commands/composite/create.ts index 13a351cc1..76bbe80f8 100644 --- a/packages/cli/src/commands/composite/create.ts +++ b/packages/cli/src/commands/composite/create.ts @@ -29,6 +29,7 @@ export default class CreateComposite extends Command { composite = await Composite.fromJSON({ ceramic: this.ceramic, definition }) modelsToExtract = allArgs } else if (this.stdin === undefined && allArgs.length >= 2) { - composite = await readEncodedComposite(this.ceramic, allArgs[0]) + composite = await readEncodedComposite(this.ceramic, allArgs[0], false) modelsToExtract = allArgs.splice(1) } else if (this.stdin !== undefined && allArgs.length < 1) { this.spinner.fail( diff --git a/packages/cli/src/commands/composite/from-model.ts b/packages/cli/src/commands/composite/from-model.ts index 252994f5d..4559f11bc 100644 --- a/packages/cli/src/commands/composite/from-model.ts +++ b/packages/cli/src/commands/composite/from-model.ts @@ -5,6 +5,7 @@ import { writeEncodedComposite } from '@composedb/devtools-node' type Flags = CommandFlags & { output?: string + deploy: boolean } export default class CompositeFromModel extends Command { @@ -18,6 +19,12 @@ export default class CompositeFromModel extends Command { char: 'o', description: 'path to the file where the composite representation should be saved', }), + deploy: Flags.boolean({ + char: 'd', + description: + 'Deploy the composite to the ceramic node, which will start indexing on the composite', + default: true, + }), } async run(): Promise { @@ -44,6 +51,7 @@ export default class CompositeFromModel extends Command { const composite = await Composite.fromModels({ ceramic: this.ceramic, models: allModelStreamIDs, + index: this.flags.deploy, }) if (this.flags.output != null) { const output = this.flags.output diff --git a/packages/cli/src/commands/composite/merge.ts b/packages/cli/src/commands/composite/merge.ts index e1a5d776c..7f34abcea 100644 --- a/packages/cli/src/commands/composite/merge.ts +++ b/packages/cli/src/commands/composite/merge.ts @@ -43,7 +43,7 @@ export default class CompositeMerge extends Command { try { this.spinner.start('Merging composites...') const composites = await Promise.all( - compositePaths.map(async (path) => await readEncodedComposite(this.ceramic, path)), + compositePaths.map(async (path) => await readEncodedComposite(this.ceramic, path, false)), ) const commonEmbedsFlag = this.flags['common-embeds'] as string | undefined diff --git a/packages/cli/src/commands/composite/models.ts b/packages/cli/src/commands/composite/models.ts index ce1004a58..261ecbd03 100644 --- a/packages/cli/src/commands/composite/models.ts +++ b/packages/cli/src/commands/composite/models.ts @@ -45,7 +45,7 @@ export default class CompositeModels extends Command< const definition = JSON.parse(this.stdin) as EncodedCompositeDefinition composite = await Composite.fromJSON({ ceramic: this.ceramic, definition }) } else if (this.args.compositePath !== undefined) { - composite = await readEncodedComposite(this.ceramic, this.args.compositePath) + composite = await readEncodedComposite(this.ceramic, this.args.compositePath, false) } else { this.spinner.fail( 'You need to pass a path to encoded composite either via an arg or through stdin', diff --git a/packages/cli/test/composites.test.ts b/packages/cli/test/composites.test.ts index 732d05d0a..e5768c4d6 100644 --- a/packages/cli/test/composites.test.ts +++ b/packages/cli/test/composites.test.ts @@ -68,6 +68,7 @@ describe('composites', () => { 'composite:create', 'test/mocks/composite.profile.post.schema', `--did-private-key=${seed}`, + `--no-deploy`, ]) const output = create.stdout.toString() const def = JSON.parse(output) as EncodedCompositeDefinition diff --git a/packages/devtools-node/src/fs.ts b/packages/devtools-node/src/fs.ts index 25b1bae26..d4ea8f99b 100644 --- a/packages/devtools-node/src/fs.ts +++ b/packages/devtools-node/src/fs.ts @@ -31,7 +31,7 @@ export async function createComposite( deploy: boolean, ): Promise { const file = await readFile(getFilePath(path)) - return await Composite.create({ ceramic, schema: file.toString(), index: !deploy }) + return await Composite.create({ ceramic, schema: file.toString(), index: deploy }) } /** @@ -40,12 +40,12 @@ export async function createComposite( export async function readEncodedComposite( ceramic: CeramicClient | string, path: PathInput, - index?: boolean, + deploy: boolean, ): Promise { const client = typeof ceramic === 'string' ? new CeramicClient(ceramic) : ceramic const file = getFilePath(path) const definition = (await readJSON(file)) as EncodedCompositeDefinition - return Composite.fromJSON({ ceramic: client, definition: definition, index: index }) + return Composite.fromJSON({ ceramic: client, definition: definition, index: deploy }) } /** @@ -116,7 +116,7 @@ export async function writeEncodedCompositeRuntime( runtimePath: PathInput, schemaPath?: PathInput, ): Promise { - const definition = await readEncodedComposite(ceramic, definitionPath) + const definition = await readEncodedComposite(ceramic, definitionPath, false) const runtime = definition.toRuntime() await writeRuntimeDefinition(runtime, runtimePath) if (schemaPath != null) { @@ -134,7 +134,7 @@ export async function mergeEncodedComposites( ): Promise { const sources = Array.isArray(source) ? source : [source] const composites = await Promise.all( - sources.map(async (path) => await readEncodedComposite(ceramic, path)), + sources.map(async (path) => await readEncodedComposite(ceramic, path, false)), ) const file = getFilePath(destination) await writeEncodedComposite(Composite.from(composites), file) diff --git a/packages/devtools-node/src/server.ts b/packages/devtools-node/src/server.ts index 8d26476e3..ebcd6a5de 100644 --- a/packages/devtools-node/src/server.ts +++ b/packages/devtools-node/src/server.ts @@ -79,6 +79,6 @@ export async function serveEncodedDefinition( params: ServeDefinitionParams, ): Promise { const { path, ...rest } = params - const composite = await readEncodedComposite(params.ceramicURL, path) + const composite = await readEncodedComposite(params.ceramicURL, path, false) return await serveGraphQL({ ...rest, definition: composite.toRuntime() }) } diff --git a/packages/devtools/src/composite.ts b/packages/devtools/src/composite.ts index e37fa71ae..52803480a 100644 --- a/packages/devtools/src/composite.ts +++ b/packages/devtools/src/composite.ts @@ -236,7 +236,7 @@ export type FromModelsParams = CompositeOptions & { * Whether to add the Models to the index or not. If `true`, the Ceramic instance must be * authenticated with an admin DID. Defaults to `false`. */ - index?: boolean + index: boolean } /** diff --git a/website/docs/api/commands/cli.composite.md b/website/docs/api/commands/cli.composite.md index 305705a99..c22ae6ad4 100644 --- a/website/docs/api/commands/cli.composite.md +++ b/website/docs/api/commands/cli.composite.md @@ -46,9 +46,9 @@ Create an encoded composite definition from GraphQL [Composite Schema](https://d You can find a detailed guide on the creation of Composites [here](https://developers.ceramic.network/docs/composedb/guides/data-modeling/composites) -If updating your composite, run this command with `--no-deploy`. Your GraphQL -definition will still be updated, but Ceramic will not attempt to re-index -your composite. +If updating your composite by specifying additional fields to filter on using the `createIndex` directive, run this +command with `--no-deploy`. Your GraphQL definition will still be updated, but Ceramic will not attempt to re-index your +composite. For other updates to your composite, such as adding new models, run with `--deploy`. ``` USAGE @@ -61,7 +61,8 @@ OPTIONS -c, --ceramic-url Ceramic API URL -k, --did-private-key DID Private Key (you can generate a fresh private key using composedb did:generate-private-key) -o, --output a path to file where the resulting encoded composite definition should be saved - -d, --deploy Deploy the composite to the ceramic node, which will start indexing on the composite + -d, --deploy Deploy the composite to the ceramic node, which will cause the node to start indexing the +models contained within the composite --no-deploy Do not deploy the composite to the ceramic node ``` From dce5745fc6a59cb778ab6485a33395e4f300eb53 Mon Sep 17 00:00:00 2001 From: Danny Browning Date: Thu, 7 Dec 2023 08:20:37 -0700 Subject: [PATCH 3/4] changes based on review --- packages/cli/src/commands/composite/from-model.ts | 2 +- website/docs/api/commands/cli.composite.md | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/commands/composite/from-model.ts b/packages/cli/src/commands/composite/from-model.ts index 4559f11bc..36e5545e0 100644 --- a/packages/cli/src/commands/composite/from-model.ts +++ b/packages/cli/src/commands/composite/from-model.ts @@ -23,7 +23,7 @@ export default class CompositeFromModel extends Command { char: 'd', description: 'Deploy the composite to the ceramic node, which will start indexing on the composite', - default: true, + default: false, }), } diff --git a/website/docs/api/commands/cli.composite.md b/website/docs/api/commands/cli.composite.md index c22ae6ad4..860549b35 100644 --- a/website/docs/api/commands/cli.composite.md +++ b/website/docs/api/commands/cli.composite.md @@ -47,8 +47,8 @@ Create an encoded composite definition from GraphQL [Composite Schema](https://d You can find a detailed guide on the creation of Composites [here](https://developers.ceramic.network/docs/composedb/guides/data-modeling/composites) If updating your composite by specifying additional fields to filter on using the `createIndex` directive, run this -command with `--no-deploy`. Your GraphQL definition will still be updated, but Ceramic will not attempt to re-index your -composite. For other updates to your composite, such as adding new models, run with `--deploy`. +command with `--no-deploy`. Your GraphQL definition will still be updated, but Ceramic will not attempt to re-index the +models in your composite. For other updates to your composite, such as adding new models, run with `--deploy`. ``` USAGE @@ -124,11 +124,9 @@ available on the Ceramic Node that yor DApp connects to. You can find a detailed guide on Composites' deployment [here](https://developers.ceramic.network/docs/composedb/guides/data-modeling/composites#deploying-composites) -If updating your composite to add additional query fields, do not run this command. -It should only be run _the first time_ you add your composite to the Ceramic node. - -If you are reusing a model multiple times in different composites, you can also -skip this command. +If updating your composite by specifying additional fields to filter on using the `createIndex` directive, do not run +this command. Your GraphQL definition will still be updated, but Ceramic will not attempt to re-index the +models in your composite. For other updates to your composite, such as adding new models, run with `--deploy`. ``` USAGE From 946b19b4735a4d9ebdcc414665e4887dc5f1f89a Mon Sep 17 00:00:00 2001 From: Danny Browning Date: Thu, 14 Dec 2023 17:38:12 -0700 Subject: [PATCH 4/4] changes based on review --- packages/devtools-node/src/fs.ts | 2 +- packages/devtools/src/composite.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/devtools-node/src/fs.ts b/packages/devtools-node/src/fs.ts index d4ea8f99b..de81610ce 100644 --- a/packages/devtools-node/src/fs.ts +++ b/packages/devtools-node/src/fs.ts @@ -40,7 +40,7 @@ export async function createComposite( export async function readEncodedComposite( ceramic: CeramicClient | string, path: PathInput, - deploy: boolean, + deploy?: boolean, ): Promise { const client = typeof ceramic === 'string' ? new CeramicClient(ceramic) : ceramic const file = getFilePath(path) diff --git a/packages/devtools/src/composite.ts b/packages/devtools/src/composite.ts index 52803480a..e37fa71ae 100644 --- a/packages/devtools/src/composite.ts +++ b/packages/devtools/src/composite.ts @@ -236,7 +236,7 @@ export type FromModelsParams = CompositeOptions & { * Whether to add the Models to the index or not. If `true`, the Ceramic instance must be * authenticated with an admin DID. Defaults to `false`. */ - index: boolean + index?: boolean } /**