Merge pull request #354 from OpenFn/pull-with-id

Add support to supplying a project id when pulling

packages/cli/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @openfn/cli
 
+## 0.3.1
+
+### Patch Changes
+
+- Fix expected Lightning provisining path for versions greater than Lightning v0.7.3
+- Updated dependencies
+  - @openfn/deploy@0.2.5
+
+## 0.3.0
+
+### Minor Changes
+
+- add a projectId option to pull, allowing to pull a project without a local state file
+
+### Patch Changes
+
+- 4b23423: Internal refactor of options
+- Updated dependencies
+  - @openfn/deploy@0.2.4
+
 ## 0.2.4
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "0.2.4",
+  "version": "0.3.1",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",

packages/cli/src/options.ts

@@ -1,4 +1,5 @@
 import path from 'node:path';
+
 import yargs from 'yargs';
 import type { ExecutionPlan } from '@openfn/runtime';
 import type { CommandList } from './commands';
@@ -48,6 +49,7 @@ export type Opts = {
   useAdaptorsMonorepo?: boolean;
   workflow?: CLIExecutionPlan | ExecutionPlan;
   workflowPath?: string;
+  projectId?: string;
 };
 
 // Definition of what Yargs returns (before ensure is called)
@@ -209,6 +211,20 @@ const getBaseDir = (opts: { path?: string }) => {
   return basePath;
 };
 
+export const projectId: CLIOption = {
+  name: 'project-id',
+  yargs: {
+    hidden: true,
+  },
+  ensure: (opts) => {
+      const projectId = opts.projectId;
+      //check that this is a uuid
+      return projectId;
+    },
+};
+
+
+
 // Input path covers jobPath and workflowPath
 export const inputPath: CLIOption = {
   name: 'input-path',

packages/cli/src/pull/command.ts

@@ -3,25 +3,29 @@ import { build, ensure } from '../util/command-builders';
 import { Opts } from '../options';
 import * as o from '../options';
 
-export type DeployOptions = Required<
+export type PullOptions = Required<
   Pick<
     Opts,
-    'command' | 'log' | 'logJson' | 'statePath' | 'projectPath' | 'configPath'
+    'command' | 'log' | 'logJson' | 'statePath' | 'projectPath' | 'configPath' | 'projectId' | 'confirm'
   >
 >;
 
 const options = [o.statePath, o.projectPath, o.configPath, o.log, o.logJson];
 
-const pullCommand: yargs.CommandModule<DeployOptions> = {
-  command: 'pull',
+const pullCommand: yargs.CommandModule<PullOptions> = {
+  command: 'pull [projectId]',
   describe:
     "Pull a project's state and spec from a Lightning Instance to the local directory",
-  builder: (yargs: yargs.Argv<DeployOptions>) => {
-    return build(options, yargs).example(
-      'pull',
-      'Pull an updated copy of a project spec and state from a Lightning Instance'
-    );
-  },
+  builder: (yargs: yargs.Argv<PullOptions>) => 
+     build(options, yargs)
+    .positional('projectId', {
+                describe: 
+                    'The id of the project that should be pulled shouled be a UUID',
+                demandOption: true,
+    }).example(
+      'pull 57862287-23e6-4650-8d79-e1dd88b24b1c',
+      'Pull an updated copy of a the above spec and state from a Lightning Instance'
+    ),
   handler: ensure('pull', options),
 };
 

packages/cli/src/pull/handler.ts

@@ -1,23 +1,57 @@
 import path from 'path';
 import fs from 'node:fs/promises';
-import { DeployConfig, getProject, getConfig, getState } from '@openfn/deploy';
+import {
+  DeployConfig,
+  getConfig,
+  getState,
+  mergeSpecIntoState,
+  getSpec,
+} from '@openfn/deploy';
 import type { Logger } from '../util/logger';
-import { DeployOptions } from '../deploy/command';
+import { PullOptions } from '../pull/command';
+import assertPath from '../util/assert-path';
 
-async function pullHandler(options: DeployOptions, logger: Logger) {
+async function pullHandler(options: PullOptions, logger: Logger) {
   try {
+    assertPath(options.projectId);
     const config = mergeOverrides(await getConfig(options.configPath), options);
-    logger.always('Downloading project yaml and  state from instance');
+    logger.always('Downloading project yaml and state from instance');
 
     const state = await getState(config.statePath);
-    const { data: new_state } = await getProject(config, state.id);
-    const url = new URL(`/download/yaml?id=${state.id}`, config.endpoint);
-    const res = await fetch(url);
+    const url = new URL(
+      `api/provision/yaml?id=${options.projectId}`,
+      config.endpoint
+    );
+    logger.debug('Fetching project spec from', url);
 
+    const res = await fetch(url, {
+      headers: {
+        Authorization: `Bearer ${config.apiKey}`,
+        Accept: 'application/json',
+      },
+    });
+
+    // TODO - what if the request was denied (406) or 404?
+
+    const resolvedPath = path.resolve(config.specPath);
+    logger.debug('reading spec from', resolvedPath);
     // @ts-ignore
-    await fs.writeFile(path.resolve(config.specPath), res.body);
-    // @ts-ignore
-    await fs.writeFile(path.resolve(config.statePath), new_state);
+    await fs.writeFile(resolvedPath, res.body!);
+    const spec = await getSpec(config.specPath);
+    logger.debug('validated spec: ', spec);
+
+    if (spec.errors.length > 0) {
+      logger.error('ERROR: invalid spec');
+      logger.error(spec.errors);
+      process.exitCode = 1;
+      process.exit(1);
+    }
+
+    const nextState = mergeSpecIntoState(state, spec.doc);
+    await fs.writeFile(
+      path.resolve(config.statePath),
+      JSON.stringify(nextState, null, 2)
+    );
 
     logger.success('Project pulled successfully');
     process.exitCode = 0;
@@ -33,7 +67,7 @@ async function pullHandler(options: DeployOptions, logger: Logger) {
 // Options
 function mergeOverrides(
   config: DeployConfig,
-  options: DeployOptions
+  options: PullOptions
 ): DeployConfig {
   return {
     ...config,

packages/deploy/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @openfn/deploy
 
+## 0.2.5
+
+### Patch Changes
+
+- Fix expected Lightning provisining path for versions greater than Lightning v0.7.3
+
+## 0.2.4
+
+### Patch Changes
+
+- Update typings and some small refactoring
+
 ## 0.2.3
 
 ### Patch Changes

packages/deploy/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/deploy",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "description": "Deploy projects to Lightning instances",
   "type": "module",
   "exports": {
@@ -16,7 +16,7 @@
   "scripts": {
     "test": "pnpm ava",
     "test:watch": "pnpm ava -w",
-    "_test:types": "pnpm tsc --noEmit --project tsconfig.json --excludeDirectories test",
+    "test:types": "pnpm tsc --project tsconfig.test.json",
     "build": "tsup --config ../../tsup.config.js src/index.ts",
     "build:watch": "pnpm build --watch",
     "pack": "pnpm pack --pack-destination ../../dist"

packages/deploy/src/index.ts

@@ -9,7 +9,7 @@ import {
   mergeSpecIntoState,
   toProjectPayload,
 } from './stateTransform';
-import { deployProject,  getProject } from './client';
+import { deployProject, getProject } from './client';
 import { DeployError } from './deployError';
 import { Logger } from '@openfn/logger';
 // =============== Configuration ===============
@@ -27,7 +27,7 @@ function mergeDefaultOptions(options: Partial<DeployConfig>): DeployConfig {
   };
 }
 
-export { getProject }; 
+export { getProject, mergeSpecIntoState };
 
 export async function getConfig(path?: string): Promise<DeployConfig> {
   try {
@@ -39,8 +39,6 @@ export async function getConfig(path?: string): Promise<DeployConfig> {
   }
 }
 
-
-
 export function validateConfig(config: DeployConfig) {
   if (!config.apiKey) {
     throw new DeployError('Missing API key', 'CONFIG_ERROR');
@@ -84,7 +82,7 @@ function writeState(config: DeployConfig, nextState: {}): Promise<void> {
 // ==================== Spec ===================
 
 // Given a path to a project spec, read and validate it.
-async function getSpec(path: string) {
+export async function getSpec(path: string) {
   try {
     const body = await readFile(path, 'utf8');
     return parseAndValidate(body);
@@ -97,8 +95,6 @@ async function getSpec(path: string) {
   }
 }
 
-
-
 // =============================================
 
 export async function deploy(config: DeployConfig, logger: Logger) {
@@ -109,7 +105,7 @@ export async function deploy(config: DeployConfig, logger: Logger) {
 
   logger.debug('spec', spec);
   if (spec.errors.length > 0) {
-    spec.errors.forEach((e) => logger.warn(e.message));
+    spec.errors.forEach((e: any) => logger.warn(e.message));
     throw new DeployError(`${config.specPath} has errors`, 'VALIDATION_ERROR');
   }
   const nextState = mergeSpecIntoState(state, spec.doc);