Merge pull request #46 from Serverless-Devs/zxy/dev-secret

feat: @serverless-devs/secret

packages/component-interface/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/component-interface",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "description": "request for serverless-devs",
   "main": "lib/index.js",
   "scripts": {

packages/component-interface/src/index.ts

@@ -22,4 +22,5 @@ export interface IInputs {
   cwd: string;
   outputs?: Record<string, any>;
   output?: Record<string, any>; // 当前步骤输出
+  diffs?: Record<string, any>[]; // 当 --baseline-template 参数存在时，和基线yaml做diff
 }

packages/credential/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/credential",
-  "version": "0.0.9-beta.1",
+  "version": "0.0.9",
   "description": "credential for serverless-devs",
   "main": "lib/index.js",
   "scripts": {

packages/credential/src/actions/set/index.ts

@@ -1,5 +1,5 @@
 import { each, keys, set, intersection, get, isEmpty, merge, isNumber } from 'lodash';
-import { getYamlContent, writeData, Alibaba, IAliCredential } from '../../utils';
+import { prompt, getYamlContent, writeData, Alibaba, IAliCredential, validateInput } from '../../utils';
 import { CRYPTO_STRING, PROVIDER, PROVIDER_CREDENTIAL_KEYS } from '../../constant';
 import Logger from '../../logger';
 import * as inquirer from './inquirer';
@@ -69,10 +69,23 @@ export default class SetCredential {
       }
       set(credInformation, 'AccountID', accountId);
     } catch (ex: any) {
-      Logger.logger.warn(get(ex, 'data.Message'));
-      Logger.logger.warn('Please make sure provided access is legal, or serverless-devs service on Cloud Providers may fail.');
+      Logger.logger.debug(get(ex, 'data.Message'));
+      // Logger.logger.warn('Please make sure provided access is legal, or serverless-devs service on Cloud Providers may fail.');
       if (!uid) {
-        throw ex;
+        Logger.logger.warn('AccountID auto get failed, please input AccountID manually.');
+        Logger.logger.warn('The RAM policy of this AccountID must be set properly. Docs: https://help.aliyun.com/zh/ram/use-cases/ensure-security-of-alibaba-cloud-resources');
+        const option = {
+          type: 'input',
+          message: `AccountID: `,
+          name: 'AccountID',
+          validate: validateInput,
+        };
+        const { AccountID } = await prompt([option]);
+        if (AccountID) {
+          uid = AccountID;
+        } else {
+          throw ex;
+        }
       }
       set(credInformation, 'AccountID', uid);
     }

packages/credential/src/constant.ts

@@ -31,13 +31,13 @@ export const PROVIDER_LIST = [
 
 // 厂商密钥获取文档
 export const PROVIDER_DOCS = {
-  [PROVIDER.alibaba]: 'http://config.devsapp.net/account/alibaba',
-  [PROVIDER.aws]: 'http://config.devsapp.net/account/aws',
-  [PROVIDER.huawei]: 'http://config.devsapp.net/account/huawei',
-  [PROVIDER.azure]: 'http://config.devsapp.net/account/azure',
-  [PROVIDER.baidu]: 'http://config.devsapp.net/account/baidu',
-  [PROVIDER.google]: 'http://config.devsapp.net/account/gcp',
-  [PROVIDER.tencent]: 'http://config.devsapp.net/account/tencent',
+  [PROVIDER.alibaba]: 'https://docs.serverless-devs.com/user-guide/config/',
+  [PROVIDER.aws]: 'https://docs.serverless-devs.com/user-guide/builtin/config/#aws',
+  [PROVIDER.huawei]: 'https://docs.serverless-devs.com/user-guide/builtin/config/#_16',
+  [PROVIDER.azure]: 'https://docs.serverless-devs.com/user-guide/builtin/config/#azure',
+  [PROVIDER.baidu]: 'https://docs.serverless-devs.com/user-guide/builtin/config/#_15',
+  [PROVIDER.google]: 'https://docs.serverless-devs.com/user-guide/builtin/config/#google-cloud',
+  [PROVIDER.tencent]: 'https://docs.serverless-devs.com/user-guide/builtin/config/#_17',
   [PROVIDER.volcengine]: 'https://www.volcengine.com/docs/6291/65568',
 };
 

packages/engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/engine",
-  "version": "0.1.4-beta.9",
+  "version": "0.1.4",
   "description": "a engine lib for serverless-devs",
   "main": "lib/index.js",
   "scripts": {
@@ -23,20 +23,23 @@
   "dependencies": {
     "@serverless-cd/debug": "^4.3.4",
     "@serverless-devs/credential": "workspace:^",
+    "@serverless-devs/load-application": "workspace:^",
     "@serverless-devs/load-component": "workspace:^",
     "@serverless-devs/logger": "workspace:^",
     "@serverless-devs/parse-spec": "workspace:^",
+    "@serverless-devs/secret": "workspace:^",
     "@serverless-devs/utils": "workspace:^",
-    "@serverless-devs/load-application": "workspace:^",
     "ajv": "^8.12.0",
     "chalk": "4.x",
+    "deep-diff": "^1.0.2",
     "flatted": "^3.2.7",
     "fs-extra": "^11.1.0",
     "lodash": "^4.17.21",
     "string-argv": "^0.3.2",
     "xstate": "^4.37.2"
   },
   "devDependencies": {
+    "@types/deep-diff": "^1.0.5",
     "@types/fs-extra": "^11.0.1",
     "@types/lodash": "^4.14.195"
   }

packages/engine/src/actions/index.ts

@@ -1,7 +1,7 @@
 import { IAction, IActionLevel, IActionType, IAllowFailure, IComponentAction, IHookType, IPluginAction, IRunAction, getInputs } from '@serverless-devs/parse-spec';
 import { isEmpty, filter, includes, set, get } from 'lodash';
 import * as utils from '@serverless-devs/utils';
-import { DevsError, ETrackerType, isDevsDebugMode } from '@serverless-devs/utils';
+import { DevsError, ETrackerType } from '@serverless-devs/utils';
 import fs from 'fs-extra';
 import { spawn } from 'child_process';
 import loadComponent from '@serverless-devs/load-component';
@@ -12,7 +12,7 @@ import { ILoggerInstance } from '@serverless-devs/logger';
 import { EXIT_CODE } from '../constants';
 import { IStepOptions } from '../types';
 
-const debug = isDevsDebugMode() ? require('@serverless-cd/debug')('serverless-devs:engine') : (i: any) => {};
+const debug = require('@serverless-cd/debug')('serverless-devs:engine');
 
 interface IRecord {
   magic: Record<string, any>; // 记录魔法变量
@@ -240,7 +240,7 @@ You can still use them now, but we suggest to modify them.`)
   private async plugin(hook: IPluginAction) {
     try {
       // Load the plugin component.
-      const instance = await loadComponent(hook.value);
+      const instance = await loadComponent(hook.value, { logger: this.logger });
       // Determine the inputs for the plugin based on the record's pluginOutput.
       const inputs = isEmpty(this.record.pluginOutput) ? this.inputs : this.record.pluginOutput;
       // Execute the plugin with the determined inputs and provided arguments.

packages/engine/src/index.ts

@@ -1,22 +1,23 @@
 import { createMachine, interpret } from 'xstate';
 import { isEmpty, get, each, map, isFunction, has, uniqueId, filter, omit, includes, set, isNil, isUndefined, keys, size, cloneDeep, find } from 'lodash';
-import { IStepOptions, IRecord, IStatus, IEngineOptions, IContext, IEngineError, STEP_STATUS } from './types';
-import { getProcessTime, getCredential, stringify, getAllowFailure } from './utils';
+import { IStepOptions, IRecord, IStatus, IEngineOptions, IContext, IEngineError, STEP_STATUS, IDiff } from './types';
+import { getProcessTime, getCredential, stringify, getAllowFailure, getDiffs } from './utils';
 import ParseSpec, { getInputs, ISpec, IHookType, IStep as IParseStep, IActionLevel } from '@serverless-devs/parse-spec';
 import path from 'path';
 import chalk from 'chalk';
 import Actions from './actions';
 import Credential from '@serverless-devs/credential';
 import loadComponent from '@serverless-devs/load-component';
 import Logger, { ILoggerInstance } from '@serverless-devs/logger';
-import { DevsError, ETrackerType, emoji, getAbsolutePath, getRootHome, getUserAgent, traceid, isDevsDebugMode } from '@serverless-devs/utils';
+import SecretManager from '@serverless-devs/secret';
+import { DevsError, ETrackerType, emoji, getAbsolutePath, getRootHome, getUserAgent, traceid } from '@serverless-devs/utils';
 import { EXIT_CODE, INFO_EXP_PATTERN, COMPONENT_EXP_PATTERN } from './constants';
 import assert from 'assert';
 import Ajv from 'ajv';
 export * from './types';
 export { verify, preview, init } from './utils';
 
-const debug = isDevsDebugMode() ? require('@serverless-cd/debug')('serverless-devs:engine') : (i: any) => {};
+const debug = require('@serverless-cd/debug')('serverless-devs:engine');
 
 /**
  * Engine Class
@@ -33,12 +34,16 @@ class Engine {
   } as IContext;
   private record = { status: STEP_STATUS.PENDING, editStatusAble: true } as IRecord;
   private spec = {} as ISpec;
+  private baselineSpec = {} as ISpec;
   private glog!: Logger;
   private logger!: ILoggerInstance;
   private parseSpecInstance!: ParseSpec;
+  private parseSpecInstanceBaseline!: ParseSpec;
   private globalActionInstance!: Actions; // 全局的 action
   private actionInstance!: Actions; // 项目的 action
   private info: Record<string, any> = {}; // 存储全局变量
+  private secretManager!: SecretManager; // 敏感参数管理
+  private diffs: IDiff[] = []; // baseline diff
 
   constructor(private options: IEngineOptions) {
     debug('engine start');
@@ -56,12 +61,30 @@ class Engine {
     // 初始化 logger
     this.glog = this.getLogger() as Logger;
     this.logger = this.glog.__generate('engine');
+    // 初始化 secretManager
+    this.secretManager = SecretManager.getInstance();
+    // 加密所有敏感值
+    const secrets = this.secretManager.getAllSecrets();
+    for (const i of keys(secrets)) {
+      this.glog.__setSecret([i, secrets[i]]);
+      this.glog.__setSecret([i, this.secretManager.getSecret(i)]);
+    }
     // 初始化 spec
     this.parseSpecInstance = new ParseSpec(get(this.options, 'template'), {
       argv: this.options.args,
       logger: this.logger,
     });
     this.spec = await this.parseSpecInstance.start();
+    // 20240808: Add baselineTemplate, do diff when --baseline-template is set
+    if (this.spec.baselineTemplate) {
+      this.logger.debug(`baselineTemplate: ${this.spec.baselineTemplate}`);
+      this.parseSpecInstanceBaseline = new ParseSpec(get(this.spec, 'baselineTemplate'), {
+        argv: this.options.args,
+        logger: this.logger,
+      });
+      this.baselineSpec = await this.parseSpecInstanceBaseline.start();
+      this.diffs = getDiffs(get(this.spec, 'yaml.content'), get(this.baselineSpec, 'yaml.content')) || [];
+    }
     // 初始化行参环境变量 > .env (parse-spec require .env)
     each(this.options.env, (value, key) => {
       process.env[key] = value;
@@ -645,6 +668,7 @@ class Engine {
         const res = await new Credential({ logger: this.logger }).get(item.access);
         return get(res, 'credential', {});
       },
+      diffs: filter(this.diffs, (diff) => { return diff.path?.startsWith(`resources.${item.projectName}`) }),
     };
     this.recordContext(item, { props: newInputs });
     debug(`get props: ${JSON.stringify(result)}`);

packages/engine/src/types.ts

@@ -3,6 +3,7 @@ import { IOptions as ILogConfig } from '@serverless-devs/logger/lib/type';
 import Logger, { ILoggerInstance } from '@serverless-devs/logger';
 import { AssertionError } from 'assert';
 import { DevsError } from '@serverless-devs/utils';
+import { Diff } from 'deep-diff';
 export interface IEngineOptions {
   args?: string[]; //默认 process.argv.slice(2)
   template?: string;
@@ -15,6 +16,10 @@ export interface IEngineOptions {
   serverlessDevsVersion?: string;
 }
 
+export type IDiff = Omit<Diff<Object, Object>, "path"> & {
+  path?: string | undefined;
+}
+
 export type IStepOptions = IStep & {
   logger: ILoggerInstance; // logger实例
   instance?: any; //组件实例

packages/engine/src/utils/index.ts

@@ -8,13 +8,25 @@ import Ajv from 'ajv';
 import loadComponent from '@serverless-devs/load-component';
 import loadApplication from '@serverless-devs/load-application';
 import path from 'path';
+import deepDiff from 'deep-diff';
 
 export function getLogPath(filePath: string) {
   return `step_${filePath}.log`;
 }
 
 export const randomId = () => Math.random().toString(16).slice(2);
 
+export const getDiffs = (leftObject: Object, rightObject: Object) => {
+  const diffs = deepDiff.diff(leftObject, rightObject);
+  const formattedDiffs = diffs?.map((diff) => {
+    return {
+      ...diff,
+      path: diff.path?.join('.'),
+    }
+  });
+  return formattedDiffs || [];
+};
+
 export function getProcessTime(time: number) {
   return (Math.round((Date.now() - time) / 10) * 10) / 1000;
 }
@@ -54,6 +66,11 @@ export const getAllowFailure = (allowFailure: boolean | IAllowFailure | undefine
   if (typeof allowFailure === 'boolean') {
     return allowFailure;
   }
+  // 20240730: feat: when -o, --output-format exist, allow failure
+  const args = process.argv.slice(2);
+  if (args.includes('-o') || args.includes('--output-format')) {
+    return true;
+  }
   if (typeof allowFailure !== 'object') return false;
   if ('exit_code' in allowFailure && 'command' in allowFailure) {
     return includes(get(allowFailure, 'exit_code'), get(data, 'exitCode')) && includes(get(allowFailure, 'command'), get(data, 'command'));

packages/load-application/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serverless-devs/load-application",
-  "version": "0.0.14-beta.5",
+  "version": "0.0.14",
   "description": "load application for serverless-devs",
   "main": "lib/index.js",
   "scripts": {
@@ -18,10 +18,11 @@
   },
   "dependencies": {
     "@serverless-cd/debug": "^4.3.4",
-    "@serverless-devs/art-template": "^4.13.16-beta.12",
+    "@serverless-devs/art-template": "^4.13.16-beta.20",
     "@serverless-devs/credential": "workspace:^",
     "@serverless-devs/downloads": "workspace:^",
     "@serverless-devs/utils": "workspace:^",
+    "@serverless-devs/secret": "workspace:^",
     "art-template": "^4.13.2",
     "axios": "^1.6.0",
     "chalk": "^4.1.2",

packages/load-application/src/constant.ts

@@ -16,4 +16,5 @@ export const CONFIGURE_LATER = 'configure_later';
 export const DEFAULT_MAGIC_ACCESS = '{{ access }}';
 export const REGX = /\${([\w\W]*?)}/;
 
-
+// Dipper
+export const DIPPER_VARIABLES_PATH = 'variable.yaml'

packages/load-application/src/index.ts

@@ -4,8 +4,7 @@ import assert from 'assert';
 import { IOptions } from './types';
 import { includes, get } from 'lodash';
 import { REGISTRY } from './constant';
-import { isDevsDebugMode } from '@serverless-devs/utils';
-const debug = isDevsDebugMode() ? require('@serverless-cd/debug')('serverless-devs:load-application') : (i: any) => {};
+const debug = require('@serverless-cd/debug')('serverless-devs:load-application');
 
 export default async (template: string, options: IOptions = {}) => {
   debug(`load application, template: ${template}, options: ${JSON.stringify(options)}`);

packages/load-application/src/types.ts

@@ -43,6 +43,10 @@ export interface IOptions {
    * Whether overwrite all files in the destination directory
    */
   overwrite?: boolean;
+  /**
+   * Whether to use for inner apis
+   */
+  inner?: boolean;
 }
 
 export enum IProvider {

packages/load-application/src/utils/index.ts

@@ -1,6 +1,7 @@
 import { keys, replace, split } from 'lodash';
 import { RANDOM_PATTERN, REGISTRY, GITHUB_REGISTRY } from '../constant';
 import Credential from '@serverless-devs/credential';
+import SecretManager from '@serverless-devs/secret';
 
 export { default as getInputs } from './get-inputs';
 
@@ -32,3 +33,8 @@ export const getDefaultValue = (value: any) => {
   if (typeof value !== 'string') return;
   return replace(value, RANDOM_PATTERN, randomId());
 };
+
+export const getSecretManager = () => {
+  const secretManager = SecretManager.getInstance();
+  return secretManager;
+};

packages/load-application/src/v2.ts

@@ -3,7 +3,7 @@ import fs from 'fs-extra';
 import axios from 'axios';
 import download from '@serverless-devs/downloads';
 import artTemplate from 'art-template';
-import { getYamlContent, isCiCdEnvironment, getYamlPath, isDevsDebugMode } from '@serverless-devs/utils';
+import { getYamlContent, isCiCdEnvironment, getYamlPath } from '@serverless-devs/utils';
 import { isEmpty, includes, split, get, has, set, sortBy, map, concat, keys, find, startsWith } from 'lodash';
 import parse from './parse';
 import { IProvider, IOptions } from './types';
@@ -14,7 +14,7 @@ import chalk from 'chalk';
 import Credential from '@serverless-devs/credential';
 import { gray } from './constant';
 import assert from 'assert';
-const debug = isDevsDebugMode() ? require('@serverless-cd/debug')('serverless-devs:load-application') : (i: any) => {};
+const debug = require('@serverless-cd/debug')('serverless-devs:load-application');
 
 class LoadApplication {
   private provider: `${IProvider}`;