Test scenario main (Azure#599)

Support test scenario and test runner

.gitignore

@@ -89,6 +89,14 @@ dist/
 coverage/
 .nyc_output
 test-results.xml
-
+newman/
+fixtures/
+lib/testScenario/_testAKS.ts
+snapshot.patch
 .clinic/*
 junit.xml
+generated_collections/
+env.json
+generated_examples/
+generated_reports/
+generated_diff/

.vscode/launch.json

@@ -4,6 +4,18 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "address": "TCP/IP address of process to be debugged",
+      "localRoot": "${workspaceFolder}",
+      "name": "Attach to Remote",
+      "port": 9229,
+      "remoteRoot": "Absolute path to the remote directory containing the program",
+      "request": "attach",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "type": "pwa-node"
+    },
     {
       "type": "node",
       "request": "launch",
@@ -135,7 +147,7 @@
       "type": "node",
       "request": "attach",
       "name": "Attach to Process",
-      "port": 5858
+      "processId": "${command:PickProcess}"
     }
   ]
 }

ChangeLog.md

@@ -1,5 +1,9 @@
 # Change Log - oav
 
+## 03/24/2021 2.3.0
+
+- Support load test-scenario file
+- Support generate postman collection from test-scenario
 ## 02/08/2021 2.2.7
 
 - Add new rules of 'LRO_RESPONSE_HEADER' and 'LRO_RESPONSE_CODE'.

azure-pipelines.yml

@@ -12,6 +12,13 @@ steps:
     inputs:
       verbose: false
 
+  - task: Npm@1
+    displayName: "npm run build"
+    inputs:
+      command: custom
+      verbose: false
+      customCommand: run build
+
   - task: Npm@1
     displayName: "npm test"
     inputs:

cli.ts

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 
-/* tslint:disable */
-
+import "reflect-metadata";
 import * as yargs from "yargs";
 import { log } from "./lib/util/logging";
 
 const defaultLogDir = log.directory;
 
 const packageVersion = require("../package.json").version;
 
+// eslint-disable-next-line no-unused-expressions
 yargs
   .version(packageVersion)
   .commandDir("lib/commands")
@@ -31,7 +31,7 @@ yargs
     describe: `Pretty print`,
   })
   .global(["h", "l", "f", "p"])
-  .help();
+  .help().argv;
 
 if (yargs.argv._.length === 0 && yargs.argv.h === false) {
   yargs.coerce("help", (_) => true);

index.ts

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
+import "reflect-metadata";
 import * as C from "./lib/util/constants";
 
 // Easy to use methods from validate.ts
@@ -54,13 +55,20 @@ export {
   TestScenario,
   TestStep,
   TestStepArmTemplateDeployment,
-  TestStepExampleFileRestCall,
+  TestStepRestCall,
 } from "./lib/testScenario/testResourceTypes";
 export { VariableEnv } from "./lib/testScenario/variableEnv";
 export {
   TestScenarioRunner,
   TestScenarioRunnerClient,
 } from "./lib/testScenario/testScenarioRunner";
 
+export { PostmanCollectionRunnerClient } from "./lib/testScenario/postmanCollectionRunnerClient";
+export {
+  PostmanCollectionGenerator,
+  PostmanCollectionGeneratorOption,
+} from "./lib/testScenario/postmanCollectionGenerator";
+
+export { getAutorestConfig } from "./lib/util/getAutorestConfig";
 // Constants
 export const Constants = C;

jest.config.js

@@ -17,6 +17,7 @@ module.exports = {
       tsConfig: "tsconfig.json",
     },
   },
+  setupFilesAfterEnv: ["<rootDir>/jest.setup-file.js"],
   testMatch: ["**/test/**/*.ts", "!**/test/**/*.d.ts", "!**/test/sample.ts"],
   verbose: true,
 };

jest.setup-file.js

@@ -0,0 +1 @@
+require("reflect-metadata");

lib/commands/example-quality.ts

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+import * as yargs from "yargs";
+
+import { ExampleQualityValidator } from "../exampleQualityValidator/exampleQualityValidator";
+import { cliSuppressExceptions } from "../cliSuppressExceptions";
+import { log } from "../util/logging";
+
+export const command = "example-quality <spec-path>";
+
+export const describe =
+  "Performs example quality validation of x-ms-examples and examples present in the spec.";
+
+export async function handler(argv: yargs.Arguments): Promise<void> {
+  await cliSuppressExceptions(async () => {
+    log.debug(argv.toString());
+    const specPath = argv.specPath;
+    const validator = ExampleQualityValidator.create({
+      swaggerFilePaths: [specPath],
+    });
+    const res = await validator.validateSwaggerExamples();
+    console.log(JSON.stringify(res, null, 2));
+    return 0;
+  });
+}

lib/commands/generate-collection.ts

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+/* eslint-disable id-blacklist */
+
+import * as fs from "fs";
+import * as path from "path";
+import * as yargs from "yargs";
+import {
+  PostmanCollectionGenerator,
+  PostmanCollectionGeneratorOption,
+} from "../testScenario/postmanCollectionGenerator";
+
+import { cliSuppressExceptions } from "../cliSuppressExceptions";
+import { getAutorestConfig } from "../util/getAutorestConfig";
+export const command = "generate-collection [testScenario-path]";
+
+export const describe = "Generate postman collection file from test scenario.";
+
+export const builder: yargs.CommandBuilder = {
+  tag: {
+    describe: "the readme tag name.",
+    string: true,
+  },
+  output: {
+    alias: "outputDir",
+    describe: "the output folder.",
+    string: true,
+    default: "generated_collections",
+  },
+  readme: {
+    describe: "path to readme.md file",
+    string: true,
+    demandOption: true,
+  },
+  e: {
+    alias: "envFile",
+    describe: "the env file path.",
+    string: true,
+  },
+};
+
+export async function handler(argv: yargs.Arguments): Promise<void> {
+  await cliSuppressExceptions(async () => {
+    const readmeMd: string = argv.readme;
+    argv["try-require"] = "readme.test.md";
+    const autorestConfig = await getAutorestConfig(argv, readmeMd);
+    console.log(autorestConfig["input-file"]);
+    console.log(autorestConfig["test-resources"]);
+    if (autorestConfig["test-resources"] === undefined) {
+      throw new Error(`No test-scenario file found in '${argv.tag || "default"}'`);
+    }
+    const swaggerFilePaths: string[] = autorestConfig["input-file"];
+    for (const testResources of autorestConfig["test-resources"]) {
+      const testScenarioFile = testResources.test;
+      const fileRoot: string = path.dirname(readmeMd);
+      let env = {
+        subscriptionId: "<mySubcriptionId>",
+        location: "westus",
+      };
+      if (argv.e !== undefined) {
+        env = JSON.parse(fs.readFileSync(argv.e).toString());
+      }
+      console.log(
+        `generating postman collection from ${testScenarioFile}. outputDir: ${argv.output}`
+      );
+      const opt: PostmanCollectionGeneratorOption = {
+        name: testScenarioFile.replace(/^.*[\\\/]/, "").replace(".yaml", ""),
+        testDef: testScenarioFile,
+        swaggerFilePaths: swaggerFilePaths,
+        fileRoot: fileRoot,
+        env: env,
+        outputFolder: argv.output,
+      };
+      if (!fs.existsSync(argv.output)) {
+        fs.mkdirSync(argv.output);
+      }
+      const generator = new PostmanCollectionGenerator(opt);
+      await generator.GenerateCollection();
+    }
+    return 0;
+  });
+}

lib/commands/generate-report.ts

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+/* eslint-disable id-blacklist */
+
+import * as fs from "fs";
+import * as path from "path";
+import * as yargs from "yargs";
+
+import { cliSuppressExceptions } from "../cliSuppressExceptions";
+import { getAutorestConfig } from "../util/getAutorestConfig";
+import { ReportGenerator } from "../testScenario/reportGenerator";
+export const command = "generate-report [raw-report-path]";
+
+export const describe = "Generate report from postman report.";
+
+export const builder: yargs.CommandBuilder = {
+  newmanReport: {
+    describe: "The newman report",
+    string: true,
+  },
+  tag: {
+    describe: "the readme tag name.",
+    string: true,
+  },
+  output: {
+    alias: "outputDir",
+    describe: "the output folder.",
+    string: true,
+    default: "generated",
+  },
+  readme: {
+    describe: "path to readme.md file",
+    string: true,
+    demandOption: true,
+  },
+};
+
+export async function handler(argv: yargs.Arguments): Promise<void> {
+  await cliSuppressExceptions(async () => {
+    const readmeMd: string = argv.readme;
+    const newmanReport: string = argv.newmanReport;
+    argv["try-require"] = "readme.test.md";
+    const autorestConfig = await getAutorestConfig(argv, readmeMd);
+    const fileRoot: string = path.dirname(readmeMd);
+    const swaggerFilePaths: string[] = autorestConfig["input-file"];
+    const testScenarioFile = autorestConfig["test-resources"][0].test;
+    console.log(
+      `generating report from ${newmanReport}. test-scenario: ${testScenarioFile} outputDir: ${argv.output}`
+    );
+    if (!fs.existsSync(argv.output)) {
+      fs.mkdirSync(argv.output);
+    }
+    const generator = new ReportGenerator(
+      newmanReport,
+      argv.output,
+      fileRoot,
+      swaggerFilePaths,
+      testScenarioFile,
+      readmeMd,
+      argv.tag || "default"
+    );
+    await generator.generateReport();
+    console.log(`generate report successfully!`);
+    return 0;
+  });
+}

lib/commands/generate-test-scenario.ts

@@ -0,0 +1,81 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+/* eslint-disable id-blacklist */
+
+import { dirname, relative as pathRelative, resolve as pathResolve } from "path";
+import globby from "globby";
+import * as yargs from "yargs";
+import { inversifyGetInstance } from "../inversifyUtils";
+import { TestRecordingLoader } from "../testScenario/gen/testRecordingLoader";
+import { RequestTracking, TestScenarioGenerator } from "../testScenario/gen/testScenarioGenerator";
+import { getAutorestConfig } from "../util/getAutorestConfig";
+
+export const command = "generate-test-scenario";
+export const describe = "Generate swagger examples from real payload records.";
+
+export const builder: yargs.CommandBuilder = {
+  tag: {
+    describe: "the readme tag name.",
+    string: true,
+  },
+  recording: {
+    describe:
+      "path glob pattern for recording files. Supported format: DotNet recording, Azure Powershell recording, Azure Cli recording",
+    string: true,
+    demandOption: true,
+  },
+  readme: {
+    describe: "path to readme.md file",
+    string: true,
+    demandOption: true,
+  },
+  output: {
+    describe: "path to output test scenario",
+    string: true,
+    demandOption: true,
+  },
+};
+
+export async function handler(argv: yargs.Arguments): Promise<void> {
+  const readmeMd: string = argv.readme;
+  let output: string = argv.output;
+  const recording: string = argv.recording;
+  argv["try-require"] = "readme.test.md";
+
+  const recordingFilePaths = await globby(recording);
+  recordingFilePaths.sort();
+  const autorestConfig = await getAutorestConfig(argv, readmeMd);
+  const swaggerFilePaths: string[] = autorestConfig["input-file"];
+  const fileRoot = dirname(readmeMd);
+  output = pathResolve(fileRoot, output);
+  output = pathRelative(fileRoot, output);
+
+  console.log("input-file:");
+  console.log(swaggerFilePaths);
+  console.log("recording-file:");
+  console.log(recordingFilePaths);
+  console.log("output-file:");
+  console.log(`\t${output}\n`);
+
+  const generator = TestScenarioGenerator.create({
+    useJsonParser: false,
+    checkUnderFileRoot: false,
+    fileRoot,
+    swaggerFilePaths,
+  });
+
+  await generator.initialize();
+
+  const recordingLoader = inversifyGetInstance(TestRecordingLoader, {});
+  const trackingList: RequestTracking[] = [];
+  for (const filePath of recordingFilePaths) {
+    console.log(`Transforming:\t${filePath}`);
+    const tracking = await recordingLoader.load(filePath);
+    trackingList.push(tracking);
+  }
+  await generator.generateTestDefinition(trackingList, output);
+
+  await generator.writeGeneratedFiles();
+  process.exit(0);
+}