Separate unit and integration tests.

.github/workflows/test-tools.yml

@@ -46,11 +46,5 @@ jobs:
       - name: Lint
         run: npm run lint
 
-      - name: Test Spec Merger
-        run: npm run test -- tools/tests/merger
-
-      - name: Test Spec Linter
-        run: npm run test -- tools/tests/linter
-
-      - name: Test Spec Tester
-        run: npm run test -- tools/tests/tester --runInBand
+      - name: Tests
+        run: npm run test

CHANGELOG.md

@@ -14,6 +14,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added API coverage ([#210](https://github.com/opensearch-project/opensearch-api-specification/pull/210))
 - Added license headers to TypeScript code ([#311](https://github.com/opensearch-project/opensearch-api-specification/pull/311))
 - Added `npm run test:spec -- --dry-run --verbose` ([#303](https://github.com/opensearch-project/opensearch-api-specification/pull/303))
+- Added `npm run test:unit` and `test:integ` ([#320](https://github.com/opensearch-project/opensearch-api-specification/pull/320))
 
 ### Changed
 

DEVELOPER_GUIDE.md

@@ -383,6 +383,8 @@ Specify the test path to run tests for one of the tools:
 npm run test -- tools/tests/tester/
 ```
 
+The test suite contains unit tests and integration tests. Integration tests, such as [these](tools/tests/tester/integ/), require a local instance of OpenSearch and are placed into a folder named `integ`. Unit tests are run in parallel and integration tests are run sequentially using `--runInBand`. You can run unit tests with `npm run test:unit` separately from integration tests with `npm run test:integ`.
+
 #### Lints
 
 All code is linted using [ESLint](https://eslint.org/) in combination with [typescript-eslint](https://typescript-eslint.io/). Linting can be run via:

package.json

@@ -11,7 +11,9 @@
     "lint": "eslint .",
     "lint--fix": "eslint . --fix",
     "merge": "ts-node tools/src/merger/merge.ts",
-    "test": "jest",
+    "test": "npm run test:unit && npm run test:integ",
+    "test:unit": "jest --testMatch='**/*.test.ts' --testPathIgnorePatterns=/integ/",
+    "test:integ": "jest --testMatch='**/integ/*.test.ts' --runInBand",
     "test:spec": "ts-node tools/src/tester/start.ts"
   },
   "dependencies": {

tools/tests/tester/helpers.test.ts

@@ -0,0 +1,25 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import { overall_result } from '../../src/tester/helpers'
+import { type Evaluation, Result } from '../../src/tester/types/eval.types'
+
+describe('helpers', () => {
+  function e (...results: Result[]): Evaluation[] {
+    return results.map(result => ({ result }))
+  }
+
+  test('overall_result', () => {
+    expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED, Result.ERROR))).toBe(Result.ERROR)
+    expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED))).toBe(Result.FAILED)
+    expect(overall_result(e(Result.PASSED, Result.SKIPPED))).toBe(Result.SKIPPED)
+    expect(overall_result(e(Result.PASSED))).toBe(Result.PASSED)
+    expect(overall_result(e())).toBe(Result.PASSED)
+  })
+})

tools/tests/tester/StoryEvaluator.test.ts → tools/tests/tester/integ/StoryEvaluator.test.ts

@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { construct_tester_components, load_actual_evaluation, load_expected_evaluation } from './helpers'
+import { construct_tester_components, load_actual_evaluation, load_expected_evaluation } from '../helpers'
 
 const { story_evaluator } = construct_tester_components('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
 

tools/tests/tester/TestRunner.test.ts → tools/tests/tester/integ/TestRunner.test.ts

@@ -7,8 +7,8 @@
 * compatible open source license.
 */
 
-import { construct_tester_components, flatten_errors, load_expected_evaluation } from './helpers'
-import { type StoryEvaluation } from '../../src/tester/types/eval.types'
+import { construct_tester_components, flatten_errors, load_expected_evaluation } from '../helpers'
+import { type StoryEvaluation } from '../../../src/tester/types/eval.types'
 
 test('stories folder', async () => {
   const { test_runner } = construct_tester_components('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')