Add tool to calculate coverage

Signed-off-by: Thomas Farr <[email protected]>

.github/workflows/determine-changes.yml → .github/workflows/analyze-pr-changes.yml

@@ -1,7 +1,10 @@
-name: Determine API Changes
+name: Analyze PR Changes
 
 on: [pull_request]
 
+env:
+  OPENSEARCH_INITIAL_ADMIN_PASSWORD: BobgG7YrtsdKf9M
+
 jobs:
   determine-changes:
     runs-on: ubuntu-latest
@@ -13,6 +16,34 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Build Docker Image
+        run: docker build coverage --tag opensearch-with-api-plugin
+
+      - name: Run OpenSearch
+        shell: bash -eo pipefail {0}
+        run: |
+          docker run \
+            --name opensearch \
+            --rm -d \
+            -p 9200:9200 -p 9600:9600 \
+            -e "discovery.type=single-node" \
+            -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="$OPENSEARCH_INITIAL_ADMIN_PASSWORD" \
+            opensearch-with-api-plugin
+          
+          for attempt in {1..20}; do
+            sleep 5
+            if curl -ksS -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" https://localhost:9200; then
+              echo '=> ready'
+              exit 0
+            fi
+            echo '=> waiting...'
+          done
+          exit 1
+
+      - name: Dump Cluster's API
+        shell: bash -eo pipefail {0}
+        run: curl -ksS -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" https://localhost:9200/_plugins/api | jq > /tmp/opensearch-openapi-CLUSTER.json
+
       - name: Determine Branch Point
         shell: bash -eo pipefail {0}
         run: |
@@ -44,15 +75,35 @@ jobs:
           npm install
           npm run merge -- --source ./spec --output /tmp/opensearch-openapi-CHANGED.yaml
 
+      - name: Calculate Coverage
+        shell: bash -eo pipefail {0}
+        run: |
+          for variant in ORIGINAL CHANGED ; do
+            echo "Calculating coverage of ${variant}"
+            npm run coverage:spec -- \
+              --cluster /tmp/opensearch-openapi-CLUSTER.json \
+              --specification /tmp/opensearch-openapi-${variant}.yaml \
+              --output /tmp/coverage-api-${variant}.json
+          done
+
+      - name: Upload Coverage Data
+        id: upload-coverage
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-api
+          path: |
+            /tmp/coverage-api-ORIGINAL.json
+            /tmp/coverage-api-CHANGED.json
+
       - name: Install openapi-changes
         shell: bash -eo pipefail {0}
         run: npm install --global @pb33f/openapi-changes
 
-      - name: Generate Report
+      - name: Generate HTML Report
         shell: bash -eo pipefail {0}
         run: openapi-changes html-report --no-logo --no-color /tmp/opensearch-openapi-ORIGINAL.yaml /tmp/opensearch-openapi-CHANGED.yaml
 
-      - name: Upload Report
+      - name: Upload HTML Report
         id: upload-report
         uses: actions/upload-artifact@v4
         with:

package.json

@@ -5,6 +5,7 @@
   "author": "opensearch-project",
   "license": "Apache-2.0",
   "scripts": {
+    "coverage:spec": "ts-node tools/src/coverage/coverage.ts",
     "merge": "ts-node tools/src/merger/merge.ts",
     "lint:spec": "ts-node tools/src/linter/lint.ts",
     "lint": "eslint .",

tools/helpers.ts

@@ -1,6 +1,9 @@
 import fs from 'fs'
 import YAML from 'yaml'
 import _ from 'lodash'
+import { OpenAPIV3 } from 'openapi-types'
+
+export const HTTP_METHODS: OpenAPIV3.HttpMethods[] = Object.values(OpenAPIV3.HttpMethods)
 
 export function resolve_ref (ref: string, root: Record<string, any>): Record<string, any> | undefined {
   const paths = ref.replace('#/', '').split('/')

tools/src/coverage/CoverageCalculator.ts

@@ -0,0 +1,81 @@
+import fs from 'fs'
+import { type OpenAPIV3 } from 'openapi-types'
+import { HTTP_METHODS, read_yaml } from '../../helpers'
+
+export default class CoverageCalculator {
+  private readonly _cluster_spec: OpenAPIV3.Document
+  private readonly _input_spec: OpenAPIV3.Document
+  private readonly _output_path: string
+
+  constructor (cluster_spec_path: string, input_spec_path: string, output_path: string) {
+    this._cluster_spec = read_yaml(cluster_spec_path) as OpenAPIV3.Document
+    this._input_spec = read_yaml(input_spec_path) as OpenAPIV3.Document
+    this._output_path = output_path
+  }
+
+  calculate (): void {
+    type Endpoints = Record<string, Set<OpenAPIV3.HttpMethods>>
+    const collect = (document: OpenAPIV3.Document): Endpoints =>
+      Object.fromEntries(
+        Object.entries(document.paths)
+          .map(([path, path_item]): [string, Set<OpenAPIV3.HttpMethods>] => {
+            // Sanitize path params to ignore naming of params in route templates
+            path = path.replaceAll(/\{[^}]+}/g, '{}')
+            if (path_item == null) return [path, new Set()]
+            return [path, new Set(HTTP_METHODS.filter(method => path_item[method] != null))]
+          })
+      )
+    const count = (endpoints: Endpoints): number =>
+      Object.values(endpoints).map(methods => methods.size).reduce((acc, v) => acc + v, 0)
+    const prune = (endpoints: Endpoints): Endpoints =>
+      Object.fromEntries(Object.entries(endpoints).filter(([_, methods]) => methods.size > 0))
+
+    const uncovered = collect(this._cluster_spec)
+    const specified_but_not_provided = collect(this._input_spec)
+    const covered: Endpoints = {}
+
+    for (const [path, methods] of Object.entries(uncovered)) {
+      if (specified_but_not_provided[path] === undefined) continue
+
+      for (const method of [...methods]) {
+        if (!specified_but_not_provided[path].delete(method)) continue
+
+        if (covered[path] === undefined) covered[path] = new Set()
+        covered[path].add(method)
+        uncovered[path].delete(method)
+      }
+    }
+
+    const uncovered_count = count(uncovered)
+    const covered_count = count(covered)
+    const total_count = uncovered_count + covered_count
+
+    const json = JSON.stringify(
+      {
+        $description: {
+          uncovered: 'Endpoints provided by the OpenSearch cluster but DO NOT exist in the specification',
+          covered: 'Endpoints both provided by the OpenSearch cluster and exist in the specification',
+          specified_but_not_provided: 'Endpoints NOT provided by the OpenSearch cluster but exist in the specification'
+        },
+        counts: {
+          uncovered: uncovered_count,
+          uncovered_pct: Math.round(uncovered_count / total_count * 100 * 100) / 100,
+          covered: covered_count,
+          covered_pct: Math.round(covered_count / total_count * 100 * 100) / 100,
+          specified_but_not_provided: count(specified_but_not_provided)
+        },
+        endpoints: {
+          uncovered: prune(uncovered),
+          covered: prune(covered),
+          specified_but_not_provided: prune(specified_but_not_provided)
+        }
+      },
+      (_, value) => {
+        if (value instanceof Set) return [...value]
+        return value
+      },
+      2)
+
+    fs.writeFileSync(this._output_path, json)
+  }
+}

tools/src/coverage/coverage.ts

@@ -0,0 +1,15 @@
+import { Command, Option } from '@commander-js/extra-typings'
+import CoverageCalculator from './CoverageCalculator'
+import { resolve } from 'path'
+
+const command = new Command()
+  .description('Calculates the coverage of a specification against an OpenSearch clusters generated specification.')
+  .addOption(new Option('--cluster <path>', 'path to the cluster\'s generated specification.').makeOptionMandatory())
+  .addOption(new Option('--specification <path>', 'path to the specification to calculate coverage of.').makeOptionMandatory())
+  .addOption(new Option('-o, --output <path>', 'path to the output file.').default(resolve(__dirname, '../../../build/coverage.json')))
+  .allowExcessArguments(false)
+  .parse()
+
+const opts = command.opts()
+const calculator = new CoverageCalculator(opts.cluster, opts.specification, opts.output)
+calculator.calculate()