Merge pull request clearlydefined#80 from qtomlinson/qt/auto-detect-s…

…chema-versions

Add auto detect schema versions

tools/integration/README.md

@@ -14,7 +14,7 @@
 
    - Components to be harvested,
    - Base URLs for the development and production systems, along with polling interval and timeout settings,
-   - Current harvest schema versions. This is for polling harvest results to check whether the harvest is complete. When scan tool versions are updated, these need to be updated as well.
+   - Current harvest tools. This is used for polling harvest results to check whether the harvest is complete. When scan tools are added or removed during the harvest process, this list needs to be updated as well.
 
 1. Test fixtures are grouped by endpoints at [./test/integration/fixtures](./test/integration/fixtures). You can use these fixtures to override responses from the production system when necessary.
 1. The classes used in the integration tests are located at [./lib](./lib). Tests for those tooling classes are located at ./test/lib. Run `npm test` to test the tooling classes.

tools/integration/lib/harvestResultFetcher.js

@@ -0,0 +1,78 @@
+// (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
+// SPDX-License-Identifier: MIT
+const { callFetch } = require('./fetch')
+
+const defaultTools = ['licensee', 'reuse', 'scancode']
+
+class HarvestResultFetcher {
+  constructor(apiBaseUrl, coordinates, fetch = callFetch) {
+    this.apiBaseUrl = apiBaseUrl
+    this._fetch = fetch
+    this._coordinates = coordinates
+  }
+
+  async fetchToolVersions(tools = defaultTools) {
+    const listHarvestResultApi = `${this.apiBaseUrl}/harvest/${this._coordinates}?form=list`
+    const harvestResultUrls = await this._fetch(listHarvestResultApi).then(r => r.json())
+    return tools.flatMap(tool =>
+      harvestResultUrls
+        .filter(url => url.includes(`/${tool}/`))
+        .map(url => url.substring(`${this._coordinates}/${tool}/`.length))
+        .map(version => [tool, version])
+    )
+  }
+
+  async _pollForCompletion(poller) {
+    try {
+      const completed = await poller.poll()
+      console.log(`Completed ${this._coordinates}: ${completed}`)
+      return completed
+    } catch (error) {
+      if (error.code === 'ECONNREFUSED') throw error
+      console.log(`Error polling for ${this._coordinates}: ${error}`)
+      return false
+    }
+  }
+
+  async pollForToolVersionsComplete(poller, startTime, tools) {
+    const statuses = new Map()
+    poller.with(async () => {
+      const toolVersions = await this.fetchToolVersions(tools)
+      return this.isHarvestComplete(toolVersions, startTime, statuses)
+    })
+    const completed = await this._pollForCompletion(poller)
+    if (!completed) throw new Error(`Schema versions not detected`)
+    return [...statuses.entries()].map(([k, v]) => [k, v.toolVersion])
+  }
+
+  async pollForHarvestComplete(poller, toolVersions, startTime) {
+    const statuses = new Map()
+    poller.with(async () => this.isHarvestComplete(toolVersions, startTime, statuses))
+    return this._pollForCompletion(poller)
+  }
+
+  async isHarvestComplete(toolVersions, startTime, statuses = new Map()) {
+    const harvestChecks = (toolVersions || []).map(async ([tool, toolVersion]) => {
+      const completed = statuses.get(tool)?.completed || (await this.isHarvestedbyTool(tool, toolVersion, startTime))
+      if (completed) statuses.set(tool, { toolVersion, completed })
+      return tool
+    })
+    return Promise.all(harvestChecks).then(tools => tools.every(tool => statuses.get(tool)?.completed))
+  }
+
+  async isHarvestedbyTool(tool, toolVersion, startTime = 0) {
+    const harvested = await this.fetchHarvestResult(tool, toolVersion)
+    if (!harvested._metadata) return false
+    const fetchedAt = new Date(harvested._metadata.fetchedAt)
+    console.log(`${this._coordinates} ${tool}, ${toolVersion} fetched at ${fetchedAt}`)
+    return fetchedAt.getTime() > startTime
+  }
+
+  async fetchHarvestResult(tool, toolVersion) {
+    return this._fetch(`${this.apiBaseUrl}/harvest/${this._coordinates}/${tool}/${toolVersion}?form=raw`).then(r =>
+      r.headers.get('Content-Length') === '0' ? Promise.resolve({}) : r.json()
+    )
+  }
+}
+
+module.exports = HarvestResultFetcher

tools/integration/lib/harvester.js

@@ -2,28 +2,22 @@
 // SPDX-License-Identifier: MIT
 
 const { callFetch, buildPostOpts } = require('./fetch')
-
-//The versions correspond to the schema versions of the tools which are used in /harvest/{type}/{provider}/{namespace}/{name}/{revision}/{tool}/{toolVersion}
-//See https://api.clearlydefined.io/api-docs/#/harvest/get_harvest__type___provider___namespace___name___revision___tool___toolVersion_
-const defaultToolChecks = [
-  ['licensee', '9.14.0'],
-  ['scancode', '32.3.0'],
-  ['reuse', '3.2.1']
-]
+const HarvestResultFetcher = require('./harvestResultFetcher')
 
 class Harvester {
   constructor(apiBaseUrl, harvestToolChecks, fetch = callFetch) {
     this.apiBaseUrl = apiBaseUrl
-    this.harvestToolChecks = harvestToolChecks || defaultToolChecks
+    this._harvestToolChecks = harvestToolChecks
     this._fetch = fetch
   }
 
   async harvest(components, reharvest = false) {
+    if (components.length === 0) return
     return await this._fetch(`${this.apiBaseUrl}/harvest`, buildPostOpts(this._buildPostJson(components, reharvest)))
   }
 
   _buildPostJson(components, reharvest) {
-    const tool = this.harvestToolChecks.length === 1 ? this.harvestToolChecks[0][0] : 'component'
+    const tool = this._harvestToolChecks?.length === 1 ? this._harvestToolChecks[0][0] : 'component'
     return components.map(coordinates => {
       const result = { tool, coordinates }
       if (reharvest) result.policy = 'always'
@@ -32,52 +26,52 @@ class Harvester {
   }
 
   async pollForCompletion(components, poller, startTime) {
+    if (!this._harvestToolChecks) throw new Error('Harvest tool checks not set')
     const status = new Map()
     for (const coordinates of components) {
       const completed = await this._pollForOneCompletion(coordinates, poller, startTime)
       status.set(coordinates, completed)
     }
 
     for (const coordinates of components) {
-      const completed =
-        status.get(coordinates) || (await this.isHarvestComplete(coordinates, startTime).catch(() => false))
+      const completed = status.get(coordinates) || (await this._isHarvestComplete(coordinates, startTime))
       status.set(coordinates, completed)
     }
     return status
   }
 
   async _pollForOneCompletion(coordinates, poller, startTime) {
-    try {
-      const completed = await poller.poll(async () => this.isHarvestComplete(coordinates, startTime))
-      console.log(`Completed ${coordinates}: ${completed}`)
-      return completed
-    } catch (error) {
-      if (error.code === 'ECONNREFUSED') throw error
-      console.log(`Error polling for ${coordinates}: ${error}`)
-      return false
-    }
+    return this.resultChecker(coordinates).pollForHarvestComplete(poller, this._harvestToolChecks, startTime)
   }
 
-  async isHarvestComplete(coordinates, startTime) {
-    const harvestChecks = this.harvestToolChecks.map(([tool, toolVersion]) =>
-      this.isHarvestedbyTool(coordinates, tool, toolVersion, startTime)
-    )
-
-    return Promise.all(harvestChecks).then(results => results.every(r => r))
+  async _isHarvestComplete(coordinates, startTime) {
+    return this.resultChecker(coordinates)
+      .isHarvestComplete(this._harvestToolChecks, startTime)
+      .catch(error => {
+        console.log(`Error polling for ${coordinates} completion: ${error}`)
+        return false
+      })
   }
 
-  async isHarvestedbyTool(coordinates, tool, toolVersion, startTime = 0) {
-    const harvested = await this.fetchHarvestResult(coordinates, tool, toolVersion)
-    if (!harvested._metadata) return false
-    const fetchedAt = new Date(harvested._metadata.fetchedAt)
-    console.log(`${coordinates} ${tool}, ${toolVersion} fetched at ${fetchedAt}`)
-    return fetchedAt.getTime() > startTime
-  }
+  async detectSchemaVersions(component, poller, tools) {
+    if (!component) throw new Error('Component not set')
+    const startTime = Date.now()
+    //make sure that we have one entire set of harvest results (old or new)
+    await this.harvest([component])
+    //trigger a reharvest to overwrite the old result, so we can verify the timestamp is new for completion
+    await this.harvest([component], true)
 
-  async fetchHarvestResult(coordinates, tool, toolVersion) {
-    return this._fetch(`${this.apiBaseUrl}/harvest/${coordinates}/${tool}/${toolVersion}?form=raw`).then(r =>
-      r.headers.get('Content-Length') === '0' ? Promise.resolve({}) : r.json()
+    const detectedToolVersions = await this.resultChecker(component).pollForToolVersionsComplete(
+      poller,
+      startTime,
+      tools
     )
+    console.log(`Detected schema versions: ${detectedToolVersions}`)
+    this._harvestToolChecks = detectedToolVersions
+  }
+
+  resultChecker(coordinates) {
+    return new HarvestResultFetcher(this.apiBaseUrl, coordinates, this._fetch)
   }
 }
 

tools/integration/lib/poller.js

@@ -7,11 +7,17 @@ class Poller {
     this.maxTime = maxTime
   }
 
-  async poll(activity) {
+  with(activity) {
+    this._activity = activity
+    return this
+  }
+
+  async poll() {
+    if (typeof this._activity !== 'function') throw new Error('Activity not set')
     let counter = 0
     while (counter * this.interval < this.maxTime) {
       console.log(`Polling ${counter}`)
-      const isDone = await activity()
+      const isDone = await this._activity()
       if (isDone) return true
       await new Promise(resolve => setTimeout(resolve, this.interval))
       counter++

tools/integration/test/integration/e2e-test-service/curationTest.js

@@ -3,15 +3,15 @@
 
 const { deepStrictEqual, strictEqual, ok } = require('assert')
 const { callFetch, buildPostOpts } = require('../../../lib/fetch')
-const { devApiBaseUrl, components, definition } = require('../testConfig')
+const { devApiBaseUrl, definition } = require('../testConfig')
 
 describe('Validate curation', function () {
   this.timeout(definition.timeout)
 
   //Rest a bit to avoid overloading the servers
   afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout / 2)))
 
-  const coordinates = components[0]
+  const coordinates = 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16'
 
   describe('Propose curation', function () {
     const [type, provider, namespace, name, revision] = coordinates.split('/')

tools/integration/test/integration/e2e-test-service/definitionTest.js

@@ -118,8 +118,10 @@ function filesToMap(result) {
 
 async function findDefinition(coordinates) {
   const [type, provider, namespace, name, revision] = coordinates.split('/')
+  let coordinatesString = `type=${type}&provider=${provider}&name=${name}`
+  coordinatesString += namespace && namespace !== '-' ? `&namespace=${namespace}` : ''
   const response = await callFetch(
-    `${devApiBaseUrl}/definitions?type=${type}&provider=${provider}&namespace=${namespace}&name=${name}&sortDesc=true&sort=revision`
+    `${devApiBaseUrl}/definitions?${coordinatesString}&sortDesc=true&sort=revision`
   ).then(r => r.json())
   return response.data.find(d => d.coordinates.revision === revision)
 }

tools/integration/test/integration/harvestTest.js

@@ -19,8 +19,14 @@ describe('Tests for harvesting different components', function () {
 })
 
 async function harvestTillCompletion(components) {
-  const { harvestSchemaVersions, poll } = harvest
-  const harvester = new Harvester(devApiBaseUrl, harvestSchemaVersions)
+  if (components.length === 0) return new Map()
+
+  const { poll, tools } = harvest
+  const harvester = new Harvester(devApiBaseUrl)
+
+  const oneComponent = components.shift()
+  const versionPoller = new Poller(poll.interval / 5, poll.maxTime)
+  await harvester.detectSchemaVersions(oneComponent, versionPoller, tools)
 
   //make sure that we have one entire set of harvest results (old or new)
   console.log('Ensure harvest results exist before starting tests')

tools/integration/test/integration/testConfig.js

@@ -5,19 +5,14 @@ const devApiBaseUrl = 'https://dev-api.clearlydefined.io'
 const prodApiBaseUrl = 'https://api.clearlydefined.io'
 
 const pollingInterval = 1000 * 60 * 5 // 5 minutes
-const pollingMaxTime = 1000 * 60 * 30 // 30 minutes
+const pollingMaxTime = 1000 * 60 * 60 // 60 minutes
 
-//Havest results to check for harvest completeness
-//The versions correspond to the schema versions of the tools which are used in /harvest/{type}/{provider}/{namespace}/{name}/{revision}/{tool}/{toolVersion}
-//See https://api.clearlydefined.io/api-docs/#/harvest/get_harvest__type___provider___namespace___name___revision___tool___toolVersion_
-const harvestSchemaVersions = [
-  ['licensee', '9.14.0'],
-  ['scancode', '32.3.0'],
-  ['reuse', '3.2.1']
-]
+//Havest tools to check for harvest completeness
+const harvestTools = ['licensee', 'reuse', 'scancode']
 
 //Components to test
 const components = [
+  'pypi/pypi/-/platformdirs/4.2.0', //Keep this as the first element to test, it is relatively small
   'maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16',
   'maven/mavengoogle/android.arch.lifecycle/common/1.0.1',
   'maven/gradleplugin/io.github.lognet/grpc-spring-boot-starter-gradle-plugin/4.6.0',
@@ -26,7 +21,6 @@ const components = [
   'npm/npmjs/-/redis/0.1.0',
   'git/github/ratatui-org/ratatui/bcf43688ec4a13825307aef88f3cdcd007b32641',
   'gem/rubygems/-/sorbet/0.5.11226',
-  'pypi/pypi/-/platformdirs/4.2.0',
   'pypi/pypi/-/sdbus/0.12.0',
   'go/golang/rsc.io/quote/v1.3.0',
   'nuget/nuget/-/NuGet.Protocol/6.7.1',
@@ -42,9 +36,9 @@ module.exports = {
   prodApiBaseUrl,
   components,
   harvest: {
-    poll: { interval: pollingInterval, maxTime: pollingMaxTime },
-    harvestSchemaVersions,
-    timeout: 1000 * 60 * 60 * 2 // 2 hours for harvesting all the components
+    poll: { interval: pollingInterval, maxTime: pollingMaxTime }, // for each component
+    tools: harvestTools,
+    timeout: 1000 * 60 * 60 * 4 // 4 hours for harvesting all the components
   },
   definition: {
     timeout: 1000 * 10 // for each component