Fixing the CI and addition bugs found along the way (#79)

* Fixing issues with GitHub updating the docker compose version and the docker compose syntax no longer working.
* Pinning the docker compose compatible file for the integration tests.
* Adding a better way to wait for the Report Portal services to be up.
* Changing the method for creating an API token for the integration tests as it's deprecated.
* Changing the way we are verifying the logs for the tests as it was inaccurate.
* Fixing issues with JSON objects not being displayed properly in the Report Portal logs.
* General improvement of the tests and the testing infrastructure.

.github/workflows/ci.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Docker version
         run: docker -v
       - name: Setting up a local Report Portal environment
-        run: sudo docker-compose -f docker-compose.yml -p reportportal up -d --force-recreate
+        run: sudo docker compose -f docker-compose.yml -p reportportal up -d --force-recreate
       - name: Waiting for services to be up
         run: ./scripts/healthcheck.bash
       - name: Report Portal server status

package.json

@@ -31,6 +31,7 @@
     "run-full-unit-tests": "npm run build-local-reporter && npm run unit-tests && npm run unit-retry-tests && npm run unit-no-live-reporting-tests && npm run unit-display-debug-logs-tests",
     "integration-tests": "npm run test ./tests/integration/integration.executor.ts -- --display-debug-logs",
     "run-integration-tests": "npm run build-local-reporter && npm run integration-tests",
+    "download-report-portal-docker-compose-file": "curl https://raw.githubusercontent.com/reportportal/reportportal/a9f2fe18dfe9cb2a631d8a60fdbe254ced6963e7/docker-compose.yml -o docker-compose.yml",
     "download-report-portal-latest-docker-compose-file": "curl https://raw.githubusercontent.com/reportportal/reportportal/master/docker-compose.yml -o docker-compose.yml",
     "up": "docker compose -p reportportal up",
     "start": "docker compose -p reportportal start",

scripts/healthcheck.bash

@@ -1,12 +1,14 @@
 #!/bin/bash
-set -eux
+while true; do
+  # Run the command and capture the output
+  output=$(npm run status)
 
-wait-for-status() {
-    echo "Waiting for services status"
-    echo ""
-    bash -c \
-    'while [[ "$(npm run status)" == *"(starting)"* ]];\
-    do echo "Waiting for services" && sleep 2;\
-    done'
-}
-wait-for-status
+  # Check if the output contains 'starting' or 'exited'
+  if echo "$output" | grep -q -e "starting" -e "exited"; then
+    echo "Detected 'starting' or 'exited' in the output. Sleeping for 2 seconds..."
+    sleep 2
+  else
+    echo "No 'starting' or 'exited' found in the output. Continuing..."
+    break
+  fi
+done

src/uat.js → src/api-testing-client.js

@@ -1,6 +1,6 @@
 const axios = require('axios');
 
-class UAT {
+class ApiTestingClient {
     constructor (options) {
         this.baseURL = `${options.protocol}://${options.domain}${options.apiPath}`;
         this.token = options.token;
@@ -61,7 +61,7 @@ class UAT {
             });
             const token = Buffer.from('ui:uiman').toString('base64');
             this.setUiToken(token);
-            const response = await this.client.post(`/sso/oauth/token?${encodedURI}`);
+            const response = await this.client.post(`uat/sso/oauth/token?${encodedURI}`);
             return this.handleResponse(response);
         }
         catch (error) {
@@ -70,18 +70,20 @@ class UAT {
     }
 
     /**
-     * Generating an API token
-     * @param {*} token The UI token to authenticate with
-     * @returns A response obj with the API token data
+     * We create an API key to use it later on in our tests.
+     * @param {*} token The UAT token to gain permissions to create an API key
+     * @param {*} userId The id of the user
+     * @param {*} name The name of the token
+     * @returns The api key object
      */
-    async generateApiToken(token) {
-        this.setApiToken(token);
+    async createApiKey(token, userId, name) {
         try {
-            const response = await this.client.post('/sso/me/apitoken?authenticated=true');
+            this.setApiToken(token);
+            const response = await this.client.post(`api/users/${userId}/api-keys`, {name: name});
             return this.handleResponse(response);
         }
-        catch(error) {
-            return this.handleError(error); 
+        catch (error) {
+            return this.handleError(error);
         }
     }
 
@@ -150,4 +152,4 @@ class UAT {
     }
 }
 
-module.exports = UAT;
+module.exports = ApiTestingClient;

src/api.js

@@ -18,7 +18,7 @@ class API {
      */
     async checkConnect () {
         try {
-            return this.handleResponse(await this.client.get('/user'));
+            return this.handleResponse(await this.client.get('/users'));
         }
         catch (error) {
             if(this.displayDebugLogs === true){
@@ -35,7 +35,7 @@ class API {
      */
     async createLaunch (projectName, options) {
         try {
-            return this.handleResponse(await this.client.post(`/${projectName}/launch`, options));
+            return this.handleResponse(await this.client.post(`/v1/${projectName}/launch`, options));
         }
         catch (error) {
             return this.handleError(error);
@@ -50,7 +50,7 @@ class API {
      */
     async finishLaunch (projectName, launchId, options) {
         try {
-            return this.handleResponse(await this.client.put(`/${projectName}/launch/${launchId}/finish`, options));
+            return this.handleResponse(await this.client.put(`/v1/${projectName}/launch/${launchId}/finish`, options));
         }
         catch (error) {
             return this.handleError(error);
@@ -65,7 +65,7 @@ class API {
      */
     async forceStopLaunch (projectName, launchId, options) {
         try {
-            return this.handleResponse(await this.client.put(`/${projectName}/launch/${launchId}/stop`, options));
+            return this.handleResponse(await this.client.put(`/v1/${projectName}/launch/${launchId}/stop`, options));
         }
         catch (error) {
             return this.handleError(error);
@@ -79,7 +79,7 @@ class API {
      */
     async createTestItem (projectName, options) {
         try {
-            return this.handleResponse(await this.client.post(`/${projectName}/item`, options));
+            return this.handleResponse(await this.client.post(`/v1/${projectName}/item`, options));
         }
         catch (error) {
             return this.handleError(error);
@@ -94,7 +94,7 @@ class API {
      */
     async createChildTestItem (projectName, parentItem, options) {
         try {
-            return this.handleResponse(await this.client.post(`/${projectName}/item/${parentItem}`, options));
+            return this.handleResponse(await this.client.post(`/v1/${projectName}/item/${parentItem}`, options));
         }
         catch (error) {
             return this.handleError(error);
@@ -109,7 +109,7 @@ class API {
      */
     async finishTestItem (projectName, testItemId, options) {
         try {
-            return this.handleResponse(await this.client.put(`/${projectName}/item/${testItemId}`, options));
+            return this.handleResponse(await this.client.put(`/v1/${projectName}/item/${testItemId}`, options));
         }
         catch (error) {
             return this.handleError(error);
@@ -159,36 +159,72 @@ class API {
                     headers: { 'Content-type': `multipart/form-data; boundary=${MULTIPART_BOUNDARY}`, 'Authorization': `Bearer ${this.token}` }
                 });
 
-                await instance.post(`${this.baseURL}/${projectName}/log`, this.buildMultiPartStream([options], {
+                await instance.post(`${this.baseURL}/v1/${projectName}/log`, this.buildMultiPartStream([options], {
                     name:    options.file.name,
                     type:    'image/png',
                     content: fs.readFileSync(fullPath)
                 }, MULTIPART_BOUNDARY));
             }
-            else this.handleResponse(await this.client.post(`/${projectName}/log`, options));
+            else this.handleResponse(await this.client.post(`/v1/${projectName}/log`, options));
         }
         catch (error) {
             this.handleError(error);
         }
     }
 
     /**
-     * Retrieving all logs in a project
-     * @param {*} projectName The name of the project
-     * @returns A list of logs
+     * Retrieves a list of the project latest launches
+     * @param {*} projectName The project name
+     * @returns A list of the latest project launches
+     */
+    async getLaunches(projectName) {
+        try {
+            const response = await this.client.get(`/v1/${projectName}/launch/latest`);
+            return this.handleResponse(response).content;
+        }
+        catch (error) {
+            return this.handleError(error);
+        }
+    }
+
+    /**
+     * Retrieves a list of test items 
+     * @param {*} projectName The project name
+     * @param {*} launchId The launch id
+     * @returns A list of test items that are part of a project and a launch
+     */
+    async getTestItems(projectName, launchId) {
+        try {
+            const response = await this.client.get(`/v1/${projectName}/item?filter.eq.launchId=${launchId}&isLatest=false&launchesLimit=0`);
+            return this.handleResponse(response).content;
+        }
+        catch (error) {
+            return this.handleError(error);
+        }
+    }
+
+    /**
+     * Retrieves a list of logs under a test item
+     * @param {*} projectName The project name
+     * @param {*} testItemId The test item id
+     * @param {*} logLevel The log level. Default: info
+     * @returns A list of test item logs
      */
-    async getLogs(projectName) {
+    async getTestItemLogs(projectName, testItemId, logLevel='info') {
         try {
-            const response = await this.client.get(`/${projectName}/log`);
-            return this.handleResponse(response);
+            const response = await this.client.post(`/v1/${projectName}/log/under`, {
+                itemIds: [testItemId],
+                logLevel: logLevel
+            });
+            return this.handleResponse(response)[testItemId];
         }
         catch (error) {
             return this.handleError(error);
         }
     }
 
     /**
-     * Checking if item is a valid JSON
+     * Checking if item is a valid JSON by attempting to parse it
      * @param {*} json The string of the JSON
      */
     isJSON (json) {

src/index.js

@@ -78,12 +78,13 @@ exports['default'] = () => {
         async reportLogs(testId, level, message, time, attachment) {
             if(message !== undefined) {
                 const isJSON = this.reporter.client.isJSON(message) || Array.isArray(message);
+                const isException = isJSON && JSON.parse(message).errMsg !== undefined;
                 //If the log is a stacktrace, and we want to focus on printing the error message itself.
-                if(isJSON && JSON.parse(message).errMsg !== undefined) message = JSON.parse(message).errMsg;
+                if(isException) message = JSON.parse(message).errMsg;
                 //If the log is a JS Object
                 else if(isJSON) message = JSON.parse(message);
                 else if(typeof message === 'object') message = `"${message}"`;
-                message = this.reporter.client.isJSON(message) ? JSON.stringify(message): message;
+                message = isJSON ? JSON.stringify(message): message;
             }
             await this.reporter.sendTestLogs(testId, level, message, time, attachment);
         },

src/report-portal.js

@@ -17,7 +17,7 @@ class ReportPortal {
         this.client = new RPClient({
             protocol: (cliArguments.rprotocol) ? cliArguments.rprotocol: 'https',
             domain:   cliArguments.rdomain,
-            apiPath:  '/api/v1',
+            apiPath:  '/api',
             token:    cliArguments.rtoken,
         });
         this.connected = true;

tests/integration/integration.executor.ts

@@ -2,21 +2,23 @@ import { loadArguments } from '../utils/cli-loader';
 import createTestCafe from 'testcafe';
 import { cliArguments } from 'cli-argument-parser';
 import { expect } from 'chai';
-import UAT from '../../src/uat.js'
+import ApiTestingClient from '../../src/api-testing-client.js'
 let testcafeServer: TestCafe;
 
 describe('Performing Integration testing', async function() {
     this.timeout(10 * 60 * 60 * 60);
     before(async () => {
         loadArguments();
-        const client = new UAT({
-            protocol: 'http',
-            domain:  'localhost:8080',
-            apiPath:  '/uat',
+        const client = new ApiTestingClient({
+            protocol: cliArguments.rprotocol,
+            domain:  cliArguments.rdomain,
+            apiPath:  '/',
         });
+
+        //Using the default user provided by report portal
         const token = await client.getApiToken('default', '1q2w3e');
-        const apiToken = await client.generateApiToken(token.access_token);
-        cliArguments.rtoken = apiToken.access_token;
+        const apiToken = await client.createApiKey(token.access_token, 1, `testing-${new Date().getTime()}` );
+        cliArguments.rtoken = apiToken.api_key;
         testcafeServer = await createTestCafe('localhost', 1337, 1338);
     });
 
@@ -27,6 +29,7 @@ describe('Performing Integration testing', async function() {
     });
 
     it('Running TestCafe Tests', async () => {
+        cliArguments.rlaunch="TestCafe Tests"
         const runner = testcafeServer.createRunner();
         const failedCount = await runner
         .src(['tests/integration/integration.testcafe.ts'])
@@ -37,6 +40,7 @@ describe('Performing Integration testing', async function() {
         console.log('Tests failed: ' + failedCount);
     });
     it('Retry mechanism Tests', async () => {
+        cliArguments.rlaunch="Retry mechanism Tests"
         const runner = testcafeServer.createRunner();
         const failedCount = await runner
         .src(['tests/integration/integration.retry.testcafe.ts'])

tests/integration/integration.testcafe.ts

@@ -6,44 +6,53 @@ fixture `First fixture`
     .page('https://google.com')
     .before(async () => {
         api = new API({
-            protocol: 'http',
-            domain: 'localhost:8080',
-            apiPath: '/api/v1',
+            protocol: cliArguments.rprotocol,
+            domain:  cliArguments.rdomain,
+            apiPath: '/api',
             token: cliArguments.rtoken,
         })
     });
 test('Taking screenshot', async () => {
-    await logAndVerify('About to take a screenshot');
+    await logAndVerify('Taking screenshot', 'About to take a screenshot');
     await t.takeScreenshot()
-    await logAndVerify('The screenshot was succesfully taken!');
+    await logAndVerify('Taking screenshot', 'The screenshot was succesfully taken!');
 })
 test('Negative testing, verifying Error display', async () => {
-    await logAndVerify('About to fail..');
-    await logAndVerify(`${{obj: 'X', obj2: { x: 'Y'}}}`)
-    await logAndVerify({obj: 'X', obj2: { x: 'Y'}})
+    await logAndVerify('Negative testing, verifying Error display', 'About to fail..');
+    await logAndVerify('Negative testing, verifying Error display', `${{obj: 'X', obj2: { x: 'Y'}}}`)
+    await logAndVerify('Negative testing, verifying Error display', {obj: 'X', obj2: { x: 'Y'}})
     await t.expect('X').eql('Y', 'OMG');
-    await logAndVerify('The test failed!');
+    await logAndVerify('Negative testing, verifying Error display', 'The test failed!');
 })
 
 fixture `Second fixture`
 test.skip('Skipping the test', async () => {
-    await logAndVerify('The test is skipped. This log shoud not be appearing.');
+    await logAndVerify('Skipping the test', 'The test is skipped. This log shoud not be appearing.');
 })
 test('Basic print', async () => {
-    await logAndVerify('Printing the test contents');
+    await logAndVerify('Basic print', 'Printing the test contents');
 })
 
 /**
  * Logging a message via console.log and verifying it exists in report portal
  * @param logMsg The log message
  */
-async function logAndVerify(logMsg: any) {
-    console.log(logMsg);
+async function logAndVerify(testName: string, logMsg: any) {
+    const message = typeof logMsg !== 'string' ? JSON.stringify(logMsg): logMsg
+    console.log(message);
     await sleepInSeconds(5 * 1000);
-    let logs = await api.getLogs(cliArguments.rproject);
-    let log = logs.content.filter(l => l.message === logMsg);
-    process.stdout.write(`\n[Test logs]: Found ${log.length} occurances for message '${logMsg}'\n`);
-    await t.expect(log.length).gte(1, `Log appearances for '${logMsg}'`);
+    let launches = await api.getLaunches(cliArguments.rproject);
+    launches = launches.filter(l => l.name === cliArguments.rlaunch)
+    const launchId = launches[0].id;
+    const items = await api.getTestItems(cliArguments.rproject, launchId, testName)
+
+    const item = items.find(item => item.name === testName && item.type === 'TEST')
+    const logs = await api.getTestItemLogs(cliArguments.rproject, item.id)
+
+    const filteredLogs = logs.filter(l => l.message === message);
+
+    process.stdout.write(`\n[Test logs]: Found ${filteredLogs.length} occurances for message '${message}'\n`);
+    await t.expect(filteredLogs.length).gte(1, `Log appearances for '${message}'`);
 }
 
 /**

tests/unit/mock.ts

@@ -1,7 +1,7 @@
 import { Route, Request } from 'dmock-server';
 
 export const mock: Route[] = [{
-    path: '/api/v1/user',
+    path: '/api/users',
     method: 'get',
     response: {}
 },{