diff --git a/Docs/Request_Overrides.md b/Docs/Request_Overrides.md new file mode 100644 index 00000000..84be5a09 --- /dev/null +++ b/Docs/Request_Overrides.md @@ -0,0 +1,183 @@ +# Request overrides + +## fetchPerformanceThreshold: + +- Request:
+ Makes an HTTP request to graphite with deviceMac, processType with how much percentile, and from what time to fetch the metrics.
+ Format: + ``` + { + method: 'performance.fetchPerformanceThreshold', + params: {'type': '<(device|process|all)>', process: '<(memory|load|set size|required)>', percentile: 70, threshold: ''} + } + ``` + Examples: + ``` + { + method: 'performance.fetchPerformanceThreshold', + params: {'type': 'device', process: 'memory', percentile: 70, threshold: '35000000'} + } + { + method: 'performance.fetchPerformanceThreshold', + params: {'type': 'process', process: 'set size', percentile: 70, threshold: '75000000'} + } + { + method: 'performance.fetchPerformanceThreshold', + params: {'type': 'all', process: 'required', percentile: 70, threshold: '75000000'} + } + ``` +- Response:
+ Receives an array of objects, which contains success and message properties, success defines the execution is a success or failure and message defines either response or any custom message that descibes the pass/fail. + Example: + ``` + [ + { + "success": true, + "message": "Expected received threshold for set sizeRSS is 37748736 to be less than the expected threshold of 1073741824" + }, + { + "success": true, + "message": "Expected received threshold for set sizePSS is 41964544 to be less than the expected threshold of 1073741824" + } + ] + ``` + +## createMarker: + +- Request:
+ Making an HTTP call to grafana to create a marker on dashboard with given description.
+ Format: + ``` + { + method: 'performance.createMarker', + params: + } + ``` + Examples: + ``` + { + method: 'performance.createMarker', + params: 'Account.id - Positive Scenario: Validate account ID' + } + ``` +- Response:
+ Recieves an object with success and message properties. + Example: + ``` + { + "success": true, + "message": "Marker has been created successfully" + } + { + "success": false, + "message": `Unable to create marker, failed with status code- 200 and error- unable to find the dashboard` + } + ``` + +## setTestProvider: + +- Request:
+ Making a call which sends necessary message to the platform to use a test provider for simulating user inputs.
+ Format: + ``` + { + method: 'fcs.setTestProvider', + params: + } + ``` + Examples: + ``` + { + method: 'fcs.setTestProvider', + params: 'pinChallenge' + } + ``` +- Response:
+ Receives an object with intent message. + Example: + ``` + { + "action": "search", + "context": { + "source": "device" + }, + "data": { + "query": "{\"task\":\"registerProviderHandler\",\"params\":{\"provider\":\"pinChallenge\"},\"action\":\"CORE\",\"context\":{\"communicationMode\":\"SDK\"},\"asynchronous\":false}" + } + } + ``` + +## recordLifecycleHistory: + +- Request:
+ Making a call to the platform to start/stop the lifecycle history recording.
+ Format: + ``` + { + "method": "fcs.recordLifecycleHistory", + "params": { + "task": "", + "appId": "" + } + } + + ``` + Examples: + ``` + { + "method": "fcs.recordLifecycleHistory", + "params": { + "task": "start", + "appId": "test" + } + } + ``` +- Response:
+ Receives an object with intent message and transport type. + Example: + ``` + "transport": "", + "payload": { + "action": "search", + "context": { + "source": "device" + }, + "data": { + "query": "{\"task\":\"start\",\"params\":{\"provider\":\"pinChallenge\"},\"action\":\"CORE\",\"context\":{\"communicationMode\":\"SDK\"},\"asynchronous\":false}" + } + } + ``` + +## fetchDeviceDetails: + +- Request:
+ Making a call to fetch device details and the token required for dynamically fetching details .
+ + Format: + ``` + { + "method": "fcs.fetchDeviceDetails", + "params": + } + + ``` + Examples: + ``` + { + + "method": "fcs.fetchDeviceDetails", + "params": "354444327" + } + ``` +- Response:
+ Receives the updated environment variable device_data with dynamic device details. +Example: + + ``` + { + "DEVICEID": "354444327", + "DEVICE_TYPE": "ipstb", + "DEVICE_MODEL": "VALUE", + ... +} + ``` diff --git a/README.md b/README.md index e5db23cf..1fca1099 100644 --- a/README.md +++ b/README.md @@ -253,150 +253,8 @@ HTTP call to the platform:
## Request overrides -### fetchPerformanceThreshold: +Documentation added in [Request_Overrides.md](/Docs/Request_Overrides.md) -- Request:
- Makes an HTTP request to graphite with deviceMac, processType with how much percentile, and from what time to fetch the metrics.
- Format: - ``` - { - method: 'performance.fetchPerformanceThreshold', - params: {'type': '<(device|process|all)>', process: '<(memory|load|set size|required)>', percentile: 70, threshold: ''} - } - ``` - Examples: - ``` - { - method: 'performance.fetchPerformanceThreshold', - params: {'type': 'device', process: 'memory', percentile: 70, threshold: '35000000'} - } - { - method: 'performance.fetchPerformanceThreshold', - params: {'type': 'process', process: 'set size', percentile: 70, threshold: '75000000'} - } - { - method: 'performance.fetchPerformanceThreshold', - params: {'type': 'all', process: 'required', percentile: 70, threshold: '75000000'} - } - ``` -- Response:
- Receives an array of objects, which contains success and message properties, success defines the execution is a success or failure and message defines either response or any custom message that descibes the pass/fail. - Example: - ``` - [ - { - "success": true, - "message": "Expected received threshold for set sizeRSS is 37748736 to be less than the expected threshold of 1073741824" - }, - { - "success": true, - "message": "Expected received threshold for set sizePSS is 41964544 to be less than the expected threshold of 1073741824" - } - ] - ``` -### createMarker: - -- Request:
- Making an HTTP call to grafana to create a marker on dashboard with given description.
- Format: - ``` - { - method: 'performance.createMarker', - params: - } - ``` - Examples: - ``` - { - method: 'performance.createMarker', - params: 'Account.id - Positive Scenario: Validate account ID' - } - ``` -- Response:
- Recieves an object with success and message properties. - Example: - ``` - { - "success": true, - "message": "Marker has been created successfully" - } - { - "success": false, - "message": `Unable to create marker, failed with status code- 200 and error- unable to find the dashboard` - } - ``` - ### setTestProvider: - -- Request:
- Making a call which sends necessary message to the platform to use a test provider for simulating user inputs.
- Format: - ``` - { - method: 'fcs.setTestProvider', - params: - } - ``` - Examples: - ``` - { - method: 'fcs.setTestProvider', - params: 'pinChallenge' - } - ``` -- Response:
- Receives an object with intent message. - Example: - ``` - { - "action": "search", - "context": { - "source": "device" - }, - "data": { - "query": "{\"task\":\"registerProviderHandler\",\"params\":{\"provider\":\"pinChallenge\"},\"action\":\"CORE\",\"context\":{\"communicationMode\":\"SDK\"},\"asynchronous\":false}" - } - } - ``` -### recordLifecycleHistory: - -- Request:
- Making a call to the platform to start/stop the lifecycle history recording.
- Format: - ``` - { - "method": "fcs.recordLifecycleHistory", - "params": { - "task": "", - "appId": "" - } - } - - ``` - Examples: - ``` - { - "method": "fcs.recordLifecycleHistory", - "params": { - "task": "start", - "appId": "test" - } - } - ``` -- Response:
- Receives an object with intent message and transport type. - Example: - ``` - "transport": "", - "payload": { - "action": "search", - "context": { - "source": "device" - }, - "data": { - "query": "{\"task\":\"start\",\"params\":{\"provider\":\"pinChallenge\"},\"action\":\"CORE\",\"context\":{\"communicationMode\":\"SDK\"},\"asynchronous\":false}" - } - } - ``` ## Interaction Log service diff --git a/cypress/plugins/testDataProcessor.js b/cypress/plugins/testDataProcessor.js index a7826d30..5a8e38ef 100644 --- a/cypress/plugins/testDataProcessor.js +++ b/cypress/plugins/testDataProcessor.js @@ -280,14 +280,8 @@ function testDataHandler(requestType, dataIdentifier, fireboltObject) { if (!deviceMac) { logger.info('Falling back to default device data path'); } - let deviceData = fetchAndParseDataFromJson(deviceDataPath, data.type); - if (deviceData === CONSTANTS.NO_DATA) { - logger.info( - `Expected deviceData not found for ${data.type}. Returning ${data.type} as is.` - ); - deviceData = data.type; - } - data.type = deviceData; + const deviceData = fetchDataFromFile(deviceDataPath); + envVariables[CONSTANTS.DEVICE_DATA] = deviceData; break; default: diff --git a/cypress/support/constants/constants.js b/cypress/support/constants/constants.js index cf52bfd7..d04daa8e 100644 --- a/cypress/support/constants/constants.js +++ b/cypress/support/constants/constants.js @@ -84,10 +84,12 @@ module.exports = { DEVICEMODULEFORMAT: 'deviceModelFormat', DEVICE_AUDIO: 'device.audio', DEVICE_CONTENT_VALIDATION: 'deviceContentValidation', + DEVICE_DATA: 'deviceData', DEVICE_ENV: 'Device', DEVICE_FIRMWARE: 'Device Firmware', DEVICE_HDCP: 'device.hdcp', DEVICE_HDR: 'device.hdr', + DEVICE_ID: 'device.id', DEVICE_IP: 'deviceIp', DEVICE_MAC: 'deviceMac', DEVICE_MAC_UNAVAILABLE: 'Device MAC unavailable', @@ -99,6 +101,7 @@ module.exports = { DEVICE_PLATFORM: 'device.platform', DEVICE_VIDEORESOLUTION: 'device.videoResolution', DISCOVERY_LAUNCH: 'discovery.launch', + DYNAMIC_DEVICE_DETAILS_MODULES: ['Device'], EMAIL: 'email', ENV_PLATFORM_SDK_VERSION: 'platformSdkVersion', ENV_PLATFORM: 'platform', @@ -192,6 +195,7 @@ module.exports = { CUSTOM: 'custom', VALIDATION_FUNCTION: 'validationFunction', NOT_SUPPORTED: 'NOT_SUPPORTED', + FETCH_DEVICE_DETAILS_DYNAMICALLY_FLAG: 'fetch_device_details_dynamically', FIRST_PARTY_APPID: 'firstPartyAppId', GENERATE_HTML_REPORT: true, GETEVENTRESPONSE_ERROR_MSG: 'Received error: Requested listener not found', @@ -362,6 +366,7 @@ module.exports = { SETFIREBOLTINTERACTIONSHANDLER: 'fcs.setFireboltInteractionsHandler', CREATE_MARKER: 'performance.createMarker', UNLOADAPP: 'fcs.unloadApp', + FETCHDEVICEDETAILS: 'fcs.fetchDeviceDetails', }, REQUEST_MAP_INTERACTIONS_SERVICE: 'Request map for firebolt interactions service : ', RESPONSE: 'Response: ', diff --git a/cypress/support/step_definitions/dynamicContentValidations.md b/cypress/support/step_definitions/dynamicContentValidations.md new file mode 100644 index 00000000..09f0ccc8 --- /dev/null +++ b/cypress/support/step_definitions/dynamicContentValidations.md @@ -0,0 +1,92 @@ +# Dynamic Content Validation + +### Background: +While executing the validations step in fcs tests, there are mainly seven modes of validation, which are mentioned in [Supported Validations](./validations.md#supported-validations). Out of these seven, deviceContentValidation is used when the validation is to be done based on device specific value of the device which we are currently testing on. +While executing the test, the validation key will be fetched from the cypress/fixtures/fireboltCalls/ folder based on the current executing module. For example, if the "expected device id" is the validation key used in the validation step, it will be placed in the device.json file inside the fireboltCalls folder as: + +``` + "EXPECTED_DEVICE_ID": { + "method": "device.id", + "validationJsonPath": "result", + "content": + + } +``` +Later, the content specified in this object is searched inside the cypress/fixtures/objects/validationObjects/ folder, and it is expected to be in the following format: + +``` + + : { + "data": [ + { + "type": "fixture", + "validations": [ + { + "mode": "deviceContentValidation", + "type": "DEVICEID", + "description": "Validation of the Device Id Format" + } + ] + } + ] + }, +``` + +In the validationObject, when the validation mode is "deviceContentValidation", the source of truth for the validation can be one of the following two ways: +1. Static data +2. Dynamic data + +Static data : +For static data, the source of truth is stored in a file placed inside configModule's cypress/fixtures/devices/ folder, based on the deviceMac value of the device currently being tested. When the test begins, these values are loaded into an object and stored in an environment variable called "deviceData". If deviceMac.json is not found in the specified path, the values are saved as they are, for example: DEVICEID as "DEVICEID". + +Dynamic data : +For dynamic data, the source of truth is fetched from the configModule's override function called "fetchDeviceDetails" explained in [Request overrides](https://github.com/rdkcentral/firebolt-certification-suite?tab=readme-ov-file#request-overrides). + +### Dynamic data usage : + +To fetch dynamic data from configModule, we have to set an environment variable called "fetch_device_details_dynamically" as true in the config.json file inside configModule's (constants/) folder. If this environment variable is true, dynamic device details will be fetched from configModule's override function called "fetchDeviceDetails". Later, the static data saved in "deviceData" environment variable will be overriden with the dynamic data fetched from the override fucntion. If the dynamic device details fetching fails in configModule, the data will default to the static data. +FCS expects the source of truth to be generated for the modules listed in constant "DYNAMIC_DEVICE_DETAILS_MODULES" inside [constants.js file](../constants/constants.js). + + +### Dynamic data implementation : + +The configModule should have an override function "fetchDeviceDetails" as explained [here](https://github.com/rdkcentral/firebolt-certification-suite?tab=readme-ov-file#request-overrides). So, if the "fetch_device_details_dynamically" is set to true and "fetchDeviceDetails" override function is implemented in configModule, when the test executes the step "Given the environment has been set up for '' tests", the "fetchDeviceDetails" override function is called if the "testType" is present in "DYNAMIC_DEVICE_DETAILS_MODULES" constant. + +In the override function "fetchDeviceDetails" the logic to fetch dynamic data is added as per platform's requirement using corresponding urls. Once dynamic data is fetched inside configModule, it is then saved into the "deviceData" environment variable, where the existing static data is overriden with the dynamic data. For eg:, from override function if we gets the values for deviceId, deviceType and distributor, we have to override the "deviceData" environment variable with these values as : + +#### Format + +``` + let deviceData = Cypress.env(CONSTANTS.DEVICE_DATA) #saves the env variable to "deviceData" + + deviceData.DEVICEID = deviceId; + deviceData.DEVICE_TYPE = deviceType; + deviceData.DEVICE_DISTRIBUTOR = distributor; + + Cypress.env(CONSTANTS.DEVICE_DATA, deviceData); #updated the env variable with dynamic data + +``` + +### Example : +From FCS, the override function is invoked in the following format: +Format: + ``` + { + "method": "fcs.fetchDeviceDetails", + "params": + } + +The response by overriding the static values in the environment will be as follows: +Example: + +``` + { + "DEVICEID": "12345", + "DEVICE_TYPE": "xxxxxx", + "DEVICE_DISTRIBUTOR": "xxxxxx", + ... +} + +``` + +Here, "deviceId" is the only parameter we are passing from fcs while invoking override function, as this value might be used to extract the required data, if dynamic response contains more than one value. diff --git a/cypress/support/step_definitions/testSetup.js b/cypress/support/step_definitions/testSetup.js index 91eabab4..8a4cc0b7 100644 --- a/cypress/support/step_definitions/testSetup.js +++ b/cypress/support/step_definitions/testSetup.js @@ -54,6 +54,23 @@ Given('the environment has been set up for {string} tests', (test) => { if (Cypress.env(CONSTANTS.TEST_TYPE).includes('rpc-Only')) { Cypress.env(CONSTANTS.IS_RPC_ONLY, true); } + // fetch device details dynamically + if (Cypress.env(CONSTANTS.FETCH_DEVICE_DETAILS_DYNAMICALLY_FLAG)) { + if (CONSTANTS.DYNAMIC_DEVICE_DETAILS_MODULES.includes(Cypress.env(CONSTANTS.TEST_TYPE))) { + cy.getDeviceData(CONSTANTS.DEVICE_ID, {}, CONSTANTS.ACTION_CORE.toLowerCase()).then( + (response) => { + if (response) { + const method = CONSTANTS.REQUEST_OVERRIDE_CALLS.FETCHDEVICEDETAILS; + const requestMap = { + method: method, + params: response, + }; + cy.sendMessagetoPlatforms(requestMap); + } + } + ); + } + } } }); diff --git a/cypress/support/step_definitions/validations.js b/cypress/support/step_definitions/validations.js index 390f30b6..8c06ccf3 100644 --- a/cypress/support/step_definitions/validations.js +++ b/cypress/support/step_definitions/validations.js @@ -18,7 +18,7 @@ import { Given, Then } from '@badeball/cypress-cucumber-preprocessor'; const CONSTANTS = require('../constants/constants'); const { _ } = Cypress; -import UTILS from '../cypress-support/src/utils'; +import UTILS, { fireLog } from '../cypress-support/src/utils'; /** * @module validations @@ -63,6 +63,34 @@ Given( : CONSTANTS.NULL_RESPONSE; const expectingError = item.expectingError; const isNullCase = item.isNullCase || false; + // check if the device details env is present + if ( + Cypress.env(CONSTANTS.DEVICE_DATA) && + Object.keys(Cypress.env(CONSTANTS.DEVICE_DATA)).length > 0 + ) { + let type; + // if the validation object used for current validation contains the required data + if (contentObject && contentObject.data) { + for (let i = 0; i < contentObject.data.length; i++) { + if ( + contentObject.data[i].validations && + contentObject.data[i].validations[0] && + contentObject.data[i].validations[0].type && + contentObject.data[i].validations[0].mode == CONSTANTS.DEVICE_CONTENT_VALIDATION + ) { + type = contentObject.data[i].validations[0].type; + // if the dynamic device details env contains the validation key + if (Cypress.env(CONSTANTS.DEVICE_DATA).hasOwnProperty(type)) { + contentObject.data[i].validations[0].type = Cypress.env(CONSTANTS.DEVICE_DATA)[ + type + ]; + } + } + } + } + } else { + fireLog.info('deviceData environment variable does not have the required data'); + } // If the app ID is not passed from the feature, the default app ID will be retrieved. appId = !appId diff --git a/cypress/support/step_definitions/validations.md b/cypress/support/step_definitions/validations.md index 9e2e06f2..e49b3621 100644 --- a/cypress/support/step_definitions/validations.md +++ b/cypress/support/step_definitions/validations.md @@ -408,6 +408,10 @@ The 'schemaOnly` validation type allows to skip content validation and stops at } ``` +# Dynamic Content Validation + +Documentation added in [dynamicContentValidations.md](./dynamicContentValidations.md) + # Validation Override ## Format @@ -445,6 +449,7 @@ While validating, if a key is present in both fcs-validation jsons (eg: cypress/ ] } + ## Error Content Validation ### Background