Pulled from sl master

.dockerignore

@@ -1,5 +1,4 @@
 node_modules
 npm-debug.log
 .vscode
-logs
-public
+logs

.env.sample

@@ -10,6 +10,10 @@ OBSERVATION_DATASOURCE_NAME = "OBSERVATION_DATASOURCE_NAME"               // Obs
 OBSERVATION_EVIDENCE_DATASOURCE_NAME = "EVIDENCE_DATASOURCE_NAME"         // Observation evidences data source name
 SURVEY_DATASOURCE_NAME = "SURVEY_DATASOURCE_NAME"                         // Survey data source name
 SURVEY_EVIDENCE_DATASOURCE_NAME = "SURVEY_EVIDENCE_DATASOURCE_NAME"       // Survey evidence data source name
+PROJECT_RESOURCE_DATASOURCE_NAME = "PROJECT_RESOURCE_DATASOURCE_NAME"            // Project resource data source name
+OBSERVATION_RESOURCE_DATASOURCE_NAME = "OBSERVATION_RESOURCE_DATASOURCE_NAME"    // Observation resource data source name
+SURVEY_RESOURCE_DATASOURCE_NAME = "SURVEY_RESOURCE_DATASOURCE_NAME"              // Survey resource data source name
+PROGRAM_RESOURCE_DATASOURCE_NAME = "PROGRAM_RESOURCE_DATASOURCE_NAME"     // Program resource data source name
 CONTENT_REPORT_THRESHOLD = 5                                              // Restrict number of records to be shown for container reports
 ENTITY_SCORE_REPORT_THRESHOLD = 5                                         // Restrict number of submission for entity score report
 OBSERVATION_SCORE_REPORT_THRESHOLD = 2                                    // Restrict number of submission per entity in observation report

.github/pull_request_template.md

@@ -0,0 +1,36 @@
+
+# Description
+These recommendations are intended to promote code quality and team communication during software development. They cover a variety of topics, including ensuring that pull requests are submitted to the correct branch, documenting new methods, preserving consistency across many services, and avoiding typical blunders like accessing APIs or DB queries within loops. Sensitive data should not be uploaded, and changes to environment variables or database models should be executed consistently. Teams may work more effectively and develop higher-quality software by adhering to these standards.
+
+
+## Type of change
+Please choose appropriate options.
+
+- [ ]  Bug fix (non-breaking change which fixes an issue)
+- [ ]  New feature (non-breaking change which adds functionality)
+- [ ]  Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ]  Enhancement (additive changes to improve performance)
+- [ ]  This change requires a documentation update
+
+## Checklist 
+
+- [ ]  It's critical to avoid making needless file modifications in contributions, such as adding new lines, console logs, or additional spaces, to guarantee cleaner and more efficient code. Furthermore, eliminating unnecessary imports from a file might enhance code readability and efficiency.
+- [ ]  Ensure that the pull request is assigned to the right base branch and that the development branch name contains the JIRA Task Id. Furthermore, each commit message should include the JIRA Task Id in the manner "ED-100: message".
+- [ ]  Only update packages if it is mentioned and authorized in the design document, and make sure that you have the required permissions.
+- [ ]  Avoid making API and database queries inside a loop as it can lead to performance issues and slow down the system.
+- [ ]  When calling another function inside a given function, add comments explaining the purpose and meaning of the passed arguments and expected return values.
+- [ ]  If adding a blank argument in a function, add a comment explaining the reason for the blank argument.
+- [ ]  Before submitting a pull request, do a self-review of your code to ensure there are no conflicts with the base branch and all comments have been addressed.
+- [ ]  Before merging a pull request, it's important to have other team members review it to catch any potential errors or issues
+- [ ]  To maintain code integrity, it's important to remove all related changes when removing code during a code review.
+- [ ]  If new constants, endpoints, or utility functions are introduced, it is important to check if they already exist in the service to avoid any duplication.
+- [ ]  Whenever a new environment variable is added to a service, it's important to ensure that the necessary changes are made to related files such as ".env.sample" and "envVariables.js" to maintain consistency and avoid errors. Additionally, the new environment variable should be added to the devops repository to ensure that it is properly documented and accessible to the team.
+- [ ]  When adding a new function to a service, it is important to document it with relevant information such as the name, parameters, and return value in a consistent format across all services. Additionally, if there are any changes to the API response, ensure that the documentation in the controllers is updated accordingly. 
+- [ ]  Write a clear and concise commit message that describes the changes made.
+- [ ]  Maintain consistent function signature and code across all services when adding a function to multiple services. Implement changes to database models in all services that use the same model. 
+- [ ]  Use only let and const. Do not use var.
+- [ ]  Make common functions for repetitive code blocks.
+- [ ]  Avoid uploading sensitive information such as secret tokens or passwords in pull requests to ensure data security.
+- [ ]  Maintain consistent indentation and spacing throughout the code.
+
+

.gitignore

@@ -64,3 +64,5 @@ typings/
 config/config.json
 tmp/*
 package-lock.json
+keycloak-public-keys/
+.vscode/

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:12
+FROM node:20
 
 WORKDIR /opt/reports
 

README.md

@@ -1,3 +1,113 @@
 # ml-reports-service
-Manage Learn reports service 
 
+This is ml-reports-services, the managed-learn component responsible for creating reports for observation, survey, projects and programs within the managed learn services.
+
+# Setup Guide
+
+## Pre-Requisite
+
+- Install any IDE in your system(eg: VScode etc..)
+- Install nodejs from : https://nodejs.org/en/download/
+
+Basic understanding of git and github is recommended.
+
+- https://www.youtube.com/watch?v=RGOj5yH7evk&t=2s
+- https://git-scm.com/book/en/v2/Getting-Started-What-is-Git%3F
+
+## Setup ml-reports-services
+
+### Clone the service repository onto your system
+
+- Create a new folder where you want to clone the repository.
+- Navigate to that directory using the terminal.
+- Execute the git commands to clone the repository using the provided link from the code tab.
+
+Git link
+
+    https://github.com/shikshalokam/ml-reports-service.git
+
+command to clone
+
+    git clone https://github.com/shikshalokam/ml-reports-service.git
+
+### Create .env file
+
+Create a file named `.env` and copy the environment-specific data corresponding to that service into the `.env` file.
+
+    # Environment configurations file
+    APPLICATION_PORT = "3000"                                                 // Application port number
+    APPLICATION_ENV = "development"                                           // Application running enviornment
+
+    INTERNAL_ACCESS_TOKEN = "Fg*************yr"                               // Internal access token for accessing internal service APIs
+
+    #Druid configuration
+    DRUID_URL = "http://localhost:9042"                                       // Druid connection url
+    OBSERVATION_DATASOURCE_NAME = "OBSERVATION_DATASOURCE_NAME"               // Observation data source name
+    OBSERVATION_EVIDENCE_DATASOURCE_NAME = "EVIDENCE_DATASOURCE_NAME"         // Observation evidences data source name
+    SURVEY_DATASOURCE_NAME = "SURVEY_DATASOURCE_NAME"                         // Survey data source name
+    SURVEY_EVIDENCE_DATASOURCE_NAME = "SURVEY_EVIDENCE_DATASOURCE_NAME"       // Survey evidence data source name
+    PROJECT_RESOURCE_DATASOURCE_NAME = "PROJECT_RESOURCE_DATASOURCE_NAME"            // Project resource data source name
+    OBSERVATION_RESOURCE_DATASOURCE_NAME = "OBSERVATION_RESOURCE_DATASOURCE_NAME"    // Observation resource data source name
+    SURVEY_RESOURCE_DATASOURCE_NAME = "SURVEY_RESOURCE_DATASOURCE_NAME"              // Survey resource data source name
+    PROGRAM_RESOURCE_DATASOURCE_NAME = "PROGRAM_RESOURCE_DATASOURCE_NAME"     // Program resource data source name
+    CONTENT_REPORT_THRESHOLD = 5                                              // Restrict number of records to be shown for container reports
+    ENTITY_SCORE_REPORT_THRESHOLD = 5                                         // Restrict number of submission for entity score report
+    OBSERVATION_SCORE_REPORT_THRESHOLD = 2                                    // Restrict number of submission per entity in observation report
+    EVIDENCE_THRESHOLD = 3                                                    // Restrict number of evidence per questions
+
+    #Gotenberg configuration
+    GOTENBERG_URL = "http://localhost:3000"                                   // Gotenberg service URL
+
+    # ML Survey Service
+    ML_SURVEY_SERVICE_URL = "http://ml-survey-service:3000"                   // ML survey service URL
+
+    # ML Core  Service
+    ML_CORE_SERVICE_URL = "http://ml-core-service:3000"                       // ML Core Service URL
+
+    # OFFLINE TOKEN VALIDATION
+    KEYCLOAK_PUBLIC_KEY_PATH = "keycloak-public-keys"                         // Keycloak public keys path
+
+### Install Dependencies
+
+To install dependencies from a `package.json` file in Visual Studio Code, you can use the integrated terminal. Here are the steps:
+
+- Open the integrated terminal by going to View > Terminal or using the shortcut Ctrl+` (backtick).
+- In the terminal, navigate to the directory where the package.json file is located.
+- Run the command `npm install` or `yarn install`, depending on your preferred package manager.
+- The package manager will read the package.json file and install all the dependencies specified in it.
+- Wait for the installation process to complete. You should see progress indicators or a success message for each installed dependency.
+- Once the installation is finished, the dependencies listed in the package.json file will be installed in a node_modules directory in your project.
+
+### Setting the keycloak
+
+- Create a folder on service directory named: `keycloak-public-keys`
+- Inside that folder create a file `GRxxx....................xxxxx60fA`
+
+**for keycloak file please contact Backend Team**
+
+### Setup Database
+
+The Reports service will communicate with the ML Core and ML Survey services to retrieve data from MongoDB for generating reports and analytics. Additionally, it will interact with Druid for the reporting and analytics functionalities.
+
+### DB Schema
+
+The schema serves as a blueprint for creating and maintaining the database that supports the ML reports services data storage and retrieval operations.
+
+![ML-Reports Service](https://ml-services-uploads.s3.ap-south-1.amazonaws.com/DBSchema/ML-Reports.png)
+
+[Click here](https://ml-services-uploads.s3.ap-south-1.amazonaws.com/DBSchema/ML-Reports.pdf) for DB schema and corresponding examples in a PDF format.
+
+### Postman Collection
+
+The ML Reports Service Postman Collection is a comprehensive resource for interacting with the ML Core Service. It includes organized endpoints, detailed documentation, and example workflows, providing a valuable reference for developers. Leverage this collection to enhance productivity and collaboration in ML Services.
+
+[Click here](https://documenter.getpostman.com/view/7997930/2s946chuaT)
+
+## IMPORTANT:
+
+Always work on branches. **Never make changes to master**.
+
+Creating a branch from master.
+
+For more information on git you can use :  
+ https://education.github.com/git-cheat-sheet-education.pdf

app.js

@@ -49,18 +49,19 @@ app.set('port', port);
 
 app.all("*", (req, res, next) => {
 
-    console.log("-------Request log starts here------------------");
-    console.log(
-      "%s %s on %s from ",
-      req.method,
-      req.url,
-      new Date(),
-      req.headers["user-agent"]
-    );
-    console.log("Request Headers: ", req.headers);
-    console.log("Request Body: ", req.body);
-    console.log("Request Files: ", req.files);
-    console.log("-------Request log ends here------------------");
+    console.log({"Debugging ML Reports Service": true});
+    // console.log("-------Request log starts here------------------");
+    // console.log(
+    //   "%s %s on %s from ",
+    //   req.method,
+    //   req.url,
+    //   new Date(),
+    //   req.headers["user-agent"]
+    // );
+    // console.log("Request Headers: ", req.headers);
+    // console.log("Request Body: ", req.body);
+    // console.log("Request Files: ", req.files);
+    // console.log("-------Request log ends here------------------");
 
 
   next();

auto_build_deploy

@@ -0,0 +1,55 @@
+@Library('deploy-conf') _
+node('build-slave') {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        ansiColor('xterm') {
+            timestamps {
+                stage('Checkout') {
+                    tag_name = env.JOB_NAME.split("/")[-1]
+                    pre_checks()
+                    if (!env.hub_org) {
+                        println(ANSI_BOLD + ANSI_RED + "Uh Oh! Please set a Jenkins environment variable named hub_org with value as registery/sunbidrded" + ANSI_NORMAL)
+                        error 'Please resolve the errors and rerun..'
+                    } else
+                        println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL)
+                }
+                cleanWs()
+                def scmVars = checkout scm
+                checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$tag_name"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]]
+                build_tag = tag_name + "_" + env.BUILD_NUMBER
+                commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
+                artifact_version = tag_name + "_" + commit_hash
+                echo "build_tag: " + build_tag
+
+              //  stage Build
+                    env.NODE_ENV = "build"
+                    print "Environment will be : ${env.NODE_ENV}"
+                    sh('git submodule update --init')
+                    sh('git submodule update --init --recursive --remote')
+                    sh('chmod 777 build.sh')
+                    sh("./build.sh ${build_tag} ${env.NODE_NAME} ${hub_org}")
+
+
+             //   stage ArchiveArtifacts
+                    archiveArtifacts "metadata.json"
+                    currentBuild.description = "${build_tag}"
+
+            }
+        }
+        currentBuild.result = "SUCCESS"    
+        slack_notify(currentBuild.result, tag_name)
+        email_notify()  
+        auto_build_deploy()
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        slack_notify(currentBuild.result, tag_name)
+        email_notify()
+        throw err
+    }
+}

common/constants.js

@@ -0,0 +1,41 @@
+/**
+ * name : constants.js
+ * author : Ankit Shahu
+ * created-date : 10-April-2023
+ * Description : This is to keep all the hard code value in form of variable.
+ */
+
+
+/**
+ * Description : projection passed by frontend
+ */
+exports.ResourceTypeProjection = {
+    DISTRICT:"district",
+    BLOCK:"block",
+    ORGANISATION:"organisation"
+
+}
+
+/**
+ * Description : resource type passed by frontend
+ */
+exports.ResourceType = {
+    PROGRAM:"program",
+    SOLUTION:"solution"
+}
+
+/**
+ * Description : solution type passed by frontend
+ */
+exports.SolutionType = {
+    PROJECT : "improvementProject",
+    OBSERVATION: "observation",
+    SURVEY :"survey"
+}
+
+/**
+ * Description : druid query interval for raw data-source
+ */
+exports.druidQueryInterval = {
+    RAW_DATA_SOURCE_INTERVAL : "1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00",
+}

common/druid_queries.json

@@ -71,6 +71,19 @@
 
     "entity_solution_score_query" : {"queryType":"groupBy","dataSource":"ml_observation_qa","dimensions":["questionName","questionExternalId","questionResponseType","minScore","maxScore","observationSubmissionId","school","schoolName","questionId","completedDate","observationName","solutionId","solutionName"],"aggregations":[{"type":"count","name":"count"}],"granularity":"all","postAggregations":[],"intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","filter":{"type":"and","fields":[{"type":"or","fields":[{"type":"selector","dimension":"questionResponseType","value":"radio"},{"type":"selector","dimension":"questionResponseType","value":"multiselect"},{"type":"selector","dimension":"questionResponseType","value":"slider"}]},{"type":"and","fields":[{"type":"selector","dimension":"","value":"" },{"type":"selector","dimension":"solutionId","value":""}]}]},"limitSpec":{"type":"default","limit":10000,"columns":[{"dimension":"count","direction":"descending"}]}},
 
-    "observations_by_entity" : {"queryType":"groupBy","dataSource":"ml_observation_dev","granularity":"all","dimensions":["entityId","entityName","observationName","observationSubmissionId","createdAt"],"filter":{"type":"or","fields":[]},"aggregations":[],"postAggregations":[],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}
+    "observations_by_entity" : {"queryType":"groupBy","dataSource":"ml_observation_dev","granularity":"all","dimensions":["entityId","entityName","observationName","observationSubmissionId","createdAt"],"filter":{"type":"or","fields":[]},"aggregations":[],"postAggregations":[],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]},
+
+    "question_response_query" : {"queryType":"scan","dataSource":"sl-observation","resultFormat": "list","columns":[ "solutionName", "criteriaName", "domainName", "questionName", "programName", "questionResponseLabel", "observationSubmissionId", "questionResponseType", "questionId", "questionSequenceByEcm","solution_type"],"filter": {"type":"and","fields": [ { "type": "or", "fields": [ { "type": "selector", "dimension": "questionResponseType", "value": "radio" }, { "type": "selector", "dimension": "questionResponseType", "value": "multiselect" }, { "type": "selector", "dimension": "questionResponseType", "value": "slider" } ] } ] },"intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"},
+
+    "solution_distric_level_query": {"queryType": "groupBy","dataSource": "","dimensions": ["district_name", "district_externalId"],"granularity": "all","intervals": "","filter": {"type": "and","fields": [{"type": "not","field": {"type": "selector","dimension": "district_externalId","value": null}}]}},
+
+    "solution_organisations_level_query": {"queryType": "groupBy","dataSource": "","dimensions": ["organisation_id", "organisation_name"],"granularity": "all","intervals": "","filter": {"type": "and","fields": [{"type": "not","field": {"type": "selector","dimension": "organisation_id","value": null}}]}},
+
+    "solution_block_level_query": {"queryType": "groupBy","dataSource": "","dimensions": ["block_externalId", "block_name"],"granularity": "all","intervals": "","filter": {"type": "and","fields": [{"type": "not","field": {"type": "selector","dimension": "block_externalId","value": null}}]}},
+
+    "program_distric_level_query": {"queryType": "groupBy","dataSource": "","dimensions": ["district_name", "district_externalId"],"granularity": "all","intervals": "","filter": {"type": "and","fields": [{"type": "not","field": {"type": "selector","dimension": "district_externalId","value": null}}]}},
+
+    "program_organisations_level_query": {"queryType": "groupBy","dataSource": "","dimensions": ["organisation_id", "organisation_name"],"granularity": "all","intervals": "","filter": {"type": "and","fields": [{"type": "not","field": {"type": "selector","dimension": "organisation_id","value": null}}]}},
 
+    "program_block_level_query": {"queryType": "groupBy","dataSource": "","dimensions": ["block_externalId", "block_name"],"granularity": "all","intervals": "","filter": {"type": "and","fields": [{"type": "not","field": {"type": "selector","dimension": "block_externalId","value": null}}]}}
 }