From e141f2bd6fac14075be5911fd92bdc3557d2b119 Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Mon, 10 Jun 2024 17:25:22 +0200
Subject: [PATCH 01/15] Extra logging

---
 runregistry_backend/app.js                                      | 2 +-
 runregistry_backend/cron/1.get_runs.js                          | 2 +-
 .../cron/saving_updating_runs_lumisections_utils.js             | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/runregistry_backend/app.js b/runregistry_backend/app.js
index 72733cf..f229335 100644
--- a/runregistry_backend/app.js
+++ b/runregistry_backend/app.js
@@ -38,7 +38,7 @@ console.error = getLogger("ERROR", console.error)
 
 // Logging for sanity
 const { database, host, port: db_port } = config[process.env.ENV];
-console.error(`Using database: ${database}@${host}:${db_port}`);
+console.info(`Using database: ${database}@${host}:${db_port}`);
 
 models.sequelize.sync({})
   .then(async () => {
diff --git a/runregistry_backend/cron/1.get_runs.js b/runregistry_backend/cron/1.get_runs.js
index 8a262e9..2f9a93b 100755
--- a/runregistry_backend/cron/1.get_runs.js
+++ b/runregistry_backend/cron/1.get_runs.js
@@ -57,7 +57,7 @@ const fetch_runs = async (
   let fetched_runs = first_time
     ? all_fetched_runs
     : all_fetched_runs.slice(fetch_amount / 2);
-
+  console.debug("Querying the last 50 updated OMS runs")
   const { data: last_saved_runs } = await axios.get(
     `${API_URL}/runs_lastupdated_50`
   );
diff --git a/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js b/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js
index 5adfa9f..62e2dcc 100644
--- a/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js
+++ b/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js
@@ -28,6 +28,7 @@ exports.get_OMS_lumisections = handleErrors(async (run_number) => {
   await new Promise((resolve) => {
     setTimeout(resolve, 2000);
   });
+  console.debug(`Requesting lumisections for run ${run_number} from OMS`)
   const oms_lumisection_url = `${OMS_URL}/${OMS_LUMISECTIONS(run_number)}`;
   // Keep fetching until totalresourcecount is # of lumisections
   const oms_lumisection_response = await instance

From ec7123308dc9f4d91d045d29c5039d4368b392fb Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Mon, 10 Jun 2024 17:26:05 +0200
Subject: [PATCH 02/15] Skip runs lower than MINIMUM_CMS_RUN_NUMBER

---
 runregistry_backend/config/config.js   | 2 ++
 runregistry_backend/cron/1.get_runs.js | 7 ++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/runregistry_backend/config/config.js b/runregistry_backend/config/config.js
index 61bbd2d..f9c3894 100644
--- a/runregistry_backend/config/config.js
+++ b/runregistry_backend/config/config.js
@@ -20,6 +20,8 @@ commonVars = {
     idle: 20000,
     acquire: 2000000,
   },
+  // Ignore run numbers lower than this one
+  MINIMUM_CMS_RUN_NUMBER: 100000,
   // DQMGUI
   WAITING_DQM_GUI_CONSTANT: 'waiting dqm gui',
   DQM_GUI_URL: 'https://cmsweb.cern.ch/dqm/offline/data/json/samples?match=',
diff --git a/runregistry_backend/cron/1.get_runs.js b/runregistry_backend/cron/1.get_runs.js
index 2f9a93b..0e6cc0a 100755
--- a/runregistry_backend/cron/1.get_runs.js
+++ b/runregistry_backend/cron/1.get_runs.js
@@ -140,7 +140,12 @@ const calculate_runs_to_update = (fetched_runs, last_saved_runs) => {
   fetched_runs.forEach((fetched_run) => {
     // If the run_number is less than the minimum of the already saved runs, then it is one from the past, which needs to be updated. Else we compare timestamps
     if (fetched_run.run_number < min_run_number) {
-      runs_to_update.push(fetched_run);
+      if (fetched_run.run_number > MINIMUM_CMS_RUN_NUMBER) {
+        runs_to_update.push(fetched_run);
+
+      } else {
+        console.log(`Run number ${fetched_run.run_number} is lower than the threshold MINIMUM_CMS_RUN_NUMBER (${MINIMUM_CMS_RUN_NUMBER}), ignoring`)
+      }
     } else {
       // If the run_number is inside the existing already saved runs, then we check for the timestamp:
       last_saved_runs.forEach((existing_run) => {

From 9650e4d6e42beff5e5f64f7ffd9c7c39bdb6fd0e Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Mon, 10 Jun 2024 17:26:18 +0200
Subject: [PATCH 03/15] Fix cronjob options

---
 runregistry_backend/config/config.js          | 28 +++++++++----------
 runregistry_backend/cron/1.get_runs.js        |  5 ++--
 .../cron_datasets/2.ping_dqm_gui.js           |  4 +--
 3 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/runregistry_backend/config/config.js b/runregistry_backend/config/config.js
index f9c3894..f5241c9 100644
--- a/runregistry_backend/config/config.js
+++ b/runregistry_backend/config/config.js
@@ -54,8 +54,8 @@ module.exports = {
       'http://localhost:9500',
     OMS_RUNS: (number_of_runs = 10) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 30,
-    SECONDS_PER_DQM_GUI_CHECK: 6000,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 30,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 6000,
     JSON_PROCESSING_ENABLED: false,
     OMS_GET_RUNS_CRON_ENABLED: false,
     DQM_GUI_PING_CRON_ENABLED: false
@@ -67,8 +67,8 @@ module.exports = {
       'http://localhost:9500',
     OMS_RUNS: (number_of_runs = 10) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 30,
-    SECONDS_PER_DQM_GUI_CHECK: 6000,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 30,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 6000,
     OMS_GET_RUNS_CRON_ENABLED: false,
     JSON_PROCESSING_ENABLED: false,
     DQM_GUI_PING_CRON_ENABLED: false
@@ -79,8 +79,8 @@ module.exports = {
     API_URL: 'http://localhost:9500',
     OMS_RUNS: (number_of_runs = 10) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 3600,
-    SECONDS_PER_DQM_GUI_CHECK: 3600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 3600,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 3600,
   },
   // Old "bare-metal" production
   production: {
@@ -88,8 +88,8 @@ module.exports = {
     API_URL: 'http://localhost:9500',
     OMS_RUNS: (number_of_runs = 15) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 180,
-    SECONDS_PER_DQM_GUI_CHECK: 3600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 180,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 3600,
   },
   // Dev kubernetes flavor which means no cronjobs, no JSON processing
   dev_kubernetes: {
@@ -97,8 +97,8 @@ module.exports = {
     API_URL: 'http://runregistry-backend:9500',
     OMS_RUNS: (number_of_runs = 49) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 30,
-    SECONDS_PER_DQM_GUI_CHECK: 600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 30,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 600,
     OMS_GET_RUNS_CRON_ENABLED: false,
     JSON_PROCESSING_ENABLED: false,
     DQM_GUI_PING_CRON_ENABLED: false
@@ -109,8 +109,8 @@ module.exports = {
     API_URL: 'http://runregistry-backend:9500',
     OMS_RUNS: (number_of_runs = 15) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 60,
-    SECONDS_PER_DQM_GUI_CHECK: 600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 10,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 15,
   },
   // Production config for kubernetes
   prod_kubernetes: {
@@ -118,8 +118,8 @@ module.exports = {
     API_URL: 'http://runregistry-backend:9500',
     OMS_RUNS: (number_of_runs = 15) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 180,
-    SECONDS_PER_DQM_GUI_CHECK: 3600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 3,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
 
   // The online components are also the rr_lumisection_whitelist
diff --git a/runregistry_backend/cron/1.get_runs.js b/runregistry_backend/cron/1.get_runs.js
index 0e6cc0a..87590c7 100755
--- a/runregistry_backend/cron/1.get_runs.js
+++ b/runregistry_backend/cron/1.get_runs.js
@@ -10,7 +10,8 @@ const {
   OMS_GET_RUNS_CRON_ENABLED,
   API_URL,
   RUNS_PER_API_CALL,
-  SECONDS_PER_API_CALL,
+  OMS_API_CALL_EVERY_NTH_MINUTE,
+  MINIMUM_CMS_RUN_NUMBER
 } = config[process.env.ENV || 'development'];
 const { save_runs, update_runs } = require('./2.save_or_update_runs');
 
@@ -95,7 +96,7 @@ const fetch_runs = async (
 
 if (OMS_GET_RUNS_CRON_ENABLED === true) {
   const job = new CronJob(
-    `*/${SECONDS_PER_API_CALL} * * * * *`,
+    `*/${OMS_API_CALL_EVERY_NTH_MINUTE} * * * *`,
     handleErrors(fetch_runs, 'cron/1.get_runs.js # Error fetching new runs ')
   ).start();
 }
diff --git a/runregistry_backend/cron_datasets/2.ping_dqm_gui.js b/runregistry_backend/cron_datasets/2.ping_dqm_gui.js
index 5e72d89..443a83c 100644
--- a/runregistry_backend/cron_datasets/2.ping_dqm_gui.js
+++ b/runregistry_backend/cron_datasets/2.ping_dqm_gui.js
@@ -24,7 +24,7 @@ const { handleErrors } = require('../utils/error_handlers');
 const {
   API_URL,
   DQM_GUI_URL,
-  SECONDS_PER_DQM_GUI_CHECK,
+  DQM_GUI_CHECK_EVERY_NTH_MINUTE,
   WAITING_DQM_GUI_CONSTANT,
   DQM_GUI_PING_CRON_ENABLED
 } = require('../config/config')[process.env.ENV || 'development'];
@@ -231,7 +231,7 @@ const ping_dqm_gui = async () => {
 // Cron job starts:
 if (DQM_GUI_PING_CRON_ENABLED === true) {
   const job = new CronJob(
-    `*/${SECONDS_PER_DQM_GUI_CHECK} * * * * *`,
+    `*/${DQM_GUI_CHECK_EVERY_NTH_MINUTE} * * * *`,
     handleErrors(ping_dqm_gui, 'cron_datasets/2.ping_dqm_gui.js # Error pinging DQM GUI')
   ).start();
 } else {

From 1dc5e03e9dcc06df58a73dcaf01cda1cac244c0f Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Mon, 10 Jun 2024 17:30:50 +0200
Subject: [PATCH 04/15] Update configs for deployments

---
 runregistry_backend/config/config.js | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/runregistry_backend/config/config.js b/runregistry_backend/config/config.js
index f5241c9..594511b 100644
--- a/runregistry_backend/config/config.js
+++ b/runregistry_backend/config/config.js
@@ -1,4 +1,4 @@
-// Common configuration used by all deployment modes
+// Common configuration used and overriden by all deployment modes.
 commonVars = {
   // Database config
   username: process.env.DB_USERNAME || 'postgres',
@@ -26,6 +26,7 @@ commonVars = {
   WAITING_DQM_GUI_CONSTANT: 'waiting dqm gui',
   DQM_GUI_URL: 'https://cmsweb.cern.ch/dqm/offline/data/json/samples?match=',
   DQM_GUI_PING_CRON_ENABLED: true,
+  DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60, // This needs to be <=60
   // OMS
   OMS_URL: `https://cmsoms.cern.ch/agg/api/v1`,
   OMS_GET_RUNS_CRON_ENABLED: true, // Get runs from OMS periodically or not
@@ -35,6 +36,7 @@ commonVars = {
   CLIENT_SECRET: process.env.CLIENT_SECRET,
   OMS_AUDIENCE: 'cmsoms-prod',
   RUNS_PER_API_CALL: 49,
+  OMS_API_CALL_EVERY_NTH_MINUTE: 30, // This needs to be <=60
   // Redis
   // redis://:<pass>@<host>:<port>
   REDIS_URL: `redis://${process.env.REDIS_PASSWORD ? ':' + process.env.REDIS_PASSWORD + '@' : ''}${process.env.REDIS_HOST || '127.0.0.1'}:${process.env.REDIS_PORT || 6379}`,
@@ -55,7 +57,7 @@ module.exports = {
     OMS_RUNS: (number_of_runs = 10) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 30,
-    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 6000,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60,
     JSON_PROCESSING_ENABLED: false,
     OMS_GET_RUNS_CRON_ENABLED: false,
     DQM_GUI_PING_CRON_ENABLED: false
@@ -68,7 +70,7 @@ module.exports = {
     OMS_RUNS: (number_of_runs = 10) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 30,
-    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 6000,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60,
     OMS_GET_RUNS_CRON_ENABLED: false,
     JSON_PROCESSING_ENABLED: false,
     DQM_GUI_PING_CRON_ENABLED: false
@@ -79,8 +81,8 @@ module.exports = {
     API_URL: 'http://localhost:9500',
     OMS_RUNS: (number_of_runs = 10) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    OMS_API_CALL_EVERY_NTH_MINUTE: 3600,
-    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 3600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 5,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
   // Old "bare-metal" production
   production: {
@@ -88,8 +90,8 @@ module.exports = {
     API_URL: 'http://localhost:9500',
     OMS_RUNS: (number_of_runs = 15) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    OMS_API_CALL_EVERY_NTH_MINUTE: 180,
-    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 3600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 3,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
   // Dev kubernetes flavor which means no cronjobs, no JSON processing
   dev_kubernetes: {
@@ -98,7 +100,7 @@ module.exports = {
     OMS_RUNS: (number_of_runs = 49) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 30,
-    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 600,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60,
     OMS_GET_RUNS_CRON_ENABLED: false,
     JSON_PROCESSING_ENABLED: false,
     DQM_GUI_PING_CRON_ENABLED: false

From b1ebd1ad92c6ff8366a74309c24f2ac201fac607 Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Mon, 10 Jun 2024 17:41:03 +0200
Subject: [PATCH 05/15] Rename config variable, change value

---
 runregistry_backend/config/config.js   | 4 +++-
 runregistry_backend/cron/1.get_runs.js | 8 ++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/runregistry_backend/config/config.js b/runregistry_backend/config/config.js
index 594511b..327dbbb 100644
--- a/runregistry_backend/config/config.js
+++ b/runregistry_backend/config/config.js
@@ -35,7 +35,7 @@ commonVars = {
   CLIENT_ID: 'rr-api-client',
   CLIENT_SECRET: process.env.CLIENT_SECRET,
   OMS_AUDIENCE: 'cmsoms-prod',
-  RUNS_PER_API_CALL: 49,
+  OMS_RUNS_PER_API_CALL: 49,
   OMS_API_CALL_EVERY_NTH_MINUTE: 30, // This needs to be <=60
   // Redis
   // redis://:<pass>@<host>:<port>
@@ -112,6 +112,7 @@ module.exports = {
     OMS_RUNS: (number_of_runs = 15) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 10,
+    OMS_RUNS_PER_API_CALL: 25,
     DQM_GUI_CHECK_EVERY_NTH_MINUTE: 15,
   },
   // Production config for kubernetes
@@ -120,6 +121,7 @@ module.exports = {
     API_URL: 'http://runregistry-backend:9500',
     OMS_RUNS: (number_of_runs = 15) =>
       `runs?sort=-last_update&page[limit]=${number_of_runs}`,
+    OMS_RUNS_PER_API_CALL: 49,
     OMS_API_CALL_EVERY_NTH_MINUTE: 3,
     DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
diff --git a/runregistry_backend/cron/1.get_runs.js b/runregistry_backend/cron/1.get_runs.js
index 87590c7..3e0da81 100755
--- a/runregistry_backend/cron/1.get_runs.js
+++ b/runregistry_backend/cron/1.get_runs.js
@@ -9,7 +9,7 @@ const {
   OMS_RUNS,
   OMS_GET_RUNS_CRON_ENABLED,
   API_URL,
-  RUNS_PER_API_CALL,
+  OMS_RUNS_PER_API_CALL,
   OMS_API_CALL_EVERY_NTH_MINUTE,
   MINIMUM_CMS_RUN_NUMBER
 } = config[process.env.ENV || 'development'];
@@ -24,7 +24,7 @@ const instance = axios.create({
 
 // Will call itself recursively if all runs are new
 const fetch_runs = async (
-  fetch_amount = RUNS_PER_API_CALL,
+  fetch_amount = OMS_RUNS_PER_API_CALL,
   first_time = true
 ) => {
   const oms_url = `${OMS_URL}/${OMS_RUNS(fetch_amount)}`;
@@ -41,7 +41,7 @@ const fetch_runs = async (
       setTimeout(resolve, 2000);
     });
   }
-
+  console.debug(`Fetching the ${OMS_RUNS(fetch_amount)} last updated OMS runs`)
   const oms_response = await instance.get(oms_url, {
     headers,
   });
@@ -58,7 +58,7 @@ const fetch_runs = async (
   let fetched_runs = first_time
     ? all_fetched_runs
     : all_fetched_runs.slice(fetch_amount / 2);
-  console.debug("Querying the last 50 updated OMS runs")
+  console.debug("Querying the last 50 updated RR runs")
   const { data: last_saved_runs } = await axios.get(
     `${API_URL}/runs_lastupdated_50`
   );

From ac31088649958592ba5b1303216aad16c06ab672 Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Mon, 10 Jun 2024 17:44:49 +0200
Subject: [PATCH 06/15] Fix log

---
 runregistry_backend/cron/1.get_runs.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runregistry_backend/cron/1.get_runs.js b/runregistry_backend/cron/1.get_runs.js
index 3e0da81..efab7e5 100755
--- a/runregistry_backend/cron/1.get_runs.js
+++ b/runregistry_backend/cron/1.get_runs.js
@@ -41,7 +41,7 @@ const fetch_runs = async (
       setTimeout(resolve, 2000);
     });
   }
-  console.debug(`Fetching the ${OMS_RUNS(fetch_amount)} last updated OMS runs`)
+  console.debug(`Fetching the ${fetch_amount} last updated OMS runs`)
   const oms_response = await instance.get(oms_url, {
     headers,
   });

From a4b12a8f63fb6f0bdb333606e2a3dc997a811c4d Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Tue, 11 Jun 2024 11:39:16 +0200
Subject: [PATCH 07/15] Adjust request periods, rename function

---
 runregistry_backend/config/config.js   | 19 ++++---------------
 runregistry_backend/cron/1.get_runs.js |  6 +++---
 2 files changed, 7 insertions(+), 18 deletions(-)

diff --git a/runregistry_backend/config/config.js b/runregistry_backend/config/config.js
index 327dbbb..a7ba26d 100644
--- a/runregistry_backend/config/config.js
+++ b/runregistry_backend/config/config.js
@@ -32,6 +32,9 @@ commonVars = {
   OMS_GET_RUNS_CRON_ENABLED: true, // Get runs from OMS periodically or not
   OMS_SPECIFIC_RUN: (run_number) => `runs?filter[run_number]=${run_number}`,
   OMS_LUMISECTIONS: (run_number) => `lumisections?filter[run_number]=${run_number}&page[limit]=5000`,
+  // The default value here does not play a role, really, it's overridden by OMS_RUNS_PER_API_CALL
+  OMS_RUNS_ENDPOINT: (number_of_runs = 15) =>
+    `runs?sort=-last_update&page[limit]=${number_of_runs}`,
   CLIENT_ID: 'rr-api-client',
   CLIENT_SECRET: process.env.CLIENT_SECRET,
   OMS_AUDIENCE: 'cmsoms-prod',
@@ -54,8 +57,6 @@ module.exports = {
     ...commonVars,
     API_URL: process.env.DOCKER_POSTGRES ? 'http://dev:9500' :
       'http://localhost:9500',
-    OMS_RUNS: (number_of_runs = 10) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 30,
     DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60,
     JSON_PROCESSING_ENABLED: false,
@@ -67,8 +68,6 @@ module.exports = {
     ...commonVars,
     API_URL: process.env.DOCKER_POSTGRES ? 'http://dev:9500' :
       'http://localhost:9500',
-    OMS_RUNS: (number_of_runs = 10) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 30,
     DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60,
     OMS_GET_RUNS_CRON_ENABLED: false,
@@ -79,8 +78,6 @@ module.exports = {
   staging: {
     ...commonVars,
     API_URL: 'http://localhost:9500',
-    OMS_RUNS: (number_of_runs = 10) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 5,
     DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
@@ -88,8 +85,6 @@ module.exports = {
   production: {
     ...commonVars,
     API_URL: 'http://localhost:9500',
-    OMS_RUNS: (number_of_runs = 15) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 3,
     DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
@@ -97,8 +92,6 @@ module.exports = {
   dev_kubernetes: {
     ...commonVars,
     API_URL: 'http://runregistry-backend:9500',
-    OMS_RUNS: (number_of_runs = 49) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 30,
     DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60,
     OMS_GET_RUNS_CRON_ENABLED: false,
@@ -109,8 +102,6 @@ module.exports = {
   staging_kubernetes: {
     ...commonVars,
     API_URL: 'http://runregistry-backend:9500',
-    OMS_RUNS: (number_of_runs = 15) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_API_CALL_EVERY_NTH_MINUTE: 10,
     OMS_RUNS_PER_API_CALL: 25,
     DQM_GUI_CHECK_EVERY_NTH_MINUTE: 15,
@@ -119,10 +110,8 @@ module.exports = {
   prod_kubernetes: {
     ...commonVars,
     API_URL: 'http://runregistry-backend:9500',
-    OMS_RUNS: (number_of_runs = 15) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
     OMS_RUNS_PER_API_CALL: 49,
-    OMS_API_CALL_EVERY_NTH_MINUTE: 3,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 2,
     DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
 
diff --git a/runregistry_backend/cron/1.get_runs.js b/runregistry_backend/cron/1.get_runs.js
index efab7e5..4a415b4 100755
--- a/runregistry_backend/cron/1.get_runs.js
+++ b/runregistry_backend/cron/1.get_runs.js
@@ -6,7 +6,7 @@ const https = require('https');
 const config = require('../config/config');
 const {
   OMS_URL,
-  OMS_RUNS,
+  OMS_RUNS_ENDPOINT,
   OMS_GET_RUNS_CRON_ENABLED,
   API_URL,
   OMS_RUNS_PER_API_CALL,
@@ -27,9 +27,9 @@ const fetch_runs = async (
   fetch_amount = OMS_RUNS_PER_API_CALL,
   first_time = true
 ) => {
-  const oms_url = `${OMS_URL}/${OMS_RUNS(fetch_amount)}`;
-  // insert cookie that will authenticate OMS request:
+  const oms_url = `${OMS_URL}/${OMS_RUNS_ENDPOINT(fetch_amount)}`;
 
+  // insert cookie that will authenticate OMS request:
   if (first_time) {
     headers = {
       Authorization: `Bearer ${await getToken()}`,

From 717a172d24fbe87a49703ef8308e12956abf13a6 Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Fri, 14 Jun 2024 12:12:04 +0200
Subject: [PATCH 08/15] Add a warning when an LS is missing from OMS

---
 .../cron/saving_updating_runs_lumisections_utils.js         | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js b/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js
index 62e2dcc..e4dd599 100644
--- a/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js
+++ b/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js
@@ -53,7 +53,11 @@ exports.get_OMS_lumisections = handleErrors(async (run_number) => {
 
   // We add luminosity information
   oms_lumisections = oms_lumisections.map(
-    ({ recorded_lumi, delivered_lumi }, index, oms_lumisections) => {
+    ({ recorded_lumi, delivered_lumi, lumisection_number }, index, oms_lumisections) => {
+      // Check that the index matches the LS number
+      if (index + 1 !== lumisection_number) {
+        console.warn(`Inconsistency in Run ${run_number}: Lumisection ${index} does not match OMS lumisection_number (${lumisection_number})`)
+      }
       // If any of them is null, then the per_lumi are all null
       if (recorded_lumi === null || delivered_lumi === null) {
         return {

From 54967a609fd138231b33068c486e322d1b51ce74 Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Fri, 14 Jun 2024 14:12:20 +0200
Subject: [PATCH 09/15] Fix warn method override

---
 runregistry_backend/app.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runregistry_backend/app.js b/runregistry_backend/app.js
index f229335..3aabd0a 100644
--- a/runregistry_backend/app.js
+++ b/runregistry_backend/app.js
@@ -33,7 +33,7 @@ function getLogger(level, originalLogger = console.log) {
 // override console log to use timestamp
 console.debug = console.log = getLogger("DEBUG")
 console.info = getLogger("INFO")
-console.warning = getLogger("WARNING")
+console.warn = getLogger("WARNING")
 console.error = getLogger("ERROR", console.error)
 
 // Logging for sanity

From 616925c6ea95c3affd2cdc3f47097753945d4a52 Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Fri, 14 Jun 2024 14:15:49 +0200
Subject: [PATCH 10/15] Properly configure loggers

---
 runregistry_backend/app.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runregistry_backend/app.js b/runregistry_backend/app.js
index 3aabd0a..9514687 100644
--- a/runregistry_backend/app.js
+++ b/runregistry_backend/app.js
@@ -32,8 +32,8 @@ function getLogger(level, originalLogger = console.log) {
 
 // override console log to use timestamp
 console.debug = console.log = getLogger("DEBUG")
-console.info = getLogger("INFO")
-console.warn = getLogger("WARNING")
+console.info = getLogger("INFO", console.info)
+console.warn = getLogger("WARNING", console.warn)
 console.error = getLogger("ERROR", console.error)
 
 // Logging for sanity

From d4095d6e7264a26749cd2f81489332afd5d0ecae Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Fri, 14 Jun 2024 14:16:33 +0200
Subject: [PATCH 11/15] Properly configure frontend loggers

---
 runregistry_frontend/server.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runregistry_frontend/server.js b/runregistry_frontend/server.js
index a95cf27..fed9060 100755
--- a/runregistry_frontend/server.js
+++ b/runregistry_frontend/server.js
@@ -31,8 +31,8 @@ function getLogger(level, originalLogger = console.log) {
 
 // override console log to use timestamp
 console.debug = console.log = getLogger("DEBUG")
-console.info = getLogger("INFO")
-console.warning = getLogger("WARNING")
+console.info = getLogger("INFO", console.info)
+console.warn = getLogger("WARNING", console.warn)
 console.error = getLogger("ERROR", console.error)
 
 app.prepare().then(() => {

From a47828ec58d8cd903bf3571adb9382047d6c6ac5 Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Fri, 14 Jun 2024 14:21:05 +0200
Subject: [PATCH 12/15] Remove temporary config

---
 runregistry_frontend/Dockerfile       | 2 +-
 runregistry_frontend/config/config.js | 6 ------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/runregistry_frontend/Dockerfile b/runregistry_frontend/Dockerfile
index 3e36680..317f248 100755
--- a/runregistry_frontend/Dockerfile
+++ b/runregistry_frontend/Dockerfile
@@ -5,7 +5,7 @@ WORKDIR /usr/src/app
 
 ENV NODE_ENV production
 # TODO: remove qa
-ENV ENV production_qa
+ENV ENV production
 ENV NEXT_TELEMETRY_DISABLED 1
 # Installing dependencies
 COPY package*.json ./
diff --git a/runregistry_frontend/config/config.js b/runregistry_frontend/config/config.js
index b2e3793..01b5ff3 100755
--- a/runregistry_frontend/config/config.js
+++ b/runregistry_frontend/config/config.js
@@ -20,12 +20,6 @@ const config = {
     root_url_prefix: '',
     api_url: 'https://dev-cmsrunregistry.web.cern.ch/api',
   },
-  // Production Openshift project, used for transitioning
-  // TODO: Remove
-  production_qa: {
-    root_url_prefix: '',
-    api_url: 'https://cmsrunregistry-qa.web.cern.ch/api',
-  },
   // Production Openshift project
   production: {
     root_url_prefix: '',

From 8a3c66fd69b7d1e8a952a45ba6aa5119d982a5fc Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Tue, 2 Jul 2024 15:00:51 +0200
Subject: [PATCH 13/15] Relative import fix

---
 scripts/runregistry_sso_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/runregistry_sso_manager.py b/scripts/runregistry_sso_manager.py
index 8f9998e..1de3b03 100644
--- a/scripts/runregistry_sso_manager.py
+++ b/scripts/runregistry_sso_manager.py
@@ -2,7 +2,7 @@
 import logging
 import argparse
 import requests
-from .utils import CernAuthToken
+from utils import CernAuthToken
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(encoding="utf-8", level=logging.DEBUG)

From 1ef52f252e0380bec9cf9c1484543930b31839f8 Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Tue, 2 Jul 2024 17:45:37 +0200
Subject: [PATCH 14/15] Log errors using console.error

---
 runregistry_backend/utils/error_handlers.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/runregistry_backend/utils/error_handlers.js b/runregistry_backend/utils/error_handlers.js
index 166de24..ebec354 100644
--- a/runregistry_backend/utils/error_handlers.js
+++ b/runregistry_backend/utils/error_handlers.js
@@ -1,16 +1,16 @@
 // Handle Generic Errors:
 exports.handleErrors = (fn, error_message) => {
-    return function(...params) {
+    return function (...params) {
         return fn(...params).catch(err => {
-            console.log(err);
-            console.log(error_message, err.message);
+            console.error(err);
+            console.error(error_message, err.message);
             // console.log(err.stack);
         });
     };
 };
 
 exports.catchAPIError = fn => {
-    return function(req, res, next) {
+    return function (req, res, next) {
         return fn(req, res, next).catch(next);
     };
 };
@@ -19,16 +19,16 @@ exports.catchAPIErrorHotShot = fn => (req, res, next) =>
     fn(req, res, next).catch(next);
 
 exports.expressError = (err, req, res, next) => {
-    console.log(err.message || err);
-    console.log(err);
+    console.error(err.message || err);
+    console.error(err);
     res.status(500).json({ err: err.message || err });
 };
 
 // We want the error to fail so that a run does not get saved/updated
 exports.handleCronErrors = (fn, error_message) => {
-    return function(...params) {
+    return function (...params) {
         return fn(...params).catch(err => {
-            console.log(error_message, err.message);
+            console.error(error_message, err.message);
             // console.log(err.stack);
             throw err;
         });

From d13da5b4f93d3bf66d05f679f119289c1fdf547a Mon Sep 17 00:00:00 2001
From: Dimitris Papagiannis <d.papag.a@gmail.com>
Date: Wed, 31 Jul 2024 17:10:46 +0200
Subject: [PATCH 15/15] Add prescale_index and prescale_name to the OMS
 attributes we save

---
 runregistry_backend/config/config.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runregistry_backend/config/config.js b/runregistry_backend/config/config.js
index a7ba26d..b2fbc33 100644
--- a/runregistry_backend/config/config.js
+++ b/runregistry_backend/config/config.js
@@ -172,6 +172,7 @@ module.exports = {
       'beam1_stable', 'hbhec_ready',
       'beam2_stable', 'beam2_present',
       'gemp_ready', 'gemm_ready',
+      'prescale_index', 'prescale_name'
     ],
   oms_lumisection_luminosity_whitelist:
     [