Merge branch 'main' into dependabot/docker/docker/main/python-3.13.0-…

…slim-bookworm

.gitignore

@@ -1,27 +1,26 @@
 .DS_Store
 __pycache__
-config.yml
-config.js
-docker/nginx/api.conf
-docker/nginx/frontend.conf
-docker/nginx/block.conf
-docker/nginx/ssl/*
-!docker/nginx/ssl/README.md
-frontend/js/helpers/config.js
-sftp-config.json
+/config.yml
+/docker/nginx/api.conf
+/docker/nginx/frontend.conf
+/docker/nginx/block.conf
+/docker/nginx/ssl/*
+!/docker/nginx/ssl/README.md
+/frontend/js/helpers/config.js
+/sftp-config.json
 TODO
-flows
-docker/compose.yml
+/docker/compose.yml
 metric-provider-binary
 .vscode
 static-binary
 .pytest_cache
-test-compose.yml
-tests/structure.sql
-tools/sgx_enable
-venv/
-lib/hardware_info_root.py
-tools/cluster/cleanup.sh
-node_modules/
-lib/c/parse_int.o
-tools/backup
+/docker/test-compose.yml
+/tests/structure.sql
+/tools/sgx_enable
+/venv/
+/lib/hardware_info_root.py
+/tools/cluster/cleanup.sh
+/node_modules/
+/lib/c/parse_int.o
+/tools/backup
+/manager-config.yml

api/api_helpers.py

@@ -220,6 +220,7 @@ def get_timeline_query(uri, filename, machine_id, branch, metrics, phase, start_
                 {detail_name_condition}
                 {limit_365_condition}
                 AND r.commit_timestamp IS NOT NULL
+                AND r.failed IS FALSE
             ORDER BY
                 p.metric ASC, p.detail_name ASC,
                 p.phase ASC, {sorting_condition}

cron/carbondb_compress.py

@@ -1,6 +1,8 @@
 import faulthandler
 faulthandler.enable()  # will catch segfaults and write to stderr
 
+import os
+
 from lib.global_config import GlobalConfig
 from lib.db import DB
 from lib import error_helpers
@@ -10,8 +12,15 @@
 # During this process we also transform all text fields and transform them to integers and drop them into normalized
 # joined tables.
 
+########### Remove NULL values from tags
+# UPDATE carbondb_data_raw
+# SET tags = array_remove(tags, NULL)
+# WHERE array_position(tags, NULL) IS NOT NULL;
+
+
 def compress_carbondb_raw():
     query = '''
+
         INSERT INTO carbondb_types (type, user_id)
         SELECT DISTINCT type, user_id
         FROM carbondb_data_raw
@@ -39,7 +48,7 @@ def compress_carbondb_raw():
 
         DROP TABLE IF EXISTS carbondb_data_raw_tmp;
 
-        CREATE TABLE carbondb_data_raw_tmp AS
+        CREATE TEMPORARY TABLE carbondb_data_raw_tmp AS
         SELECT * FROM carbondb_data_raw;
 
         UPDATE carbondb_data_raw_tmp AS cdrt
@@ -92,7 +101,7 @@ def compress_carbondb_raw():
                 DATE_TRUNC('day', TO_TIMESTAMP(cdr.time / 1000000)),
                 SUM(cdr.energy_kwh),
                 SUM(cdr.carbon_kg),
-                (SUM(cdr.carbon_kg)*1e3) / SUM(cdr.energy_kwh), -- weighted average instead of just averaging carbon_intensity
+                COALESCE(SUM(cdr.carbon_kg)*1e3 / NULLIF(SUM(cdr.energy_kwh), 0), 0), -- weighted average instead of just averaging carbon_intensity. Since the solar panel might not be producing power at all for a day, which results in 0, we need to COALESCE and insert 0 in this case
                 COUNT(*),
                 cdr.user_id
             FROM
@@ -119,6 +128,7 @@ def compress_carbondb_raw():
 
 if __name__ == '__main__':
     try:
+        GlobalConfig().override_config(config_location=f"{os.path.dirname(os.path.realpath(__file__))}/../manager-config.yml")
         compress_carbondb_raw()
     except Exception as exc: # pylint: disable=broad-except
         error_helpers.log_error(f'Processing in {__file__} failed.', exception=exc, machine=GlobalConfig().config['machine']['description'])

cron/carbondb_copy_over_and_remove_duplicates.py

@@ -1,6 +1,8 @@
 import faulthandler
 faulthandler.enable()  # will catch segfaults and write to stderr
 
+import os
+
 from lib.global_config import GlobalConfig
 from lib.db import DB
 from lib import error_helpers
@@ -75,7 +77,7 @@ def validate_table_constraints():
         FROM
             information_schema.columns
         WHERE
-            table_name = 'carbondb_data_raw'  -- replace 'your_table' with the table name
+            table_name = 'carbondb_data_raw'
             AND column_name IN ('user_id', 'time', 'energy_kwh', 'carbon_kg', 'carbon_intensity_g', 'type', 'project', 'machine', 'source', 'tags')
     ''')
 
@@ -105,6 +107,7 @@ def remove_duplicates():
 
 if __name__ == '__main__':
     try:
+        GlobalConfig().override_config(config_location=f"{os.path.dirname(os.path.realpath(__file__))}/../manager-config.yml")
         copy_over_eco_ci()
         copy_over_gmt()
         remove_duplicates()

docker/compose.yml.example

@@ -39,7 +39,7 @@ services:
       - 9142:9142
     restart: always
     volumes:
-      - green-coding-shared-tmp:/tmp
+      - green-coding-shared-sockets:/tmp/gmt-sockets
       - PATH_TO_GREEN_METRICS_TOOL_REPO:/var/www/green-metrics-tool:ro
       - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
       - ./nginx/api.conf:/etc/nginx/conf.d/api_nginx.conf:ro
@@ -57,7 +57,7 @@ services:
       - green-coding-postgres
     restart: always
     volumes:
-      - green-coding-shared-tmp:/tmp
+      - green-coding-shared-sockets:/tmp/gmt-sockets
       - PATH_TO_GREEN_METRICS_TOOL_REPO:/var/www/green-metrics-tool:ro
       # - TEST_CONFIG_SETUP:/var/www/green-metrics-tool/config.yml
 
@@ -71,10 +71,10 @@ services:
 
 
 # Although it would help us very much performance-wise the
-# Shared volume with only temporary data, green-coding-shared-tmp, must be
+# Shared volume with only temporary data, green-coding-shared-sockets, must be
 # of type overlayfs and cannot be tmpfs (https://docs.docker.com/storage/tmpfs/)
 # tmpfs being a RAM stored and faster volume is sadly not shareable between containers
 volumes:
   green-coding-postgres-data:
-  green-coding-shared-tmp:
+  green-coding-shared-sockets:
   green-coding-redis-data:

docker/nginx/api.conf.example

@@ -16,7 +16,7 @@ server {
     }
 
     location / {
-        proxy_pass http://unix:/tmp/green-coding-api.sock;
+        proxy_pass http://unix:/tmp/gmt-sockets/api.sock;
         # Do not change this
         proxy_set_header Host $http_host;
         proxy_set_header X-Real-IP $remote_addr;

docker/requirements.txt

@@ -1,17 +1,17 @@
 gunicorn==23.0.0
 psycopg[binary]==3.2.3
 psycopg_pool==3.2.3
-fastapi[standard]==0.115.2
+fastapi[standard]==0.115.4
 starlette>=0.35
 uvicorn[standard]==0.32.0
 pandas==2.2.3
 PyYAML==6.0.2
 anybadge==1.14.0
-orjson==3.10.9
+orjson==3.10.10
 scipy==1.14.1
 schema==0.7.7
 deepdiff==8.0.1
-redis==5.1.1
+redis==5.2.0
 hiredis==3.0.0
 requests==2.32.3
 uvicorn-worker==0.2.0

docker/startup_gunicorn.sh

@@ -7,7 +7,7 @@ source /var/www/startup/venv/bin/activate
 --error-logfile=- \
 --worker-tmp-dir=/dev/shm \
 --worker-class=uvicorn_worker.UvicornWorker \
---bind unix:/tmp/green-coding-api.sock \
+--bind unix:/tmp/gmt-sockets/api.sock \
 -m 007 \
 --user www-data \
 --chdir /var/www/green-metrics-tool/api \

frontend/js/carbondb.js

@@ -136,7 +136,10 @@ const fillBarChart = (y_axis, legend, labels, series) => {
             return `<strong>${escapeString(labels[params.componentIndex].date)}</strong><br>
                     Type: ${escapeString(labels[params.componentIndex].type)}<br>
                     Value: ${escapeString(labels[params.componentIndex].value)} ${escapeString(labels[params.componentIndex].unit)}<br>
-
+                    Project: ${escapeString(labels[params.componentIndex].project)}<br>
+                    Machine: ${escapeString(labels[params.componentIndex].machine)}<br>
+                    Source: ${escapeString(labels[params.componentIndex].source)}<br>
+                    Tags: ${escapeString(labels[params.componentIndex].tags)}<br>
                     `;
         }
 
@@ -220,6 +223,10 @@ const processData = (measurements) => {
         carbon_barchart_data.labels.push({
             type: dimensions_lookup['types'][type],
             date: date,
+            project: dimensions_lookup['projects'][project],
+            machine: dimensions_lookup['machines'][machine],
+            source: dimensions_lookup['sources'][source],
+            tags: tags.map( el => dimensions_lookup['tags'][el]),
             value: carbon,
             unit: 'kg',
         })

frontend/js/helpers/config.js.example

@@ -379,8 +379,8 @@ const phase_time_metric_condition = (metric) => {
     return false;
 }
 
-const machine_carbon_metric_condition = (metric) => {
-    if(metric.match(/^.*_carbon_.*_machine$/) !== null) return true;
+const psu_machine_carbon_metric_condition = (metric) => {
+    if(metric.match(/^psu_carbon_.*_machine$/) !== null) return true;
     return false;
 }
 

frontend/js/helpers/metric-boxes.js

@@ -372,7 +372,7 @@ const updateKeyMetric = (phase, metric_name, clean_name, detail_name, value, std
         selector = '.software-carbon-intensity';
     } else if(machine_power_metric_condition(metric_name)) {
         selector = '.machine-power';
-    } else if(machine_carbon_metric_condition(metric_name)) {
+    } else if(psu_machine_carbon_metric_condition(metric_name)) {
         selector = '.machine-co2';
     } else {
         return; // could not match key metric

frontend/js/helpers/phase-stats.js

@@ -187,7 +187,7 @@ const displayComparisonMetrics = (phase_stats_object) => {
                         bottom_chart_present_keys[key] = true
                     }
 
-                    if (phase_stats_object.comparison_case == null && machine_carbon_metric_condition(metric_name)) {
+                    if (phase_stats_object.comparison_case == null && psu_machine_carbon_metric_condition(metric_name)) {
                         if(co2_calculated) {
                             showWarning(phase, 'CO2 was already calculated! Do you have multiple machine energy reporters set?');
                         }

requirements-dev.txt

@@ -2,5 +2,5 @@
 pydantic==2.9.2
 pytest==8.3.3
 pylint==3.3.1
-pytest-randomly==3.15.0
+pytest-randomly==3.16.0
 pytest-playwright==0.5.2

requirements.txt

@@ -8,6 +8,6 @@ schema==0.7.7
 aiohttp==3.10.10
 
 # calibration script dep
-tqdm==4.66.5
+tqdm==4.66.6
 plotext==5.3.2
 docker==7.1.0