BITMAKER-5846: Show storage size in JobDetailPage (#245)

* Show storage size in JobDetailPage * style(docker): Fix warnings of unmatched case in FROM - AS --------- Co-authored-by: mgonnav <[email protected]>
bitmakerla · Oct 15, 2024 · 44ad862 · 44ad862
1 parent 3d39508
commit 44ad862
Show file tree

Hide file tree

Showing 9 changed files with 69 additions and 30 deletions.
diff --git a/estela-api/api/serializers/job.py b/estela-api/api/serializers/job.py
@@ -2,13 +2,18 @@
 from rest_framework import serializers
 
 from api import errors
+from api.exceptions import DataBaseError
 from api.serializers.job_specific import (
     SpiderJobArgSerializer,
     SpiderJobEnvVarSerializer,
     SpiderJobTagSerializer,
 )
-from api.utils import delete_stats_from_redis, update_stats_from_redis
-from config.job_manager import job_manager
+from api.utils import (
+    delete_stats_from_redis,
+    update_stats_from_redis,
+    get_collection_name,
+)
+from config.job_manager import job_manager, spiderdata_db_client
 from core.models import (
     DataStatus,
     SpiderJob,
@@ -31,6 +36,7 @@ class SpiderJobSerializer(serializers.ModelSerializer):
         required=False, read_only=True, help_text="Current job status."
     )
     spider = serializers.SerializerMethodField("get_spider")
+    storage_size = serializers.SerializerMethodField("get_storage_size")
 
     class Meta:
         model = SpiderJob
@@ -50,11 +56,28 @@ class Meta:
             "cronjob",
             "data_expiry_days",
             "data_status",
+            "storage_size",
         )
 
     def get_spider(self, instance):
         return {"sid": instance.spider.sid, "name": instance.spider.name}
 
+    def get_storage_size(self, instance):
+        if not spiderdata_db_client.get_connection():
+            raise DataBaseError({"error": errors.UNABLE_CONNECT_DB})
+
+        pid = str(instance.spider.project.pid)
+        collections = ["items", "requests", "logs"]
+        total_size = sum(
+            [
+                spiderdata_db_client.get_dataset_size(
+                    pid, get_collection_name(instance, collection)
+                )
+                for collection in collections
+            ]
+        )
+        return total_size
+
 
 class SpiderJobCreateEnvVarSerializer(serializers.Serializer):
     evid = serializers.IntegerField(required=False, help_text="Env var id.")

diff --git a/estela-api/api/utils.py b/estela-api/api/utils.py
@@ -108,3 +108,20 @@ def get_proxy_provider_envs(proxy_provider):
         }
     )
     return env_vars
+
+
+def get_collection_name(job, data_type):
+    if (
+        job.cronjob is not None
+        and job.cronjob.unique_collection
+        and data_type == "items"
+    ):
+        job_collection_name = "{}-scj{}-job_{}".format(
+            job.spider.sid, job.cronjob.cjid, data_type
+        )
+    else:
+        job_collection_name = "{}-{}-job_{}".format(
+            job.spider.sid, job.jid, data_type
+        )
+
+    return job_collection_name
diff --git a/estela-api/api/views/job_data.py b/estela-api/api/views/job_data.py
@@ -12,6 +12,7 @@
 from api import errors
 from api.exceptions import DataBaseError
 from api.mixins import BaseViewSet
+from api.utils import get_collection_name
 from config.job_manager import spiderdata_db_client
 from core.models import SpiderJob
 from core.tasks import get_chain_to_process_usage_data
@@ -98,7 +99,7 @@ def list(self, request, *args, **kwargs):
             raise DataBaseError({"error": errors.UNABLE_CONNECT_DB})
 
         job = SpiderJob.objects.filter(jid=kwargs["jid"]).get()
-        job_collection_name = self.get_collection_name(job, data_type)
+        job_collection_name = get_collection_name(job, data_type)
 
         count = spiderdata_db_client.get_estimated_item_count(
             kwargs["pid"], job_collection_name
@@ -147,21 +148,6 @@ def list(self, request, *args, **kwargs):
             }
         )
 
-    def get_collection_name(self, job, data_type):
-        if (
-            job.cronjob is not None
-            and job.cronjob.unique_collection
-            and data_type == "items"
-        ):
-            job_collection_name = "{}-scj{}-job_{}".format(
-                job.spider.sid, job.cronjob.cjid, data_type
-            )
-        else:
-            job_collection_name = "{}-{}-job_{}".format(
-                job.spider.sid, job.jid, data_type
-            )
-
-        return job_collection_name
 
     @swagger_auto_schema(
         methods=["GET"],
@@ -196,7 +182,7 @@ def download(self, request, *args, **kwargs):
         data_type = request.query_params.get("type", "items")
 
         job = SpiderJob.objects.filter(jid=kwargs["jid"]).get()
-        job_collection_name = self.get_collection_name(job, data_type)
+        job_collection_name = get_collection_name(job, data_type)
 
         data = []
         if data_type == "stats":
@@ -242,7 +228,7 @@ def delete(self, request, *args, **kwargs):
         if not spiderdata_db_client.get_connection():
             raise DataBaseError({"error": errors.UNABLE_CONNECT_DB})
 
-        job_collection_name = self.get_collection_name(job, data_type)
+        job_collection_name = get_collection_name(job, data_type)
         deleted_data = spiderdata_db_client.delete_dataset_data(
             kwargs["pid"], job_collection_name
         )

diff --git a/estela-api/docker-conf/Dockerfile-celery-beat b/estela-api/docker-conf/Dockerfile-celery-beat
@@ -1,5 +1,5 @@
 # Build Stage
-FROM python:3.9-slim as builder
+FROM python:3.9-slim AS builder
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1

diff --git a/estela-api/docker-conf/Dockerfile-celery-worker b/estela-api/docker-conf/Dockerfile-celery-worker
@@ -1,5 +1,5 @@
 # Build Stage
-FROM python:3.9-slim as builder
+FROM python:3.9-slim AS builder
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1

diff --git a/estela-api/docker-conf/Dockerfile-django-api b/estela-api/docker-conf/Dockerfile-django-api
@@ -1,5 +1,5 @@
 # Build Stage
-FROM python:3.9-slim as builder
+FROM python:3.9-slim AS builder
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1

diff --git a/estela-api/docs/api.yaml b/estela-api/docs/api.yaml
@@ -2386,6 +2386,10 @@ definitions:
           - PERSISTENT
           - PENDING
           - DELETED
+      storage_size:
+        title: Storage size
+        type: string
+        readOnly: true
   ProjectJob:
     required:
       - results

diff --git a/estela-web/src/pages/JobDetailPage/index.tsx b/estela-web/src/pages/JobDetailPage/index.tsx
@@ -306,6 +306,7 @@ export class JobDetailPage extends Component<RouteComponentProps<RouteParams>, J
                     loaded: true,
                     dataStatus: response.dataStatus,
                     dataExpiryDays: response.dataExpiryDays == null ? 1 : response.dataExpiryDays,
+                    storageSize: response.storageSize,
                 });
             },
             (error: unknown) => {
@@ -660,6 +661,7 @@ export class JobDetailPage extends Component<RouteComponentProps<RouteParams>, J
             totalResponseBytes,
             items,
             status,
+            storageSize,
         } = this.state;
         const getProxyTag = (): string => {
             const desiredItem = envVars.find((item) => item.name === "ESTELA_PROXY_NAME");
@@ -933,15 +935,13 @@ export class JobDetailPage extends Component<RouteComponentProps<RouteParams>, J
                     </Card>
                 </Content>
                 <Content className="my-2 grid lg:grid-cols-12 grid-cols-12 gap-1 items-start lg:w-full">
-                    <Card
-                        className="opacity-40 cursor-not-allowed w-full col-span-2 flex flex-col"
-                        style={{ borderRadius: "8px" }}
-                        bordered={false}
-                    >
+                    <Card className="w-full col-span-2 flex flex-col" style={{ borderRadius: "8px" }} bordered={false}>
                         <Text className="py-0 text-estela-black-medium font-medium text-base">Storage</Text>
                         <Row className="grid grid-cols-1 py-1 mt-3">
                             <Col>
-                                <Text className="font-bold text-estela-black-full text-lg">-/-</Text>
+                                <Text className="font-bold text-estela-black-full text-lg">
+                                    {`${formatBytes(storageSize).quantity} ${formatBytes(storageSize).type}`}
+                                </Text>
                             </Col>
                             <Col>
                                 <Text className="text-estela-black-medium text-xs">of project</Text>
@@ -950,7 +950,9 @@ export class JobDetailPage extends Component<RouteComponentProps<RouteParams>, J
                                 <Content className="w-full bg-estela-white-low rounded-full h-2.5 dark:bg-estela-white-low">
                                     <div
                                         className="bg-estela-states-green-medium h-2.5 rounded-full"
-                                        style={{ width: "0%" }}
+                                        style={{
+                                            width: "100%",
+                                        }}
                                     ></div>
                                 </Content>
                             </Col>

diff --git a/estela-web/src/services/api/generated-api/models/SpiderJob.ts b/estela-web/src/services/api/generated-api/models/SpiderJob.ts
@@ -124,6 +124,12 @@ export interface SpiderJob {
      * @memberof SpiderJob
      */
     dataStatus?: SpiderJobDataStatusEnum;
+    /**
+     * 
+     * @type {string}
+     * @memberof SpiderJob
+     */
+    readonly storageSize?: string;
 }
 
 /**
@@ -161,6 +167,7 @@ export function SpiderJobFromJSONTyped(json: any, ignoreDiscriminator: boolean):
         'cronjob': !exists(json, 'cronjob') ? undefined : json['cronjob'],
         'dataExpiryDays': !exists(json, 'data_expiry_days') ? undefined : json['data_expiry_days'],
         'dataStatus': !exists(json, 'data_status') ? undefined : json['data_status'],
+        'storageSize': !exists(json, 'storage_size') ? undefined : json['storage_size'],
     };
 }