add task manager

nexus/settings.py

@@ -57,6 +57,7 @@
     'nexus.orgs',
     'nexus.projects',
     'nexus.intelligences',
+    "nexus.task_manager",
 ]
 
 MIDDLEWARE = [

nexus/task_manager/admin.py

@@ -0,0 +1 @@
+from django.contrib import admin

nexus/task_manager/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class TaskManagerConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'nexus.task_manager'

nexus/task_manager/file_database/file_database.py

@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass
+class FileResponseDTO:
+    status: int
+    file_url: str
+    err: str
+
+class FileDataBase(ABC):
+
+    @abstractmethod
+    def add_file(file) -> FileResponseDTO:
+        ...

nexus/task_manager/file_database/s3_file_database.py

@@ -0,0 +1,27 @@
+import boto3
+import uuid
+from django.conf import settings
+
+from .file_database import FileDataBase, FileResponseDTO
+
+
+
+class s3FileDatabase(FileDataBase):
+
+    def add_file(file) -> FileResponseDTO:
+        s3_client = boto3.client(
+            's3',
+            # aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+            # aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
+            region_name=settings.AWS_S3_REGION_NAME
+        )
+        file_name = file.name + str(uuid.uuid4())
+        response = FileResponseDTO()
+        try:
+            s3_client.upload_fileobj(file, settings.AWS_STORAGE_BUCKET_NAME, file_name)
+            response.status = 0
+            response.file_url = f"https://{settings.AWS_STORAGE_BUCKET_NAME}.s3.{settings.AWS_S3_REGION_NAME}.amazonaws.com/{file_name}"
+        except Exception as exception:
+            response.status = 1
+            response.err = str(exception)
+        return response

nexus/task_manager/file_database/sentenx_file_database.py

@@ -0,0 +1,24 @@
+import requests
+from django.conf import settings
+
+from nexus.task_manager.models import TaskManager
+
+
+class SentenXFileDataBase:
+    def __init__(self):
+        pass
+    def add_file(self, task: TaskManager):
+        url = settings.SENTENX_BASE_URL + "/content_base/index"
+        headers = {
+            "Content-Type": "application/json; charset: utf-8",
+            "Authorization": f"Bearer {settings.SENTENX_AUTH_TOKEN}",
+        }
+        body = {
+            "file": task.content_base_file.file,
+            "filename": task.content_base_file.file_name,
+            "extension_file": task.content_base_file.extension_file,
+            "task_uuid": str(task.uuid),
+            "content_base": str(task.content_base_file.content_base.uuid)
+        }
+        response = requests.post(url=url, headers=headers, body=body)
+        return response.json() if response.status_code == 200 else response.text

nexus/task_manager/file_manager/celery_file_manager.py

@@ -0,0 +1,29 @@
+from .file_manager import FileManager
+from nexus.usecases.content_base.content_base_dto import ContentBaseFileDTO
+from nexus.usecases import users
+from nexus.usecases.content_base.content_base_file import get_by_uuid as content_base_usecase
+from nexus.task_manager.models import ContentBaseFileTaskManager
+from nexus.task_manager.file_database.file_database import FileDataBase
+from nexus.task_manager.tasks import add_file
+
+class CeleryFileManager(FileManager):
+
+    def __init__(self, file_database: FileDataBase):
+        self._file_database = file_database
+
+    def upload_file(self, file: bytes, content_base_uuid: str):
+        content_base_file = content_base_usecase.get_by_uuid(content_base_uuid=content_base_uuid)
+        task_manager = ContentBaseFileTaskManager.objects.create(
+            status=ContentBaseFileTaskManager.STATUS_WAITING,
+            created_by=content_base_file.created_by,
+            content_base_file=content_base_file
+        )
+        content_base_file_dto = ContentBaseFileDTO(
+            file=file,
+            user_email=content_base_file.created_by.user_email,
+            content_base_uuid=content_base_uuid,
+            extension_file=content_base_file.extension_file,
+            file_url=""
+        )
+        add_file.apply_async(args=[task_manager, content_base_file_dto, self._file_database])
+        return task_manager.uuid

nexus/task_manager/file_manager/file_manager.py

@@ -0,0 +1,5 @@
+from abc import ABC
+
+
+class FileManager(ABC):
+    pass

nexus/task_manager/models.py

@@ -0,0 +1,35 @@
+import uuid
+
+from django.db import models
+
+from nexus.intelligences.models import ContentBaseFile
+from nexus.users.models import User
+
+
+class TaskManager(models.Model):
+    STATUS_WAITING = "waiting"
+    STATUS_LOADING = "loading"
+    STATUS_PROCESSING = "Processing"
+    STATUS_SUCCESS = "success"
+    STATUS_FAIL = "fail"
+
+    STATUS_CHOICES = [
+        (STATUS_LOADING, "Loading"),
+        (STATUS_SUCCESS, "Success"),
+        (STATUS_FAIL, "Fail"),
+        (STATUS_WAITING, "Wait")
+    ]
+
+    uuid = models.UUIDField(default=uuid.uuid4)
+    created_at = models.DateTimeField(auto_now_add=True)
+    end_at = models.DateTimeField()
+    status = models.TextField(choices=STATUS_CHOICES, default=STATUS_WAITING)
+    created_by = models.ForeignKey(User, on_delete=models.SET_NULL, related_name="files")
+
+
+class ContentBaseFileTaskManager(TaskManager):
+    content_base_file = models.ForeignKey(ContentBaseFile, on_delete=models.SET_NULL, related_name="upload_tasks", blank=True, null=True)
+
+    def update_status(self, new_status):
+        self.status = new_status
+        self.save(update_fields=["status"])

nexus/task_manager/tasks.py

@@ -0,0 +1,26 @@
+from nexus.task_manager.models import ContentBaseFileTaskManager
+from nexus.task_manager.file_database.file_database import FileDataBase
+from nexus.task_manager.file_database.sentenx_file_database import SentenXFileDataBase
+from nexus.usecases.content_base.content_base_dto import ContentBaseFileDTO
+
+def add_file(task_manager: ContentBaseFileTaskManager, content_base_file_dto: ContentBaseFileDTO, file_database: FileDataBase):
+    task_manager.update_status(ContentBaseFileTaskManager.STATUS_LOADING)
+    response = file_database.add_file(content_base_file_dto.file)
+    if response.status == 0:
+        content_base_file_dto.file_url = response.url
+        task_manager.content_base_file.file_url = content_base_file_dto.file_url
+        task_manager.content_base_file.save(update_fields=["file_url"])
+        task_manager.update_status(ContentBaseFileTaskManager.STATUS_PROCESSING)
+        print(f"[ ADD FILE TASK ] - success on add file to file database {task_manager.content_base_file.uuid}")
+    else:
+        task_manager.update_status(ContentBaseFileTaskManager.STATUS_FAIL)
+        print("[ ADD FILE TASK ] - fail on add file to file database")
+        return
+    sentenx_file_database = SentenXFileDataBase()
+    sentenx_response = sentenx_file_database.add_file(task_manager)
+    if sentenx_response.status_code == 200:
+        task_manager.update_status(ContentBaseFileTaskManager.STATUS_SUCCESS)
+        print(f"ADD FILE TASK ] - success on index new file `{task_manager.content_base_file.uuid}`")
+    else:
+        task_manager.update_status(ContentBaseFileTaskManager.STATUS_FAIL)
+        print("[ ADD FILE TASK ] - fail on index file in sentenx")

nexus/task_manager/tests.py

@@ -0,0 +1 @@
+from django.test import TestCase

nexus/task_manager/views.py

@@ -0,0 +1,2 @@
+from django.shortcuts import render
+

nexus/usecases/content_base/content_base_dto.py

@@ -0,0 +1,15 @@
+from nexus.users.models import User
+from dataclasses import dataclass
+
+@dataclass
+class ContentBaseDTO:
+    title: str
+    created_by: User
+
+@dataclass
+class ContentBaseFileDTO:
+    file: bytes
+    file_url: str
+    extension_file: str
+    user_email: str
+    content_base_uuid: str

nexus/usecases/content_base/content_base_file/create.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+
+from nexus.usecases.content_base.content_base_dto import ContentBaseFileDTO
+from nexus.usecases.content_base import get_by_uuid as content_base_usecase
+from nexus.intelligences.models import ContentBaseFile
+from nexus.usecases import orgs, users
+
+
+class CreateContentBaseFileUseCase():
+
+    def create_content_base_file(self, content_base_file: ContentBaseFileDTO) -> ContentBaseFile:
+        user = users.get_by_email(content_base_file.user_email)
+        content_base = content_base_usecase.get_by_uuid(content_base_uuid=content_base_file.content_base_uuid)
+        content_base_file = ContentBaseFile.objects.create(
+            file=content_base_file.file_url,
+            extension_file=content_base_file.extension_file,
+            content_base=content_base,
+            created_by=user
+        )
+        return content_base_file

nexus/usecases/content_base/content_base_file/get_by_uuid.py

@@ -0,0 +1,9 @@
+from nexus.intelligences.models import ContentBaseFile
+
+def get_by_uuid(content_base_uuid: str) -> ContentBaseFile:
+    try:
+        return ContentBaseFile.objects.get(uuid=content_base_uuid)
+    except ContentBaseFile.DoesNotExist:
+        raise(f"[ ContentBaseFile ] - ContentBaseFile with uuid `{content_base_uuid}` does not exists.")
+    except Exception as exception:
+        raise(f"[ ContentBaseFile ] - ContentBaseFile error to get - error: `{exception}`")

nexus/usecases/content_base/get_by_uuid.py

@@ -0,0 +1,9 @@
+from nexus.intelligences.models import ContentBase
+
+def get_by_uuid(content_base_uuid: str) -> ContentBase:
+    try:
+        return ContentBase.objects.get(uuid=content_base_uuid)
+    except ContentBase.DoesNotExist:
+        raise(f"[ ContentBase ] - ContentBase with uuid `{content_base_uuid}` does not exists.")
+    except Exception as exception:
+        raise(f"[ ContentBase ] - ContentBase error to get - error: `{exception}`")