Merge pull request #94 from banodoco/main

Push

.env.sample

@@ -0,0 +1,6 @@
+SERVER=development
+SERVER_URL=http://127.0.0.1:8000
+OFFLINE_MODE=True
+GPU_INFERENCE_ENABLED=True
+HOSTED_BACKGROUND_RUNNER_MODE=False
+REPLICATE_KEY=xyz

.github/workflows/deploy.yml

@@ -3,12 +3,11 @@ name: Deploy to ECR
 on:
 
   push:
-    branches: [ main, piyush-dev ]
+    branches: [ piyush-dev ]
 
 jobs:
 
   build:
-
     name: Build Image
     runs-on: ubuntu-latest
 
@@ -59,4 +58,39 @@ jobs:
         task-definition: ${{ steps.task-def.outputs.task-definition }}
         service: ${{ env.ECS_SERVICE }}
         cluster: ${{ env.ECS_CLUSTER }}
-        wait-for-service-stability: true
+        wait-for-service-stability: true
+
+  update-runner:
+    name: Update Background Runner
+    runs-on: self-hosted
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Create and activate virtual environment
+        run: |
+          python3 -m venv venv
+          source venv/bin/activate
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          source venv/bin/activate && pip install -r requirements.txt
+
+      - name: Create .env file
+        run: |
+          touch .env
+          echo "SERVER=production" > .env
+          echo "SERVER_URL=https://api.banodoco.ai" >> .env
+          echo "HOSTED_BACKGROUND_RUNNER_MODE=True" >> .env
+          echo "admin_email=${{ secrets.ADMIN_EMAIL }}" >> .env
+          echo "admin_password=${{ secrets.ADMIN_PASSWORD }}" >> .env
+          echo "ENCRYPTION_KEY=${{ secrets.FERNET_ENCRYPTION_KEY }}" >> .env
+
+      - name: Restart runner
+        run: |
+          if pkill -0 -f "banodoco_runner"; then
+            pkill -f "banodoco_runner"
+          fi
+          . venv/bin/activate && nohup python banodoco_runner.py > script_output.log 2>&1 &

.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *$py.class
+comfyui*.log
 
 venv
 .vscode
@@ -13,6 +14,7 @@ venv
 app_settings.csv
 videos
 videos/*
+temp_dir/
 /temp/
 training_data
 inference_log/*
@@ -21,6 +23,10 @@ test.py
 doc.py
 .env
 data.json
+comfy_runner/
+ComfyUI/
+output/
+images.zip
 
 # generated file TODO: move inside videos
 depth.png

.streamlit/config.toml

@@ -0,0 +1,5 @@
+[theme]
+base="dark"
+primaryColor="#9b9bb1"
+backgroundColor="#2a3446"
+secondaryBackgroundColor="#26282f"

LICENSE.txt

@@ -0,0 +1,43 @@
+Open Source Native License (OSNL)
+Version 0.1 - March 1, 2024
+
+Preamble
+
+The Open Source Native License (OSNL) is designed to ensure that software remains free and open, fostering innovation and knowledge sharing within the community. It grants individuals, researchers, and commercial entities who open source their primary business assets, the freedom to use the software in any manner they choose.
+
+This distinctive approach aims to balance the benefits of open-source development with the realities of commercial enterprise. It ensures that software remains a shared, community-driven resource while enabling businesses to thrive in an open-source ecosystem. Additional licenses are available for non-open source commercial entities.
+
+1. Definitions
+
+- "This License" refers to Version 1.0 of the Open Source Native License.
+- "The Program" refers to the software distributed under this License.
+- "You" refers to the individual or entity utilizing or contributing to the Program.
+- "Primary Business Assets" are the core resources, capabilities, and technology that constitute the main value proposition and operational basis of your business.
+
+2. Grant of License
+
+Subject to the terms and conditions of this License, you are hereby granted a free, perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable license to use, reproduce, modify, distribute, and sublicense the Program, provided you comply with the following condition:
+
+- Individual or researcher: You are granted the rights to use, modify, distribute, and contribute to the Program for any purpose, including educational, research, and personal projects, without the necessity to make your personal projects open source, provided these activities do not constitute a commercial enterprise. For any use that transitions to commercial purposes, the conditions applicable to commercial entities as outlined in this License will then apply.
+- Commercial Entity who meets open source condition: Your primary business assets, including all core technologies, software, and platforms, must be available under an OSI-approved open source license or OSNL. This condition does not apply to ancillary or peripheral services not constituting primary business assets.
+
+2.1 Commercial Use by Non-Open Source Businesses
+
+Non-open source businesses that wish to utilize the Program or its derivatives as a component of their products or services are required to obtain an additional license. These entities must proactively contact Banodoco to request such a license. Banodoco reserves the right, at its own discretion, to grant or deny this additional license. Until an additional license is granted by Banodoco, non-open source businesses are not authorized to exercise any rights provided under this License regarding the use of the Program or its derivatives.
+
+3. Redistribution
+
+You may reproduce and distribute copies of the Program or derivative works thereof in any medium, with or without modifications, provided that you meet the following conditions:
+
+- You must give any recipients of the Program a copy of this License.
+- You must ensure that any modified files carry prominent notices stating that you changed the files.
+- You must disclose the source of the Program, and if you distribute any portion of it in a compiled or object code form, you must also provide the full source code under this License.
+- Any distribution of the Program or derivative works must comply with the Primary Business Open Source Condition.
+
+4. Disclaimer of Warranty
+
+THE PROGRAM IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY ARISING FROM THE USE OF THE PROGRAM.
+
+5. General
+
+This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Program.

app.py

@@ -1,4 +1,3 @@
-import threading
 import time
 import streamlit as st
 from moviepy.editor import *
@@ -10,7 +9,7 @@
 from shared.logging.logging import AppLogger
 from utils.common_utils import is_process_active
 
-from utils.constants import AUTH_TOKEN, RUNNER_PROCESS_NAME
+from utils.constants import AUTH_TOKEN, RUNNER_PROCESS_NAME, RUNNER_PROCESS_PORT
 from utils.local_storage.url_storage import delete_url_param, get_url_param, set_url_param
 from utils.third_party_auth.google.google_auth import get_google_auth_url
 from streamlit_server_state import server_state_lock
@@ -41,20 +40,21 @@
 )
 
 def start_runner():
-    if SERVER != ServerType.DEVELOPMENT.value and not HOSTED_BACKGROUND_RUNNER_MODE:
+    if SERVER != ServerType.DEVELOPMENT.value and HOSTED_BACKGROUND_RUNNER_MODE in [False, 'False']:
         return
 
     with server_state_lock["runner"]:
         app_logger = AppLogger()
 
-        if not is_process_active(RUNNER_PROCESS_NAME):
+        if not is_process_active(RUNNER_PROCESS_NAME, RUNNER_PROCESS_PORT):
             app_logger.info("Starting runner")
-            # _ = subprocess.Popen(["streamlit", "run", "banodoco_runner.py", "--runner.fastReruns", "false", "--server.port", "5502", "--server.headless", "true"])
-            _ = subprocess.Popen(["python", "banodoco_runner.py"])
-            while not is_process_active(RUNNER_PROCESS_NAME):
+            python_executable = sys.executable
+            _ = subprocess.Popen([python_executable, "banodoco_runner.py"])
+            while not is_process_active(RUNNER_PROCESS_NAME, RUNNER_PROCESS_PORT):
                 time.sleep(0.1)
         else:
-            app_logger.debug("Runner already running")
+            # app_logger.debug("Runner already running")
+            pass
 
 def main():
     st.set_page_config(page_title="Banodoco", page_icon="🎨", layout="wide")
@@ -65,7 +65,7 @@ def main():
         params = st.experimental_get_query_params()
 
         if params and 'code' in params:
-            st.subheader("Logging you in, please wait")
+            st.markdown("#### Logging you in, please wait...")
             # st.write(params['code'])
             data = {
                 "id_token": params['code'][0]
@@ -82,17 +82,27 @@ def main():
                 discord_url = "<a target='_self' href='https://discord.gg/zGgpH9JEw4'> Banodoco Discord </a>"
                 st.markdown(discord_url, unsafe_allow_html=True)
         else:
-            st.markdown("# :red[ba]:green[no]:orange[do]:blue[co]")
-            st.subheader("Login with Google to proceed")
+            st.markdown("# :green[D]:red[o]:blue[u]:orange[g]:green[h] :red[□] :blue[□] :orange[□]")            
+            st.markdown("#### Login with Google to proceed")
 
             auth_url = get_google_auth_url()
             st.markdown(auth_url, unsafe_allow_html=True)
+
     else:
         start_runner()
         project_init()
 
         from ui_components.setup import setup_app_ui
-        setup_app_ui()
+        from ui_components.components.welcome_page import welcome_page
+
+        data_repo = DataRepo()
+        app_setting = data_repo.get_app_setting_from_uuid()
+        if app_setting.welcome_state != 0:
+            setup_app_ui()
+        else:
+            welcome_page()
+
+        st.session_state['maintain_state'] = False
 
 if __name__ == '__main__':
     try:

backend/db_repo.py

@@ -15,7 +15,7 @@
 import subprocess
 from typing import List
 import uuid
-from shared.constants import InternalFileType, SortOrder
+from shared.constants import InferenceStatus, InternalFileTag, InternalFileType, SortOrder
 from backend.serializers.dto import  AIModelDto, AppSettingDto, BackupDto, BackupListDto, InferenceLogDto, InternalFileDto, ProjectDto, SettingDto, ShotDto, TimingDto, UserDto
 
 from shared.constants import AUTOMATIC_FILE_HOSTING, LOCAL_DATABASE_NAME, SERVER, ServerType
@@ -55,7 +55,8 @@ def __init__(self):
                 conn = sqlite3.connect(database_file)
                 conn.close()
 
-                completed_process = subprocess.run(['python', 'manage.py', 'migrate'], capture_output=True, text=True)
+                python_executable = sys.executable
+                completed_process = subprocess.run([python_executable, 'manage.py', 'migrate'], capture_output=True, text=True)
                 if completed_process.returncode == 0:
                     logger.log(LoggingType.INFO, "Migrations completed successfully")
                 else:
@@ -200,8 +201,17 @@ def get_all_file_list(self, **kwargs):
             del kwargs['data_per_page']
             sort_order = kwargs['sort_order'] if 'sort_order' in kwargs else None
             del kwargs['sort_order']
+
+            shot_uuid_list = []
+            if 'shot_uuid_list' in kwargs:
+                shot_uuid_list = kwargs['shot_uuid_list']
+                del kwargs['shot_uuid_list']
 
             file_list = InternalFileObject.objects.filter(**kwargs).all()
+
+            if shot_uuid_list and len(shot_uuid_list):
+                file_list = file_list.filter(shot_uuid__in=shot_uuid_list)
+
             if sort_order:
                 if sort_order == SortOrder.DESCENDING.value:
                     file_list = file_list.order_by('-created_on')
@@ -371,6 +381,38 @@ def update_file(self, **kwargs):
 
         return InternalResponse(payload, 'file updated successfully', True)
 
+    def get_file_count_from_type(self, file_tag, project_uuid):
+        project = Project.objects.filter(uuid=project_uuid, is_disabled=False).first()
+        file_count = InternalFileObject.objects.filter(tag=file_tag, project_id=project.id, is_disabled=False).count()
+        payload = {
+            'data': file_count
+        }
+
+        return InternalResponse(payload, 'file count fetched', True)
+
+    def get_explorer_pending_stats(self, project_uuid, log_status_list):
+        project = Project.objects.filter(uuid=project_uuid, is_disabled=False).first()
+        temp_image_count = InternalFileObject.objects.filter(tag=InternalFileTag.TEMP_GALLERY_IMAGE.value,\
+                                                              project_id=project.id, is_disabled=False).count()
+        pending_image_count = InferenceLog.objects.filter(status__in=log_status_list, is_disabled=False).count()
+        payload = {
+            'data': {
+                'temp_image_count': temp_image_count,
+                'pending_image_count': pending_image_count
+            }
+        }
+
+        return InternalResponse(payload, 'file count fetched', True)
+
+    def update_temp_gallery_images(self, project_uuid):
+        project = Project.objects.filter(uuid=project_uuid, is_disabled=False).first()
+        InternalFileObject.objects.filter(
+            tag=InternalFileTag.TEMP_GALLERY_IMAGE.value, 
+            project_id=project.id,
+            is_disabled=False).update(tag=InternalFileTag.GALLERY_IMAGE.value)
+
+        return True
+
     # project
     def get_project_from_uuid(self, uuid):
         project = Project.objects.filter(uuid=uuid, is_disabled=False).first()
@@ -568,7 +610,7 @@ def get_inference_log_from_uuid(self, uuid):
 
         return InternalResponse(payload, 'inference log fetched', True)
 
-    def get_all_inference_log_list(self, project_id=None, page=1, data_per_page=5, status_list=None, exclude_model_list=None):
+    def get_all_inference_log_list(self, project_id=None, page=1, data_per_page=5, status_list=None, exclude_model_list=None, model_name_list=""):
         if project_id:
             project = Project.objects.filter(uuid=project_id, is_disabled=False).first()
             log_list = InferenceLog.objects.filter(project_id=project.id, is_disabled=False).order_by('-created_on').all()
@@ -580,6 +622,9 @@ def get_all_inference_log_list(self, project_id=None, page=1, data_per_page=5, s
         else:
             log_list = log_list.exclude(status__in=["", None])
 
+        if model_name_list:
+            log_list = log_list.filter(model_name__in=model_name_list)
+
         log_list = log_list.exclude(model_id=None)       # hackish sol to exclude non-image/video logs
 
         paginator = Paginator(log_list, data_per_page)
@@ -603,8 +648,6 @@ def create_inference_log(self, **kwargs):
         if not attributes.is_valid():
             return InternalResponse({}, attributes.errors, False)
 
-        print(attributes.data)
-
         if 'project_id' in attributes.data and attributes.data['project_id']:
             project = Project.objects.filter(uuid=attributes.data['project_id'], is_disabled=False).first()
             if not project:
@@ -786,6 +829,7 @@ def get_timing_list_from_project(self, project_uuid=None):
         return InternalResponse(payload, 'timing list fetched', True)
 
     def get_timing_list_from_shot(self, shot_uuid):
+
         shot: Shot = Shot.objects.filter(uuid=shot_uuid, is_disabled=False).first()
         if not shot:
             return InternalResponse({}, 'invalid shot', False)
@@ -847,13 +891,13 @@ def create_timing(self, **kwargs):
 
                 attributes._data['canny_image_id'] = canny_image.id
 
-        if 'primay_image_id' in attributes.data:
-            if attributes.data['primay_image_id'] != None:
-                primay_image: InternalFileObject = InternalFileObject.objects.filter(uuid=attributes.data['primay_image_id'], is_disabled=False).first()
+        if 'primary_image_id' in attributes.data:
+            if attributes.data['primary_image_id'] != None:
+                primay_image: InternalFileObject = InternalFileObject.objects.filter(uuid=attributes.data['primary_image_id'], is_disabled=False).first()
                 if not primay_image:
                     return InternalResponse({}, 'invalid primary image uuid', False)
 
-                attributes._data['primay_image_id'] = primay_image.id
+                attributes._data['primary_image_id'] = primay_image.id
 
         timing = Timing.objects.create(**attributes.data)
         payload = {
@@ -1007,8 +1051,6 @@ def update_app_setting(self, **kwargs):
         if not attributes.is_valid():
             return InternalResponse({}, attributes.errors, False)
 
-        print(attributes.data)
-
         if 'uuid' in attributes.data and attributes.data['uuid']:
             app_setting = AppSetting.objects.filter(uuid=attributes.data['uuid'], is_disabled=False).first()
         else:
@@ -1029,7 +1071,7 @@ def update_app_setting(self, **kwargs):
         return InternalResponse({}, 'app_setting updated successfully', True)
 
 
-    def get_app_secrets_from_user_uuid(self, user_uuid):
+    def get_app_secrets_from_user_uuid(self, user_uuid, secret_access=None):
         if user_uuid:
             user: User = User.objects.filter(uuid=user_uuid, is_disabled=False).first()
             if not user:
@@ -1412,7 +1454,11 @@ def release_lock(self, key):
 
     # shot
     def get_shot_from_number(self, project_uuid, shot_number=0):
-        shot: Shot = Shot.objects.filter(project_id=project_uuid, shot_idx=shot_number, is_disabled=False).first()
+        project = Project.objects.filter(uuid=project_uuid, is_disabled=False).first()
+        if not project:
+            return InternalResponse({}, 'invalid project uuid', False)
+
+        shot: Shot = Shot.objects.filter(project_id=project.id, shot_idx=shot_number, is_disabled=False).first()
         if not shot:
             return InternalResponse({}, 'invalid shot number', False)