Merge pull request #123 from amosproj/mid-project-release

Mid project release

src/backend/.env.template

@@ -13,5 +13,7 @@ PWD=''
 AIRFLOW_SERVER_URL='http://airflow-server:8080'
 AIRFLOW_USERNAME='airflow'
 AIRFLOW_PASSWORD='airflow'
+DPMS_USERNAME=''
+DPMS_PASSWORD=''
 OIDC_SECRET_KEY='your_secret_key'
 ENABLE_KEYCLOAK=False

src/backend/api/airflow_api.py

@@ -7,7 +7,7 @@
 from dotenv import load_dotenv
 
 from services.auth_service import secure
-from services.upload_to_s3 import download_file
+from services.file_storage import download_file
 
 
 load_dotenv()

src/backend/api/dp_run.py

@@ -8,8 +8,6 @@
 dp_run = Blueprint("dp_run", __name__, template_folder="templates")
 
 
-
-
 @dp_run.route("/dp_run", methods=["GET"])
 @secure
 def get_all_dp_runs():

src/backend/api/file.py

@@ -0,0 +1,125 @@
+from flask import request, Blueprint, jsonify
+
+from database.mongo_repo import fileDetailsDB
+from services.auth_service import secure
+from services.file_storage import (
+    download_file,
+    list_file,
+    get_file_upload_url,
+    delete_file,
+)
+
+from services.file_detail import insert_file_details
+from services.s3_storage import s3_delete_file, s3_get_download_url
+
+file = Blueprint("file", __name__, template_folder="templates")
+
+
+@file.route("/s3file", methods=['GET'])
+@secure
+def get_all_s3_files():
+    # List objects in the bucket
+    try:
+        objects = list_file()
+        if objects:
+            # files = [obj['Key'] for obj in objects]
+            return jsonify(objects)
+        else:
+            return "The bucket is empty."
+
+    except Exception as e:
+        return jsonify({f"Error: {e}"})
+
+@file.route("/file", methods=['GET'])
+@secure
+def get_all_files():
+    data = fileDetailsDB.find()
+
+    allData = []
+    for d in data:
+        allData.append({
+                        'uuid': d['uuid'],
+                        'name': d['name'],
+                        'mime_type': d['mime_type'],
+                        'size': d['size'],
+                        's3_uuid': d['s3_uuid'],
+                        'content_type': d['content_type'],
+                        'storage_class': d['storage_class'],
+                        'last_modified': d['last_modified'],
+                        'created_at': d['created_at']})
+
+
+    return jsonify(allData), 201
+
+@file.route('/file/new', methods=['POST'])
+@secure
+def create_file():
+    data = request.json
+    # todo add s3_uuid check
+    if 'fileName' not in data:
+        return jsonify({'error': 'Missing fileName in request'}), 400
+    file_name = data['fileName']
+    s3_uuid = data['s3_uuid']
+    mime_type = data['mime_type']
+
+    insert_file_details(file_name, s3_uuid, mime_type)
+
+    if file_name:
+        return jsonify({'message': 'Saved successfully'})
+
+
+@file.route("/file/<id>", methods=['GET'])
+@secure
+def get_file(id):
+    try:
+        # TODO
+        return jsonify({"message": "test message"})
+
+        # Send the file for download
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@file.route("/file/<id>", methods=["DELETE"])
+@secure
+def delete_file(id):
+    try:
+
+        file_details = fileDetailsDB.find_one({"uuid": id})
+        s3_uuid = file_details['s3_uuid']
+
+        # delete s3 bucket file
+        s3_delete_file(s3_uuid)
+
+        # delete file detail
+        fileDetailsDB.delete_one({"uuid": id})
+
+        return jsonify('Sucessfull deleted')
+    except Exception as e:
+        return jsonify(f"Error: {e}")
+
+
+@file.route('/file/upload', methods=['POST'])
+@secure
+def upload_file_with_url():
+    data = request.json
+    if 'fileName' not in data:
+        return jsonify({'error': 'Missing fileName in request'}), 400
+
+    return jsonify(get_file_upload_url(data['fileName']))
+
+
+@file.route("/file/<id>/download", methods=['GET'])
+@secure
+def download_file(id):
+    try:
+        # Download the object from S3
+        file_details = fileDetailsDB.find_one({"uuid": id})
+        s3_uuid = file_details['s3_uuid']
+
+        download_url = s3_get_download_url(s3_uuid)
+        return jsonify({"download_url": download_url})
+
+        # Send the file for download
+    except Exception as e:
+        return f"Error: {e}"

src/backend/api/metadata.py

@@ -3,9 +3,9 @@
 from database.mongo_repo import metadataDB
 from database.models.metadata_details import MetadataDetails
 from services.auth_service import secure
-from services.store_s3metadata import (
+from services.file_detail import (
     insert_all_s3files_metadata,
-    insert_one_s3file_metadata,
+    insert_file_details,
     remove_s3metadata,
 )
 
@@ -101,17 +101,17 @@ def store_all_s3files_metadata():
     )
 
 
-@metadata.route("/metadata/store_single_s3metadata", methods=["POST"])
-@secure
-def store_single_s3metadata():
-    data = request.json
-    response = insert_one_s3file_metadata(metadataDB, data["file_name"])
-    if response != None:
-        return jsonify(
-            {"message": "The metadatas of uploaded file is stored successfully!"}
-        )
-    else:
-        return jsonify({"message": "There is no such a file in the S3 bucket!"})
+# @metadata.route("/metadata/store_single_s3metadata", methods=["POST"])
+# @secure
+# def store_single_s3metadata():
+#     data = request.json
+#     response = insert_file_details(metadataDB, data["file_name"])
+#     if response != None:
+#         return jsonify(
+#             {"message": "The metadatas of uploaded file is stored successfully!"}
+#         )
+#     else:
+#         return jsonify({"message": "There is no such a file in the S3 bucket!"})
 
 
 @metadata.route("/metadata/delete_all_metadata", methods=["DELETE"])

src/backend/api/misc.py

@@ -0,0 +1,7 @@
+from flask import Blueprint, jsonify
+
+misc = Blueprint("misc", __name__, template_folder="templates")
+
+@misc.route("/ping", methods=["POST"])
+def ping():
+    return jsonify({"message": "Ping successfully"})

src/backend/app.py

@@ -9,7 +9,8 @@
 from api.datapipeline import datapipeline
 from api.dp_run import dp_run
 from api.fileWP import fileWP
-from api.upload_api import upload_api
+from api.misc import misc
+from api.file import file
 from services.auth_service import secure
 from flask_cors import CORS
 
@@ -34,11 +35,12 @@
 oidc = OpenIDConnect(app)
 
 
-app.register_blueprint(upload_api)
+app.register_blueprint(file)
 app.register_blueprint(datapipeline, url_prefix="/")
 app.register_blueprint(fileWP)
 app.register_blueprint(airflow_api)
 app.register_blueprint(dp_run)
+app.register_blueprint(misc)
 
 
 @app.route("/")