diff --git a/src/backend/.env.template b/src/backend/.env.template index d395c2d..cf488cf 100644 --- a/src/backend/.env.template +++ b/src/backend/.env.template @@ -13,5 +13,7 @@ PWD='' AIRFLOW_SERVER_URL='http://airflow-server:8080' AIRFLOW_USERNAME='airflow' AIRFLOW_PASSWORD='airflow' +DPMS_USERNAME='' +DPMS_PASSWORD='' OIDC_SECRET_KEY='your_secret_key' ENABLE_KEYCLOAK=False \ No newline at end of file diff --git a/src/backend/api/airflow_api.py b/src/backend/api/airflow_api.py index b8a4846..d9ead06 100644 --- a/src/backend/api/airflow_api.py +++ b/src/backend/api/airflow_api.py @@ -7,7 +7,7 @@ from dotenv import load_dotenv from services.auth_service import secure -from services.upload_to_s3 import download_file +from services.file_storage import download_file load_dotenv() diff --git a/src/backend/api/dp_run.py b/src/backend/api/dp_run.py index 932ac96..892c84d 100644 --- a/src/backend/api/dp_run.py +++ b/src/backend/api/dp_run.py @@ -8,8 +8,6 @@ dp_run = Blueprint("dp_run", __name__, template_folder="templates") - - @dp_run.route("/dp_run", methods=["GET"]) @secure def get_all_dp_runs(): diff --git a/src/backend/api/file.py b/src/backend/api/file.py new file mode 100644 index 0000000..b3580fd --- /dev/null +++ b/src/backend/api/file.py @@ -0,0 +1,125 @@ +from flask import request, Blueprint, jsonify + +from database.mongo_repo import fileDetailsDB +from services.auth_service import secure +from services.file_storage import ( + download_file, + list_file, + get_file_upload_url, + delete_file, +) + +from services.file_detail import insert_file_details +from services.s3_storage import s3_delete_file, s3_get_download_url + +file = Blueprint("file", __name__, template_folder="templates") + + +@file.route("/s3file", methods=['GET']) +@secure +def get_all_s3_files(): + # List objects in the bucket + try: + objects = list_file() + if objects: + # files = [obj['Key'] for obj in objects] + return jsonify(objects) + else: + return "The bucket is empty." + + except Exception as e: + return jsonify({f"Error: {e}"}) + +@file.route("/file", methods=['GET']) +@secure +def get_all_files(): + data = fileDetailsDB.find() + + allData = [] + for d in data: + allData.append({ + 'uuid': d['uuid'], + 'name': d['name'], + 'mime_type': d['mime_type'], + 'size': d['size'], + 's3_uuid': d['s3_uuid'], + 'content_type': d['content_type'], + 'storage_class': d['storage_class'], + 'last_modified': d['last_modified'], + 'created_at': d['created_at']}) + + + return jsonify(allData), 201 + +@file.route('/file/new', methods=['POST']) +@secure +def create_file(): + data = request.json + # todo add s3_uuid check + if 'fileName' not in data: + return jsonify({'error': 'Missing fileName in request'}), 400 + file_name = data['fileName'] + s3_uuid = data['s3_uuid'] + mime_type = data['mime_type'] + + insert_file_details(file_name, s3_uuid, mime_type) + + if file_name: + return jsonify({'message': 'Saved successfully'}) + + +@file.route("/file/", methods=['GET']) +@secure +def get_file(id): + try: + # TODO + return jsonify({"message": "test message"}) + + # Send the file for download + except Exception as e: + return f"Error: {e}" + + +@file.route("/file/", methods=["DELETE"]) +@secure +def delete_file(id): + try: + + file_details = fileDetailsDB.find_one({"uuid": id}) + s3_uuid = file_details['s3_uuid'] + + # delete s3 bucket file + s3_delete_file(s3_uuid) + + # delete file detail + fileDetailsDB.delete_one({"uuid": id}) + + return jsonify('Sucessfull deleted') + except Exception as e: + return jsonify(f"Error: {e}") + + +@file.route('/file/upload', methods=['POST']) +@secure +def upload_file_with_url(): + data = request.json + if 'fileName' not in data: + return jsonify({'error': 'Missing fileName in request'}), 400 + + return jsonify(get_file_upload_url(data['fileName'])) + + +@file.route("/file//download", methods=['GET']) +@secure +def download_file(id): + try: + # Download the object from S3 + file_details = fileDetailsDB.find_one({"uuid": id}) + s3_uuid = file_details['s3_uuid'] + + download_url = s3_get_download_url(s3_uuid) + return jsonify({"download_url": download_url}) + + # Send the file for download + except Exception as e: + return f"Error: {e}" diff --git a/src/backend/api/metadata.py b/src/backend/api/metadata.py index bc12271..997ed98 100644 --- a/src/backend/api/metadata.py +++ b/src/backend/api/metadata.py @@ -3,9 +3,9 @@ from database.mongo_repo import metadataDB from database.models.metadata_details import MetadataDetails from services.auth_service import secure -from services.store_s3metadata import ( +from services.file_detail import ( insert_all_s3files_metadata, - insert_one_s3file_metadata, + insert_file_details, remove_s3metadata, ) @@ -101,17 +101,17 @@ def store_all_s3files_metadata(): ) -@metadata.route("/metadata/store_single_s3metadata", methods=["POST"]) -@secure -def store_single_s3metadata(): - data = request.json - response = insert_one_s3file_metadata(metadataDB, data["file_name"]) - if response != None: - return jsonify( - {"message": "The metadatas of uploaded file is stored successfully!"} - ) - else: - return jsonify({"message": "There is no such a file in the S3 bucket!"}) +# @metadata.route("/metadata/store_single_s3metadata", methods=["POST"]) +# @secure +# def store_single_s3metadata(): +# data = request.json +# response = insert_file_details(metadataDB, data["file_name"]) +# if response != None: +# return jsonify( +# {"message": "The metadatas of uploaded file is stored successfully!"} +# ) +# else: +# return jsonify({"message": "There is no such a file in the S3 bucket!"}) @metadata.route("/metadata/delete_all_metadata", methods=["DELETE"]) diff --git a/src/backend/api/misc.py b/src/backend/api/misc.py new file mode 100644 index 0000000..2855b30 --- /dev/null +++ b/src/backend/api/misc.py @@ -0,0 +1,7 @@ +from flask import Blueprint, jsonify + +misc = Blueprint("misc", __name__, template_folder="templates") + +@misc.route("/ping", methods=["POST"]) +def ping(): + return jsonify({"message": "Ping successfully"}) \ No newline at end of file diff --git a/src/backend/api/upload_api.py b/src/backend/api/upload_api.py deleted file mode 100644 index 8d1e513..0000000 --- a/src/backend/api/upload_api.py +++ /dev/null @@ -1,138 +0,0 @@ -from flask import request, render_template, send_file, Blueprint, jsonify -from werkzeug.utils import secure_filename - -from services.auth_service import secure -from services.upload_to_s3 import ( - upload_to_s3, - download_file, - list_file, - file_name_check, - get_upload_url, - delete_s3file, -) - -from services.store_s3metadata import insert_one_s3file_metadata - -upload_api = Blueprint("upload_api", __name__, template_folder="templates") -ALLOWED_EXTENSIONS = {"csv"} - - -@upload_api.route('/upload_url', methods=['GET']) -@secure -def upload_url(): - file_name = request.args.get('fileName') - if file_name: - return jsonify(get_upload_url(file_name)) - - - -@upload_api.route("/upload", methods=["GET", "POST"]) -@secure -def upload(): - if request.method == "POST": - if "file" not in request.files: - return jsonify({"error": "No file part"}) - file = request.files["file"] - if file_name_check(file.filename): - print( - f"File '{file.filename}' already exists in the bucket. Choose appropriate action." - ) - return jsonify( - { - "message": f"File '{file.filename}' already exists in the bucket. Choose appropriate action." - } - ) - else: - if file and allowed_file(file.filename): - filename = secure_filename(file.filename) - upload_to_s3(file, filename) - return jsonify({"message": "File uploaded successfully"}) - - return render_template("upload.html") - - -@upload_api.route("/download") -@secure -def download(): - # List objects in the bucket - try: - objects = list_file() - if objects: - files = [obj['Key'] for obj in objects] - # return render_template('list.html', files=files) - return jsonify({"files": files}) - else: - return "The bucket is empty." - - except Exception as e: - return jsonify({f"Error: {e}"}) - - -@upload_api.route("/download/") -@secure -def download_file_csv(filename): - try: - # Download the object from S3 - file = download_file(filename) - return jsonify(file) - - # Send the file for download - except Exception as e: - return f"Error: {e}" - - -@upload_api.route("/delete/", methods=["DELETE"]) -@secure -def delete_file_s3(file_name): - response = delete_s3file(file_name) - return jsonify(response) - - -"""@upload_api.route('/uploadcsv', methods=['POST']) -def uploadcsv(): - BUCKET_NAME = os.getenv('BUCKET_NAME') - - try: - # Check if the file part is present - if 'file' not in request.files: - return jsonify({'error': 'No file part'}) - - file = request.files['file'] - - # Check if the file is uploaded - if file.filename == '': - return jsonify({'error': 'No selected file'}) - - # Save the file to a temporary location - temp_file_path = '/tmp/' + file.filename - file.save(temp_file_path) - - # Upload the file to AWS S3 - s3_key = 'uploads/' + file.filename # S3 object key (adjust as needed) - if upload_to_s3(temp_file_path, BUCKET_NAME, s3_key): - return jsonify({'message': 'File uploaded successfully'}) - else: - return jsonify({'error': 'Failed to upload file to AWS S3'}) - - except Exception as e: - return jsonify({'error': str(e)})""" - - -@upload_api.route("/ping", methods=["POST"]) -def ping(): - return jsonify({"message": "Ping successfully"}) - - -@upload_api.route('/store_file_data', methods=['GET']) -@secure -def store_file_data(): - file_name = request.args.get('fileName') - print(file_name) - insert_one_s3file_metadata(file_name) - - if file_name: - return jsonify({'message': 'Saved successfully'}) - - -def allowed_file(filename): - return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS diff --git a/src/backend/app.py b/src/backend/app.py index 2afcc64..5d61741 100644 --- a/src/backend/app.py +++ b/src/backend/app.py @@ -9,7 +9,8 @@ from api.datapipeline import datapipeline from api.dp_run import dp_run from api.fileWP import fileWP -from api.upload_api import upload_api +from api.misc import misc +from api.file import file from services.auth_service import secure from flask_cors import CORS @@ -34,11 +35,12 @@ oidc = OpenIDConnect(app) -app.register_blueprint(upload_api) +app.register_blueprint(file) app.register_blueprint(datapipeline, url_prefix="/") app.register_blueprint(fileWP) app.register_blueprint(airflow_api) app.register_blueprint(dp_run) +app.register_blueprint(misc) @app.route("/") diff --git a/src/backend/database/models/file_details.py b/src/backend/database/models/file_details.py index a1ba8d6..a23c8c4 100644 --- a/src/backend/database/models/file_details.py +++ b/src/backend/database/models/file_details.py @@ -1,12 +1,28 @@ -class TaskExecutionDetails: - def __init__(self, execution_date, task_instance, task_log): - self.execution_date = execution_date - self.task_instance = task_instance - self.task_log = task_log +import uuid +from datetime import datetime + + +class FileDetails: + def __init__(self, name, mime_type, size, s3_uuid, content_type, storage_class, last_modified): + self.uuid = str(uuid.uuid4()) + self.name = name + self.mime_type = mime_type + self.size = size + self.s3_uuid = s3_uuid + self.content_type = content_type + self.storage_class = storage_class + self.last_modified = last_modified + self.created_at = datetime.now() def to_json(self): return { - 'execution_date': str(self.task_log), - 'task_instance': self.task_log, - 'task_log': self.task_log, + 'uuid': self.uuid, + 'name': self.name, + 'mime_type': self.mime_type, + 'size': self.size, + 's3_uuid': self.s3_uuid, + 'content_type': self.content_type, + 'storage_class': self.storage_class, + 'last_modified': self.last_modified, + 'created_at': self.created_at, } diff --git a/src/backend/database/models/s3_detials_entity.py b/src/backend/database/models/s3_detials_entity.py deleted file mode 100644 index 489b4aa..0000000 --- a/src/backend/database/models/s3_detials_entity.py +++ /dev/null @@ -1,31 +0,0 @@ - -class S3ObjectDetails: - def __init__(self, key, last_modified, size, content_type, storage_class,original_file_name): - self.key = key - self.last_modified = last_modified - self.size = size - self.content_type = content_type - - self.storage_class = storage_class - self.original_file_name = original_file_name - - def to_json(self): - return { - "key": self.key, - "last_modified": self.last_modified, - "size": self.size, - "content_type": self.content_type, - - "storage_class": self.storage_class, - "original_file_name":self.original_file_name - } - - def to_dict(self): - return { - 'key': self.key, - 'last_modified': self.last_modified, - 'size': self.size, - 'content_type': self.content_type, - 'storage_class': self.storage_class, - 'original_file_name': self.original_file_name - } \ No newline at end of file diff --git a/src/backend/database/mongo_repo.py b/src/backend/database/mongo_repo.py index db0c092..ff71963 100644 --- a/src/backend/database/mongo_repo.py +++ b/src/backend/database/mongo_repo.py @@ -19,3 +19,4 @@ metadataDB = db["metadata"] s3filename = db["S3FileNames'"] datapipelineRunDB = db["dp_run"] +fileDetailsDB = db["file_detail"] diff --git a/src/backend/docker-compose.yml b/src/backend/docker-compose.yml index d424117..f80ea0b 100644 --- a/src/backend/docker-compose.yml +++ b/src/backend/docker-compose.yml @@ -1,21 +1,21 @@ version: '3.8' services: -# app: -# environment: -# - MONGODB_URL=amos_mongodb -# - MONGODB_PORT=27017 -# - MONGODB_USER=root -# - MONGODB_PASSWORD=pass -# - HOST_URL=0.0.0.0 -# - HOST_PORT=8000 -# build: . -# volumes: -# - .:/app -# ports: -# - "8000:8000" -# command: python -u app.py -# links: -# - db + app: + environment: + - MONGODB_URL=amos_mongodb + - MONGODB_PORT=27017 + - MONGODB_USER=root + - MONGODB_PASSWORD=pass + - HOST_URL=0.0.0.0 + - HOST_PORT=8000 + build: . + volumes: + - .:/app + ports: + - "8000:8000" + command: python -u app.py + links: + - db db: image: mongo:latest hostname: amos_mongodb diff --git a/src/backend/services/dp_run.py b/src/backend/services/dp_run.py index 7b8623e..0ac724f 100644 --- a/src/backend/services/dp_run.py +++ b/src/backend/services/dp_run.py @@ -1,8 +1,6 @@ -from flask import jsonify - -from api.airflow_api import dagsExecuteById, airflow_post +from api.airflow_api import airflow_post from database.mongo_repo import datapipelineRunDB -from services.upload_to_s3 import download_file +from services.file_storage import download_file def run(executionId): diff --git a/src/backend/services/store_s3metadata.py b/src/backend/services/file_detail.py similarity index 70% rename from src/backend/services/store_s3metadata.py rename to src/backend/services/file_detail.py index 44e2af8..e413e26 100644 --- a/src/backend/services/store_s3metadata.py +++ b/src/backend/services/file_detail.py @@ -1,14 +1,8 @@ import os import boto3 from dotenv import load_dotenv -from services.upload_to_s3 import get_file_details -from database.mongo_repo import s3filename - -# from database.models.s3_detials_entity import S3ObjectDetails - - -# import upload_to_s3 - +from services.file_storage import get_file_details +from database.mongo_repo import s3filename, fileDetailsDB load_dotenv() @@ -31,10 +25,9 @@ def insert_all_s3files_metadata(collection): return response -def insert_one_s3file_metadata(s3_key): - s3file_metadata = get_file_details(s3_key) - print(get_file_details(s3_key)) - response = s3filename.insert_one(s3file_metadata) +def insert_file_details(file_name, s3_uuid, mime_type): + new_file_details = get_file_details(file_name, s3_uuid, mime_type) + response = fileDetailsDB.insert_one(new_file_details) return response diff --git a/src/backend/services/file_storage.py b/src/backend/services/file_storage.py new file mode 100644 index 0000000..0deae98 --- /dev/null +++ b/src/backend/services/file_storage.py @@ -0,0 +1,110 @@ +import string +import random +import uuid + +from botocore.exceptions import NoCredentialsError + +from database.models.file_details import FileDetails +import humanfriendly + +from services.s3_storage import s3_generate_presigned_url, s3_get_head_object, s3_get_download_url, s3_list_objects, \ + s3_delete_file + + +def generated_key_check(file_name): + if file_name_check(file_name): + get_name = file_name.split("_") + key = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(10)) + file_name = str(key) + "_" + get_name[1] + generated_key_check(file_name) + return file_name + else: + return file_name + + +def get_file_upload_url(file_name): + try: + s3_uuid = str(uuid.uuid4()) + url = s3_generate_presigned_url(s3_uuid) + response_data = {'presignedUrl': url, 'fileName': file_name, 's3_uuid': s3_uuid} + + return response_data + except Exception as e: + print(f"Error: {e}") + # TODO error + + +# TODO test this +def get_file_details(file_name, s3_uuid, mime_type): + # Get details of a specific file + try: + + response = s3_get_head_object(s3_uuid) + + new_file_details = FileDetails( + name=file_name, + # TODO add mime type to maybe remove duplicate + mime_type=mime_type, + s3_uuid=s3_uuid, + last_modified=response["LastModified"], + size=response["ContentLength"], + content_type=response["ContentType"], + storage_class="dummy storage class" + ) + return new_file_details.to_json() + + except Exception as e: + print(f"Error: {e}") + + +def download_file(file_name): + try: + try: + url = s3_get_download_url(file_name) + return {"download_url": url} + + except NoCredentialsError: + return {"error": "AWS credentials not available or incorrect."} + + except Exception as e: + print(f"Error: {e}") + + +def list_file(): + try: + response = s3_list_objects() + objects = response.get("Contents", []) + for size in objects: + size["Size"] = humanfriendly.format_size(size["Size"]) + print("s3 connected") + return objects + except Exception as e: + print(f"Error: {e}") + + +def file_name_check(file_name): + try: + # HeadObject returns metadata for an object + obj = s3_get_head_object(file_name) + if obj: + return True # File exists + return False + + except Exception as e: + print(f"Error: {e}") + return False + + +def delete_file(file_name): + try: + if file_name_check(file_name): + s3_delete_file(file_name) + return ["File is deleted seccessfuly"] + else: + return [{"Error": "File not exist"}] + except Exception as e: + return [{f"Error: {e}"}] + +ALLOWED_EXTENSIONS = {"csv"} +def allowed_file(filename): + return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS \ No newline at end of file diff --git a/src/backend/services/s3_storage.py b/src/backend/services/s3_storage.py new file mode 100644 index 0000000..d56bc0e --- /dev/null +++ b/src/backend/services/s3_storage.py @@ -0,0 +1,74 @@ +import os + +import boto3 +from botocore.exceptions import NoCredentialsError +from dotenv import load_dotenv +from flask import jsonify, request + +load_dotenv() + +# AWS S3 configuration +AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY") +AWS_SECRET_KEY = os.getenv("AWS_SECRET_KEY") +REGION = os.getenv("REGION") +BUCKET_NAME = os.getenv("BUCKET_NAME") + +def get_s3_client(): + s3 = boto3.client('s3', aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY, + region_name=REGION) + return s3 + +def s3_generate_presigned_url(file_name): + s3 = get_s3_client() + url = s3.generate_presigned_url('put_object', Params={'Bucket': BUCKET_NAME, 'Key': file_name}, ExpiresIn=3600) + return url + +def s3_get_head_object(file_name): + s3 = get_s3_client() + response = s3.head_object(Bucket=BUCKET_NAME, Key=file_name) + # TODO error handling + return response + +def s3_get_download_url(file_name): + s3 = get_s3_client() + + url = s3.generate_presigned_url( + "get_object", + Params={"Bucket": BUCKET_NAME, "Key": file_name}, + ExpiresIn=3600, + ) + return url + +def s3_list_objects(): + s3 = get_s3_client() + response = s3.list_objects(Bucket=BUCKET_NAME) + return response + +def s3_delete_file(file_name): + s3 = get_s3_client() + s3.delete_object(Bucket=BUCKET_NAME, Key=file_name) + + + +# Function to upload a file to AWS S3 +def upload_to_s3(path, s3_key): + try: + s3 = get_s3_client() + s3.upload_fileobj(path, BUCKET_NAME, s3_key) + return "file_data" + except FileNotFoundError: + return False + except NoCredentialsError: + return False + +# Route to dynamically search for files based on a partial keyword +def search_files(key): + partial_keyword = request.args.get('partial_keyword', '') + + # Search for files in MongoDB based on the partial keyword + results = ""#s3filename.find({"filename": {"$regex": partial_keyword, "$options": "i"}}).limit(10) + + # Convert MongoDB Cursor to list of dictionaries + files = list(results) + + return jsonify({"files": files}) diff --git a/src/backend/services/s3_storage_service.py b/src/backend/services/s3_storage_service.py deleted file mode 100644 index 86381b1..0000000 --- a/src/backend/services/s3_storage_service.py +++ /dev/null @@ -1,16 +0,0 @@ -from flask import Flask, jsonify, request - -from database.models.s3_detials_entity import S3FileName -from database.mongo_repo import s3filename - -# Route to dynamically search for files based on a partial keyword -def search_files(key): - partial_keyword = request.args.get('partial_keyword', '') - - # Search for files in MongoDB based on the partial keyword - results = ""#s3filename.find({"filename": {"$regex": partial_keyword, "$options": "i"}}).limit(10) - - # Convert MongoDB Cursor to list of dictionaries - files = list(results) - - return jsonify({"files": files}) diff --git a/src/backend/services/upload_to_s3.py b/src/backend/services/upload_to_s3.py deleted file mode 100644 index 1bc0cd5..0000000 --- a/src/backend/services/upload_to_s3.py +++ /dev/null @@ -1,166 +0,0 @@ -import string -import random -import uuid -import boto3 -from dotenv import load_dotenv -from botocore.exceptions import NoCredentialsError -import os -from database.models.s3_detials_entity import S3ObjectDetails -from io import BytesIO -import humanfriendly - -# AWS S3 configuration -AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY") -AWS_SECRET_KEY = os.getenv("AWS_SECRET_KEY") -REGION = os.getenv("REGION") -BUCKET_NAME = os.getenv("BUCKET_NAME") - - -# Function to upload a file to AWS S3 -def upload_to_s3(path, s3_key): - load_dotenv() - - try: - s3 = boto3.client( - "s3", - aws_access_key_id=AWS_ACCESS_KEY, - aws_secret_access_key=AWS_SECRET_KEY, - region_name=REGION, - ) - s3.upload_fileobj(path, BUCKET_NAME, s3_key) - # file_data = get_file_details(path, BUCKET_NAME, s3_key) - return "file_data" - except FileNotFoundError: - return False - except NoCredentialsError: - return False - - -def generated_key_check(file_name): - if file_name_check(file_name): - get_name = file_name.split("_") - key = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(10)) - file_name = str(key) + "_" + get_name[1] - generated_key_check(file_name) - return file_name - else: - return file_name - - -def get_upload_url(file_name): - try: - key = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(10)) - file_name = generated_key_check(str(key) + "_" + file_name) - s3 = boto3.client('s3', aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY, - region_name=REGION) - url = s3.generate_presigned_url('put_object', Params={'Bucket': BUCKET_NAME, 'Key': file_name}, ExpiresIn=3600) - response_data = {'presignedUrl': url, 'fileName': file_name} - return response_data - except Exception as e: - print(f"Error: {e}") - - -# TODO test this -def get_file_details(s3_key): - # Get details of a specific file - try: - s3 = boto3.client( - "s3", - aws_access_key_id=AWS_ACCESS_KEY, - aws_secret_access_key=AWS_SECRET_KEY, - region_name=REGION, - ) - response = s3.head_object(Bucket=bucket_name, Key=s3_key) - - # Print details - return S3ObjectDetails( - key=response["Metadata"]["Key"], - last_modified=response["LastModified"], - size=response["ContentLength"], - content_type=response["ContentType"], - etag=response["ETag"], - storage_class=response["StorageClass"], - ) - print(res.to_dict()) - - # Print details - - return res.to_dict() - except Exception as e: - print(f"Error: {e}") - - -def download_file(file_name): - try: - try: - s3 = boto3.client( - "s3", - aws_access_key_id=AWS_ACCESS_KEY, - aws_secret_access_key=AWS_SECRET_KEY, - region_name=REGION, - ) - - url = s3.generate_presigned_url( - "get_object", - Params={"Bucket": BUCKET_NAME, "Key": file_name}, - ExpiresIn=3600, - ) - return {"download_url": url} - - except NoCredentialsError: - return {"error": "AWS credentials not available or incorrect."} - - except Exception as e: - print(f"Error: {e}") - - -def list_file(): - try: - s3 = boto3.client( - "s3", - aws_access_key_id=AWS_ACCESS_KEY, - aws_secret_access_key=AWS_SECRET_KEY, - region_name=REGION, - ) - response = s3.list_objects(Bucket=BUCKET_NAME) - objects = response.get("Contents", []) - for size in objects: - size["Size"] = humanfriendly.format_size(size["Size"]) - print("s3 connected") - return objects - except Exception as e: - print(f"Error: {e}") - - -def file_name_check(file_name): - try: - s3 = boto3.client( - "s3", - aws_access_key_id=AWS_ACCESS_KEY, - aws_secret_access_key=AWS_SECRET_KEY, - region_name=REGION, - ) - # HeadObject returns metadata for an object - s3.head_object(Bucket=BUCKET_NAME, Key=file_name) - return True # File exists - - except Exception as e: - print(f"Error: {e}") - return False - - -def delete_s3file(file_name): - try: - if file_name_check(file_name): - s3 = boto3.client( - "s3", - aws_access_key_id=AWS_ACCESS_KEY, - aws_secret_access_key=AWS_SECRET_KEY, - region_name=REGION, - ) - s3.delete_object(Bucket=BUCKET_NAME, Key=file_name) - return ["File is deleted seccessfuly"] - else: - return [{"Error": "File not exist"}] - except Exception as e: - return [{f"Error: {e}"}] diff --git a/src/frontend/src/app/app-routing.module.ts b/src/frontend/src/app/app-routing.module.ts index 9c3fa34..c661878 100644 --- a/src/frontend/src/app/app-routing.module.ts +++ b/src/frontend/src/app/app-routing.module.ts @@ -1,26 +1,25 @@ import { NgModule } from '@angular/core'; import { RouterModule, Routes } from '@angular/router'; import { LandingComponent } from './pages/landing/landing.component'; -import { UploadFileComponent } from './modules/upload-file/upload-file.component'; -import { DownloadComponent } from './modules/download/download.component'; import {ListDatapipelineComponent} from "./pages/datapipeline/pages/list-datapipeline/list-datapipeline.component"; import {EditDatapipelineComponent} from "./pages/datapipeline/pages/edit-datapipeline/edit-datapipeline.component"; import { ListS3bucketfilesComponent } from './pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component'; import { CreateDatapipelineComponent } from './pages/datapipeline/pages/create-datapipeline/create-datapipeline.component'; import {S3UploadFilesComponent} from './pages/s3-upload-files/s3-upload-files.component'; -import { startDataPipelineComponent } from './pages/start-data-pipeline/start-data-pipeline.component'; +import { StartDataPipelineComponent } from './pages/start-data-pipeline/start-data-pipeline.component'; +import { + ListDatapipelineRunComponent +} from "./pages/datapipeline-run/pages/list-datapipeline-run/list-datapipeline-run.component"; const routes: Routes = [ { path: '', redirectTo: '/home', pathMatch: 'full' }, { path: 'home', component: LandingComponent }, - { path: 'upload', component: UploadFileComponent }, - { path: 'download', component: DownloadComponent }, { path: 'datapipeline',component: ListDatapipelineComponent}, - { path: 'datapipeline/new',component: EditDatapipelineComponent}, + { path: 'datapipeline/new',component: CreateDatapipelineComponent}, { path: 'datapipeline/:id',component: EditDatapipelineComponent}, - { path: 'startpipeline',component: startDataPipelineComponent}, + { path: 'dp_run',component: ListDatapipelineRunComponent}, + { path: 'startpipeline',component: StartDataPipelineComponent}, { path: 's3list', component: ListS3bucketfilesComponent }, - { path: 'newdatapipeline',component:CreateDatapipelineComponent}, { path: 's3upload',component: S3UploadFilesComponent}, // TODO // { path: '**', component: PageNotFoundComponent } diff --git a/src/frontend/src/app/app.module.ts b/src/frontend/src/app/app.module.ts index 658ddf4..c0e1343 100644 --- a/src/frontend/src/app/app.module.ts +++ b/src/frontend/src/app/app.module.ts @@ -6,16 +6,15 @@ import { AppRoutingModule } from './app-routing.module'; import { AppComponent } from './app.component'; import { HeaderComponent } from './modules/header/header.component'; import { LandingComponent } from './pages/landing/landing.component'; -import { UploadFileComponent } from './modules/upload-file/upload-file.component'; import { SideBarComponent } from './modules/side-bar/side-bar.component'; -import { DownloadComponent } from './modules/download/download.component'; import { DataTablesModule } from "angular-datatables"; import {DatapipelineModule} from "./pages/datapipeline/datapipeline.module"; import { NgbModule } from '@ng-bootstrap/ng-bootstrap'; import { ListS3bucketfilesComponent } from './pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component'; import { FormsModule } from '@angular/forms'; import { S3UploadFilesComponent } from './pages/s3-upload-files/s3-upload-files.component'; -import { startDataPipelineComponent } from './pages/start-data-pipeline/start-data-pipeline.component'; +import { StartDataPipelineComponent } from './pages/start-data-pipeline/start-data-pipeline.component'; +import {DatapipelineRunModule} from "./pages/datapipeline-run/datapipeline-run.module"; @NgModule({ @@ -24,11 +23,9 @@ import { startDataPipelineComponent } from './pages/start-data-pipeline/start-da HeaderComponent, LandingComponent, SideBarComponent, - UploadFileComponent, - DownloadComponent, ListS3bucketfilesComponent, S3UploadFilesComponent, - startDataPipelineComponent, + StartDataPipelineComponent, ], imports: [ BrowserModule, @@ -36,6 +33,7 @@ import { startDataPipelineComponent } from './pages/start-data-pipeline/start-da HttpClientModule, DataTablesModule, DatapipelineModule, + DatapipelineRunModule, NgbModule, FormsModule ], diff --git a/src/frontend/src/app/core/services/airflow/airflow.service.spec.ts b/src/frontend/src/app/core/services/airflow/airflow.service.spec.ts new file mode 100644 index 0000000..c8ffed7 --- /dev/null +++ b/src/frontend/src/app/core/services/airflow/airflow.service.spec.ts @@ -0,0 +1,16 @@ +import { TestBed } from '@angular/core/testing'; + +import { AirflowService } from './airflow.service'; + +describe('AirflowService', () => { + let service: AirflowService; + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(AirflowService); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); +}); diff --git a/src/frontend/src/app/core/services/airflow/airflow.service.ts b/src/frontend/src/app/core/services/airflow/airflow.service.ts new file mode 100644 index 0000000..d5b8452 --- /dev/null +++ b/src/frontend/src/app/core/services/airflow/airflow.service.ts @@ -0,0 +1,19 @@ +import { Injectable } from '@angular/core'; +import {HttpClient} from "@angular/common/http"; +import {Router} from "@angular/router"; +import {environment} from "../../../../environments/environment"; +import {Observable} from "rxjs"; +import {Dag} from "../../../entity/dag"; + +@Injectable({ + providedIn: 'root' +}) +export class AirflowService { + + baseUrl = '/dags'; + constructor(public http: HttpClient,private router:Router) { } + + getAllDags() { + return this.http.get(environment.SERVER_URL + this.baseUrl) as Observable; + } +} diff --git a/src/frontend/src/app/core/services/crud/crud.service.ts b/src/frontend/src/app/core/services/crud/crud.service.ts index 75cb07f..bcfe29d 100644 --- a/src/frontend/src/app/core/services/crud/crud.service.ts +++ b/src/frontend/src/app/core/services/crud/crud.service.ts @@ -10,7 +10,7 @@ import { Router } from '@angular/router'; export class CrudService { protected baseUrl: string = '/'; - constructor(private http: HttpClient,private router:Router) { + constructor(public http: HttpClient,private router:Router) { } getAll(): Observable { @@ -23,19 +23,14 @@ export class CrudService { return this.http.get(environment.SERVER_URL + this.baseUrl + "/" + id) as Observable; } - create(entity: Partial) { - console.log(environment.SERVER_URL + this.baseUrl + '/new'); - this.router.navigate(['/datapipeline']); + create(entity: Partial): Observable { return this.http.post(environment.SERVER_URL + this.baseUrl + '/new', entity, {headers: { "Access-Control-Allow-Origin": "*" - } - }).subscribe((value) => - console.log(value)); + }}); } update(id: string, updateEntity: Entity): Observable { - this.router.navigate(['/datapipeline']); return this.http.post(environment.SERVER_URL + this.baseUrl + "/" + id, updateEntity) as Observable; } diff --git a/src/frontend/src/app/core/services/datapipeline-run/datapipeline-run.service.spec.ts b/src/frontend/src/app/core/services/datapipeline-run/datapipeline-run.service.spec.ts new file mode 100644 index 0000000..509039e --- /dev/null +++ b/src/frontend/src/app/core/services/datapipeline-run/datapipeline-run.service.spec.ts @@ -0,0 +1,16 @@ +import { TestBed } from '@angular/core/testing'; + +import { DatapipelineRunService } from './datapipeline-run.service'; + +describe('DatapipelineRunService', () => { + let service: DatapipelineRunService; + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(DatapipelineRunService); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); +}); diff --git a/src/frontend/src/app/core/services/datapipeline-run/datapipeline-run.service.ts b/src/frontend/src/app/core/services/datapipeline-run/datapipeline-run.service.ts new file mode 100644 index 0000000..f57d382 --- /dev/null +++ b/src/frontend/src/app/core/services/datapipeline-run/datapipeline-run.service.ts @@ -0,0 +1,25 @@ +import { Injectable } from '@angular/core'; +import {HttpClient} from "@angular/common/http"; +import {Router} from "@angular/router"; +import {CrudService} from "../crud/crud.service"; +import {Datapipeline} from "../../../entity/datapipeline"; +import {DatapipelineRun} from "../../../entity/datapipelineRun"; +import {Observable} from "rxjs"; +import {environment} from "../../../../environments/environment"; + +@Injectable({ + providedIn: 'root' +}) +export class DatapipelineRunService extends CrudService{ + + constructor(http: HttpClient,router:Router) { + super(http,router); + this.baseUrl = '/dp_run' + } + + + startDatapipelineRun(executionId: string): Observable { + // TODO error handling + return this.http.get(environment.SERVER_URL + this.baseUrl + "/" + executionId + "/run") as Observable; + } +} diff --git a/src/frontend/src/app/core/services/file/file.service.ts b/src/frontend/src/app/core/services/file/file.service.ts index a0651a3..cb5dcaf 100644 --- a/src/frontend/src/app/core/services/file/file.service.ts +++ b/src/frontend/src/app/core/services/file/file.service.ts @@ -7,27 +7,31 @@ import {s3PresignedUploadInfo} from "../../../entity/s3"; @Injectable({ providedIn: 'root' }) -export class FileService{ +export class FileService { - baseUrl = "/download"; - deleteUrl = "/delete"; + baseUrl = "/file"; constructor(private http: HttpClient) { } - download(){ + getAll(){ return this.http.get(environment.SERVER_URL + this.baseUrl) as Observable; } downloadById(id: string) { - return this.http.get(environment.SERVER_URL + this.baseUrl + "/" + id); + return this.http.get(environment.SERVER_URL + this.baseUrl + "/" + id + "/download"); + } + + getById(id: string): Observable { + // TODO error handling + return this.http.get(environment.SERVER_URL + this.baseUrl + "/" + id) as Observable; } deleteById(id: string) { - return this.http.delete(environment.SERVER_URL + this.deleteUrl + "/" + id); + return this.http.delete(environment.SERVER_URL + this.baseUrl + "/" + id); } get_upload_url(): Observable { - return this.http.get(environment.SERVER_URL + '/upload_url') as Observable; + return this.http.get(environment.SERVER_URL + '/file/upload') as Observable; } upload_file_to_url(upload_url_info: s3PresignedUploadInfo, file: any) { diff --git a/src/frontend/src/app/core/services/lists3bucket/lists3bucket.service.ts b/src/frontend/src/app/core/services/lists3bucket/lists3bucket.service.ts index 6c56593..9a0f077 100644 --- a/src/frontend/src/app/core/services/lists3bucket/lists3bucket.service.ts +++ b/src/frontend/src/app/core/services/lists3bucket/lists3bucket.service.ts @@ -12,12 +12,12 @@ export class startdatapipeline { constructor(private http: HttpClient) {} getS3Files(): Observable<{ files: string[] }> { - const url = `${this.baseUrl}/download`; + const url = `${this.baseUrl}/file`; return this.http.get<{ files: string[] }>(url); } getAvailablePipelines(): Observable { - const url = `${this.baseUrl}/datapipelines`; + const url = `${this.baseUrl}/dags`; return this.http.get(url); } diff --git a/src/frontend/src/app/core/services/s3-file-upload.service.ts b/src/frontend/src/app/core/services/s3-file-upload.service.ts index 6fec886..dcc85bb 100644 --- a/src/frontend/src/app/core/services/s3-file-upload.service.ts +++ b/src/frontend/src/app/core/services/s3-file-upload.service.ts @@ -13,7 +13,7 @@ export class S3FileUploadService { constructor(private http: HttpClient) { } - uploadCsv(file: File): FormData { + getFormDataFromFile(file: File): FormData { const formData = new FormData(); formData.append('file', file); return formData; @@ -23,16 +23,21 @@ export class S3FileUploadService { return this.http.post(`${this.backendUrl}/upload`, formData); } - getPresignedUrl(fileName: string): Observable<{ presignedUrl: string; fileName: string }> { - return this.http.get<{ presignedUrl: string; fileName: string }>(`${this.backendUrl}/upload_url?fileName=${fileName}`); + getPresignedUrl(fileName: string): Observable<{ presignedUrl: string; fileName: string; s3_uuid: string, }> { + return this.http.post<{ presignedUrl: string; fileName: string; s3_uuid: string }>( + `${this.backendUrl}/file/upload`, + {"fileName": fileName}, + {headers: { + "Access-Control-Allow-Origin": "*" + }}); } uploadFileToS3Presigned(presignedUrl: string, formData: FormData): Observable { return this.http.put(presignedUrl, formData); } - storeFileDetails(fileName: string): Observable { - return this.http.get(`${this.backendUrl}/store_file_data?fileName=${fileName}`); + createFileDetails(fileName: string, s3_uuid: string, mime_type: string): Observable { + return this.http.post(`${this.backendUrl}/file/new`, {"fileName": fileName, "s3_uuid": s3_uuid, "mime_type": mime_type }); } } diff --git a/src/frontend/src/app/entity/dag.ts b/src/frontend/src/app/entity/dag.ts new file mode 100644 index 0000000..a29981c --- /dev/null +++ b/src/frontend/src/app/entity/dag.ts @@ -0,0 +1,6 @@ +export interface Dag { + dag_id: string | null; + description: string | null; + is_active: boolean | null; + is_paused: boolean | null; +} diff --git a/src/frontend/src/app/entity/datapipelineRun.ts b/src/frontend/src/app/entity/datapipelineRun.ts new file mode 100644 index 0000000..81f0218 --- /dev/null +++ b/src/frontend/src/app/entity/datapipelineRun.ts @@ -0,0 +1,8 @@ +export interface DatapipelineRun { + executionId: string | null; + datapipelineId: string | null; + fileId: string | null; + result: JSON | null; + create_date: Date | null; + state: string | null; +} diff --git a/src/frontend/src/app/modules/download/download.component.html b/src/frontend/src/app/modules/download/download.component.html deleted file mode 100644 index 7a8f193..0000000 --- a/src/frontend/src/app/modules/download/download.component.html +++ /dev/null @@ -1,61 +0,0 @@ - - -
- - -
- - - - Data Name - Pipeline - Size - Date - - -
- - - - {{item.invoiceNo}} - {{item.customerName}} - {{item.remarks}} - {{item.total | currency}} - {{item.tax | currency}} - {{item.netTotal | currency}} - - - - - - - - - - - - - - -
diff --git a/src/frontend/src/app/modules/download/download.component.scss b/src/frontend/src/app/modules/download/download.component.scss deleted file mode 100644 index c4eeca0..0000000 --- a/src/frontend/src/app/modules/download/download.component.scss +++ /dev/null @@ -1,3 +0,0 @@ -.container { - display: flex; - } \ No newline at end of file diff --git a/src/frontend/src/app/modules/download/download.component.spec.ts b/src/frontend/src/app/modules/download/download.component.spec.ts deleted file mode 100644 index 743cdf4..0000000 --- a/src/frontend/src/app/modules/download/download.component.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { ComponentFixture, TestBed } from '@angular/core/testing'; - -import { DownloadComponent } from './download.component'; - -describe('DownloadComponent', () => { - let component: DownloadComponent; - let fixture: ComponentFixture; - - beforeEach(() => { - TestBed.configureTestingModule({ - declarations: [DownloadComponent] - }); - fixture = TestBed.createComponent(DownloadComponent); - component = fixture.componentInstance; - fixture.detectChanges(); - }); - - it('should create', () => { - expect(component).toBeTruthy(); - }); -}); diff --git a/src/frontend/src/app/modules/download/download.component.ts b/src/frontend/src/app/modules/download/download.component.ts deleted file mode 100644 index 0504b18..0000000 --- a/src/frontend/src/app/modules/download/download.component.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { Component, OnInit } from '@angular/core'; -import { Subject } from 'rxjs'; -import { RestApiService } from 'src/app/core/services/restApi/rest-api.service' - - - -@Component({ - selector: 'app-download', - templateUrl: './download.component.html', - styleUrls: ['./download.component.scss'] -}) -export class DownloadComponent implements OnInit { - uploadedFiles: string[] = []; - downloadheader: any; - - constructor(private restapiservice: RestApiService) { } - - downloadCsv(): void { - this.restapiservice.download(); - } - // dtoptions: DataTables.Settings = {}; - // dtTrigger: Subject = new Subject(); - - ngOnInit(): void { - // this.dtoptions = { - // pagingType : "full_numbers" - // }; - // this.getUploadedFiles(); - } - // downloadFile() { - // this.restapiservice.downloadFile('my-csv-file.csv').subscribe(response => { - // const blob = new Blob([response], { type: 'text/csv' }); - // const url = window.URL.createObjectURL(blob); - // window.open(url); - // }); - // } - // getUploadedFiles(): void { - // this.restapiservice.getUploadedFiles().subscribe(files => { - // this.uploadedFiles = files; - // this.dtTrigger.next(null); - // }); - // } - // downloadFile(fileName: string): void { - // this.restapiservice.downloadFile(fileName).subscribe(data => { - // const blob = new Blob([data]); - // const url = window.URL.createObjectURL(blob); - // const a = document.createElement('a'); - // a.href = url; - // a.download = fileName; - // document.body.appendChild(a); - // a.click(); - // document.body.removeChild(a); - // window.URL.revokeObjectURL(url); - // }); - - // LoadInvoice() { - // this.service.GetAllInvoice().subscribe(res => { - // this.Invoiceheader = res; - // this.dtTrigger.next(null); - // }); - // } - -} - diff --git a/src/frontend/src/app/modules/side-bar/side-bar.component.html b/src/frontend/src/app/modules/side-bar/side-bar.component.html index abded14..6b7f3ab 100644 --- a/src/frontend/src/app/modules/side-bar/side-bar.component.html +++ b/src/frontend/src/app/modules/side-bar/side-bar.component.html @@ -35,6 +35,35 @@

+
+
+ +
+ +
Upload
+
+
+ +
+
+ +
+ +
Start Runs
+
+
+ +
+
+ +
+ +
Runs
+
+
+ + +
diff --git a/src/frontend/src/app/modules/upload-file/upload-file.component.html b/src/frontend/src/app/modules/upload-file/upload-file.component.html deleted file mode 100644 index eb78504..0000000 --- a/src/frontend/src/app/modules/upload-file/upload-file.component.html +++ /dev/null @@ -1,24 +0,0 @@ -
- -
- - - - - - - - - - - - diff --git a/src/frontend/src/app/modules/upload-file/upload-file.component.scss b/src/frontend/src/app/modules/upload-file/upload-file.component.scss deleted file mode 100644 index e69de29..0000000 diff --git a/src/frontend/src/app/modules/upload-file/upload-file.component.spec.ts b/src/frontend/src/app/modules/upload-file/upload-file.component.spec.ts deleted file mode 100644 index 0183dfb..0000000 --- a/src/frontend/src/app/modules/upload-file/upload-file.component.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { ComponentFixture, TestBed } from '@angular/core/testing'; - -import { UploadFileComponent } from './upload-file.component'; - -describe('UploadFileComponent', () => { - let component: UploadFileComponent; - let fixture: ComponentFixture; - - beforeEach(() => { - TestBed.configureTestingModule({ - declarations: [UploadFileComponent] - }); - fixture = TestBed.createComponent(UploadFileComponent); - component = fixture.componentInstance; - fixture.detectChanges(); - }); - - it('should create', () => { - expect(component).toBeTruthy(); - }); -}); diff --git a/src/frontend/src/app/modules/upload-file/upload-file.component.ts b/src/frontend/src/app/modules/upload-file/upload-file.component.ts deleted file mode 100644 index 309c93b..0000000 --- a/src/frontend/src/app/modules/upload-file/upload-file.component.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { Component } from '@angular/core'; -import { RestApiService } from 'src/app/core/services/restApi/rest-api.service'; - - -@Component({ - selector: 'app-upload-file', - templateUrl: './upload-file.component.html', - styleUrls: ['./upload-file.component.scss'] -}) - -export class UploadFileComponent { - private selectedFile: File | null = null; - constructor( private restapi: RestApiService) { - } - - onFileSelected(event: any): void { - this.selectedFile = event.target.files[0]; - } - - uploadCsv(): void { - if (this.selectedFile) { - this.restapi.uploadCsvFile(this.selectedFile) - .then(() => console.log('File uploaded successfully')) - .catch(error => console.error('Error uploading file', error)); - } else { - console.error('No file selected'); - } - } - // onFileSelected(event: any): void { - // this.selectedFile = event.target.files[0] as File; - // console.log("filepath",event.target.value) - // console.log(this.selectedFile) - // // check condition - // } - - - - - - // getallEndpoint() { - // console.log("getting all pipeline") - // this.restapi.getAllDataPipelines().then((data) => { - // console.log("all pipeline ", data) - // }, (error) => { - // console.error("creating pipeline",error) - // }) - // } - - // getDataPipelineId(id:string) { - // console.log("getting pipeline with id") - // this.restapi.getAllDataPipelines(id).then((data) => { - // console.log(" pipeline with id", data) - // }, (error) => { - // console.error("creating pipeline",error) - // }) - // } - - // createDatapipeline() { - // console.log("creating pipeline") - // this.restapi.createDataPipeline("newPipeline","trying new pipleine ").then((data) => { - // console.log("create pipeline ", data) - // }, (error) => { - // console.error("creating pipeline",error) - // }) - // } - - // uploadBackendCSV() { - // this.restapi.uploadCSV(this.selectedFile).then((data) => { - // console.log("upload succesful") - // }, (error) => { - // console.error("upload successful ",error) - // }) - - // } - - // ngOnInit(): void { - // } - - // onFileSelected(event: any) { - // const file = event.target.files[0]; - // this.restapi.uploadFile(file).subscribe(response => { - // console.log(response); - // }); - // } - - - -} diff --git a/src/frontend/src/app/pages/datapipeline-run/datapipeline-run.module.ts b/src/frontend/src/app/pages/datapipeline-run/datapipeline-run.module.ts index faeaf40..f77e676 100644 --- a/src/frontend/src/app/pages/datapipeline-run/datapipeline-run.module.ts +++ b/src/frontend/src/app/pages/datapipeline-run/datapipeline-run.module.ts @@ -1,6 +1,8 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import { ListDatapipelineRunComponent } from './pages/list-datapipeline-run/list-datapipeline-run.component'; +import {FormsModule} from "@angular/forms"; +import {RouterLink, RouterLinkActive} from "@angular/router"; @@ -8,8 +10,11 @@ import { ListDatapipelineRunComponent } from './pages/list-datapipeline-run/list declarations: [ ListDatapipelineRunComponent ], - imports: [ - CommonModule - ] + imports: [ + CommonModule, + FormsModule, + RouterLink, + RouterLinkActive + ] }) export class DatapipelineRunModule { } diff --git a/src/frontend/src/app/pages/datapipeline-run/pages/list-datapipeline-run/list-datapipeline-run.component.html b/src/frontend/src/app/pages/datapipeline-run/pages/list-datapipeline-run/list-datapipeline-run.component.html index c77903a..aa3a20b 100644 --- a/src/frontend/src/app/pages/datapipeline-run/pages/list-datapipeline-run/list-datapipeline-run.component.html +++ b/src/frontend/src/app/pages/datapipeline-run/pages/list-datapipeline-run/list-datapipeline-run.component.html @@ -1 +1,33 @@ -

list-datapipeline-run works!

+ + + + + + + + + + + + + + + + + + + + + + + + +
#ExecutionIdDatapipelineIdFileIdResultState
{{ i + 1 }} + {{ datapipelineRun.executionId }} + {{ datapipelineRun.datapipelineId }}{{ datapipelineRun.fileId }}{{ datapipelineRun.result | json}}{{ datapipelineRun.state }} +
+ + + +
+
diff --git a/src/frontend/src/app/pages/datapipeline-run/pages/list-datapipeline-run/list-datapipeline-run.component.ts b/src/frontend/src/app/pages/datapipeline-run/pages/list-datapipeline-run/list-datapipeline-run.component.ts index ff20f95..9cad066 100644 --- a/src/frontend/src/app/pages/datapipeline-run/pages/list-datapipeline-run/list-datapipeline-run.component.ts +++ b/src/frontend/src/app/pages/datapipeline-run/pages/list-datapipeline-run/list-datapipeline-run.component.ts @@ -1,10 +1,50 @@ -import { Component } from '@angular/core'; +import {Component, OnInit} from '@angular/core'; +import {Observable} from "rxjs"; +import {ActivatedRoute, Router} from "@angular/router"; +import {DatapipelineRun} from "../../../../entity/datapipelineRun"; +import {DatapipelineRunService} from "../../../../core/services/datapipeline-run/datapipeline-run.service"; @Component({ selector: 'app-list-datapipeline-run', templateUrl: './list-datapipeline-run.component.html', styleUrls: ['./list-datapipeline-run.component.scss'] }) -export class ListDatapipelineRunComponent { +export class ListDatapipelineRunComponent implements OnInit { + public datapipelineRuns = new Observable; + constructor(private datapipelineRunService: DatapipelineRunService, + private router : Router,private route: ActivatedRoute,) { + } + + ngOnInit(): void { + this.datapipelineRuns = this.datapipelineRunService.getAll(); + } + + getID(): string { + const id : string = this.route.snapshot.paramMap.get('id') ??'null value'; + console.log(id); + return id + } + + edit(uuid: string | null) { + this.router.navigate(['/datapipelineRun', uuid]); + } + + delete(uuid: string | null) { + uuid = uuid ??'null value' + this.datapipelineRunService.delete(uuid).subscribe(res => { + "Delete successful" + }, err => { + "Delete failed" + }); + // throw Error('unimplemented error'); + } + + upload(uuid: string | null) { + throw Error('unimplemented error'); + } + + rerun(uuid: any) { + throw Error('unimplemented error'); + } } diff --git a/src/frontend/src/app/pages/datapipeline/pages/create-datapipeline/create-datapipeline.component.ts b/src/frontend/src/app/pages/datapipeline/pages/create-datapipeline/create-datapipeline.component.ts index 8599311..69444a9 100644 --- a/src/frontend/src/app/pages/datapipeline/pages/create-datapipeline/create-datapipeline.component.ts +++ b/src/frontend/src/app/pages/datapipeline/pages/create-datapipeline/create-datapipeline.component.ts @@ -1,6 +1,7 @@ import { Component } from '@angular/core'; import {FormBuilder} from "@angular/forms"; import {DatapipelineService} from "../../../../core/services/datapipeline/datapipeline.service"; +import {Router} from "@angular/router"; @Component({ @@ -17,13 +18,14 @@ export class CreateDatapipelineComponent { constructor( private datapipelineService: DatapipelineService, private formBuilder: FormBuilder, + private router : Router, ) { } onSubmit(): void { - console.log(this.datapipelineForm.getRawValue()); this.datapipelineService.create(this.datapipelineForm.getRawValue()); + this.router.navigate(['/datapipeline']); } } diff --git a/src/frontend/src/app/pages/datapipeline/pages/edit-datapipeline/edit-datapipeline.component.ts b/src/frontend/src/app/pages/datapipeline/pages/edit-datapipeline/edit-datapipeline.component.ts index a45a4e0..d5f6d73 100644 --- a/src/frontend/src/app/pages/datapipeline/pages/edit-datapipeline/edit-datapipeline.component.ts +++ b/src/frontend/src/app/pages/datapipeline/pages/edit-datapipeline/edit-datapipeline.component.ts @@ -1,7 +1,7 @@ import { Component,OnInit} from '@angular/core'; import {FormBuilder} from "@angular/forms"; import { DatapipelineService } from "../../../../core/services/datapipeline/datapipeline.service"; -import { ActivatedRoute } from '@angular/router'; +import {ActivatedRoute, Router} from '@angular/router'; import { Observable } from 'rxjs'; import { Entity } from 'aws-sdk/clients/costexplorer'; @@ -16,6 +16,7 @@ export class EditDatapipelineComponent{ private datapipelineService: DatapipelineService, private formBuilder: FormBuilder, private route: ActivatedRoute, + private router : Router, ) { } @@ -43,6 +44,7 @@ export class EditDatapipelineComponent{ } ); this.datapipelineService.update(this.getID(),this.datapipelineForm.getRawValue()).subscribe(); + this.router.navigate(['/datapipeline']); } } diff --git a/src/frontend/src/app/pages/datapipeline/pages/list-datapipeline/list-datapipeline.component.html b/src/frontend/src/app/pages/datapipeline/pages/list-datapipeline/list-datapipeline.component.html index c5017ab..8830ddf 100644 --- a/src/frontend/src/app/pages/datapipeline/pages/list-datapipeline/list-datapipeline.component.html +++ b/src/frontend/src/app/pages/datapipeline/pages/list-datapipeline/list-datapipeline.component.html @@ -1,27 +1,32 @@ - + + - - - + + + + - + - + + + + diff --git a/src/frontend/src/app/pages/datapipeline/pages/list-datapipeline/list-datapipeline.component.ts b/src/frontend/src/app/pages/datapipeline/pages/list-datapipeline/list-datapipeline.component.ts index abd0233..ee7ae8e 100644 --- a/src/frontend/src/app/pages/datapipeline/pages/list-datapipeline/list-datapipeline.component.ts +++ b/src/frontend/src/app/pages/datapipeline/pages/list-datapipeline/list-datapipeline.component.ts @@ -1,10 +1,11 @@ import { Component, OnInit } from '@angular/core'; import { Router } from '@angular/router'; -import {HttpClient} from "@angular/common/http"; import {Observable} from "rxjs"; import {DatapipelineService} from "../../../../core/services/datapipeline/datapipeline.service"; import { Datapipeline } from "../../../../entity/datapipeline"; import { ActivatedRoute } from '@angular/router'; +import {AirflowService} from "../../../../core/services/airflow/airflow.service"; +import {FileService} from "../../../../core/services/file/file.service"; @@ -13,16 +14,23 @@ import { ActivatedRoute } from '@angular/router'; templateUrl: './list-datapipeline.component.html', styleUrls: ['./list-datapipeline.component.scss'] }) -export class ListDatapipelineComponent implements OnInit{ +export class ListDatapipelineComponent implements OnInit { - public datapipelines = new Observable; + public datapipelines$ = new Observable; + public dags$: Observable; + public files$: Observable; - constructor(private datapipelineService: DatapipelineService,private router : Router,private route: ActivatedRoute,) { + constructor(private datapipelineService: DatapipelineService, + private router : Router, + private route: ActivatedRoute, + private airflowService: AirflowService, + private fileService: FileService) { } ngOnInit(): void { - this.datapipelines = this.datapipelineService.getAll(); - + this.datapipelines$ = this.datapipelineService.getAll(); + this.dags$ = this.airflowService.getAllDags(); + this.files$ = this.fileService.getAll(); } getID(): string { diff --git a/src/frontend/src/app/pages/s3-upload-files/s3-upload-files.component.html b/src/frontend/src/app/pages/s3-upload-files/s3-upload-files.component.html index 19c6f48..1682cd8 100644 --- a/src/frontend/src/app/pages/s3-upload-files/s3-upload-files.component.html +++ b/src/frontend/src/app/pages/s3-upload-files/s3-upload-files.component.html @@ -1,16 +1,16 @@ +
+
+

Selected File: {{ this.selectedFile?.name }}

- + -

s3-upload-files!

- -
-
{{ successMessage }}
-
{{ errorMessage }}
- - + +
{{ successMessage }}
+
{{ errorMessage }}
+ +
- diff --git a/src/frontend/src/app/pages/s3-upload-files/s3-upload-files.component.ts b/src/frontend/src/app/pages/s3-upload-files/s3-upload-files.component.ts index 875df18..0ca9576 100644 --- a/src/frontend/src/app/pages/s3-upload-files/s3-upload-files.component.ts +++ b/src/frontend/src/app/pages/s3-upload-files/s3-upload-files.component.ts @@ -7,7 +7,7 @@ import { S3FileUploadService } from 'src/app/core/services/s3-file-upload.servic styleUrls: ['./s3-upload-files.component.scss'] }) export class S3UploadFilesComponent { - private selectedFile!: File; + public selectedFile!: File; public successMessage!: string; public errorMessage!: string; @@ -29,39 +29,19 @@ export class S3UploadFilesComponent { } } - - uploadFile() { - if (!this.selectedFile) { - return; - } - - const formData = this.fileUploadService.uploadCsv(this.selectedFile); - - this.fileUploadService.uploadFileToS3(formData).subscribe( - response => { - this.successMessage = 'File uploaded successfully!'; - console.log(response); - }, - error => { - console.error('Error uploading file:', error); - } - ); - } - - - - uploadFileWithUrl() { if (!this.selectedFile) { return; } - const formData = this.fileUploadService.uploadCsv(this.selectedFile); + const formData = this.fileUploadService.getFormDataFromFile(this.selectedFile); this.fileUploadService.getPresignedUrl(this.selectedFile.name).subscribe( - (response: { presignedUrl: string, fileName: string }) => { - const { presignedUrl, fileName } = response; - this.uploadToPresignedUrl(presignedUrl, formData, fileName); + (response: { presignedUrl: string, fileName: string, s3_uuid: string}) => { + const { presignedUrl, fileName, s3_uuid } = response; + + this.uploadToPresignedUrl(presignedUrl, formData, fileName, s3_uuid, this.selectedFile.type); + console.log(presignedUrl) }, (error) => { @@ -70,11 +50,13 @@ export class S3UploadFilesComponent { ); } - private uploadToPresignedUrl(presignedUrl: string, formData: FormData, fileName: string): void { + + + private uploadToPresignedUrl(presignedUrl: string, formData: FormData, fileName: string, s3_uuid: string, mime_type: string): void { this.fileUploadService.uploadFileToS3Presigned(presignedUrl, formData).subscribe( (response) => { this.successMessage = 'File uploaded successfully!'; - this.store_file_data(fileName) + this.createFileDetails(fileName, s3_uuid, mime_type) console.log(response); }, @@ -85,8 +67,8 @@ export class S3UploadFilesComponent { } ); } - private store_file_data(fileName: string): void { - this.fileUploadService.storeFileDetails(fileName).subscribe( + private createFileDetails(fileName: string, s3_uuid: string, mime_type: string): void { + this.fileUploadService.createFileDetails(fileName, s3_uuid, mime_type).subscribe( (response) =>{ }, (error) => { diff --git a/src/frontend/src/app/pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component.html b/src/frontend/src/app/pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component.html index da49054..f3949b7 100644 --- a/src/frontend/src/app/pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component.html +++ b/src/frontend/src/app/pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component.html @@ -1,7 +1,3 @@ -
- -
-
#NameConfigActionDAG IdDescriptionIs activeIs paused
{{ i + 1 }} - {{ datapipeline.name }} + {{ datapipeline.dag_id }} {{ datapipeline.config }}{{ datapipeline.description }}{{ datapipeline.is_active }}{{ datapipeline.is_paused }}
- - - + + +
@@ -11,22 +7,26 @@ + - + - - - - - + + + + + + + + diff --git a/src/frontend/src/app/pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component.ts b/src/frontend/src/app/pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component.ts index fc6f426..5d5170f 100644 --- a/src/frontend/src/app/pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component.ts +++ b/src/frontend/src/app/pages/s3bucketfiles/list-s3bucketfiles/list-s3bucketfiles.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit, OnDestroy} from '@angular/core'; import { RestApiService } from 'src/app/core/services/restApi/rest-api.service'; import {FileService} from "../../../core/services/file/file.service"; -import {Subject, Observable} from "rxjs"; +import {Subject, Observable, Subscription} from "rxjs"; import {s3PresignedUploadInfo} from "../../../entity/s3"; @@ -19,31 +19,32 @@ export class ListS3bucketfilesComponent implements OnInit,OnDestroy { public fileDownload: any; public upload_url_info: s3PresignedUploadInfo | null = null; - dtOptions: DataTables.Settings = {}; + dtOptions: DataTables.Settings = { + pagingType:"full_numbers" + }; dtTrigger: Subject = new Subject(); + public filesSubscription: Subscription; + private downloadSubscription: Subscription; constructor( private restapi: RestApiService, private fileService: FileService) { - - // this.fileDownload = this.fileService.download(); + this.filesSubscription = this.fileService.getAll().subscribe(res => { + this.fileDownload = res; + this.dtTrigger.next(null); + }) } ngOnInit(): void { - this.dtOptions = { - pagingType:"full_numbers" - }; - this.s3_file_details(); this.fileService.get_upload_url().subscribe((value) => this.upload_url_info = value); } ngOnDestroy(): void { - + this.dtTrigger.unsubscribe(); + if (this.filesSubscription) { + this.filesSubscription.unsubscribe(); + } + if (this.downloadSubscription) { + this.downloadSubscription.unsubscribe(); + } } - s3_file_details(): void{ - this.fileDownload = this.fileService.download().subscribe(res => { - this.fileDownload = res; - this.dtTrigger.next(null); - }) - } - onFileSelected(event: any): void { this.selectedFile = event.target.files[0]; } @@ -60,7 +61,11 @@ export class ListS3bucketfilesComponent implements OnInit,OnDestroy { handleDownload(id: string) { // TODO bad subscibe as the subscription is not ending here, - this.fileService.downloadById(id).subscribe((value: any) => + if (this.downloadSubscription) { + this.downloadSubscription.unsubscribe(); + } + + this.downloadSubscription = this.fileService.downloadById(id).subscribe((value: any) => { if (value.download_url){ window.open(value.download_url) @@ -74,11 +79,12 @@ export class ListS3bucketfilesComponent implements OnInit,OnDestroy { {}); } - upload_file_to_url(file: any) { - if (this.upload_url_info) { - this.fileService.upload_file_to_url(this.upload_url_info, file); - } - } + // TODO + // upload_file_to_url(file: any) { + // if (this.upload_url_info) { + // this.fileService.upload_file_to_url(this.upload_url_info, file); + // } + // } } diff --git a/src/frontend/src/app/pages/start-data-pipeline/start-data-pipeline.component.html b/src/frontend/src/app/pages/start-data-pipeline/start-data-pipeline.component.html index 654ffc1..0497f56 100644 --- a/src/frontend/src/app/pages/start-data-pipeline/start-data-pipeline.component.html +++ b/src/frontend/src/app/pages/start-data-pipeline/start-data-pipeline.component.html @@ -1,25 +1,49 @@

Start Pipeline

- - - - -

Select S3 Files

- - -
- -
- -
-

Available Pipelines:

-
    -
  • {{ pipeline.name }}
    • -
+
+
+
+ +
+ + + +
+

+ Datapipeline: {{this.selectedDag?.dag_id}} +

+
+
+
+
+
+
+ +
+ + + +
+

+ File: {{this.selectedFile?.name}} +

+
+
+ +
+
+
+ +
+
diff --git a/src/frontend/src/app/pages/start-data-pipeline/start-data-pipeline.component.ts b/src/frontend/src/app/pages/start-data-pipeline/start-data-pipeline.component.ts index 3318f4e..5cd6a0f 100644 --- a/src/frontend/src/app/pages/start-data-pipeline/start-data-pipeline.component.ts +++ b/src/frontend/src/app/pages/start-data-pipeline/start-data-pipeline.component.ts @@ -1,7 +1,10 @@ -import { Component } from '@angular/core'; +import {Component, OnInit} from '@angular/core'; import { Observable } from 'rxjs/internal/Observable'; import { HttpClient } from '@angular/common/http'; import { startdatapipeline } from 'src/app/core/services/lists3bucket/lists3bucket.service'; +import {AirflowService} from "../../core/services/airflow/airflow.service"; +import {FileService} from "../../core/services/file/file.service"; +import {DatapipelineRunService} from "../../core/services/datapipeline-run/datapipeline-run.service"; @Component({ @@ -10,41 +13,40 @@ import { startdatapipeline } from 'src/app/core/services/lists3bucket/lists3buck styleUrls: ['./start-data-pipeline.component.scss'] }) -export class startDataPipelineComponent { - s3Files: string[] = []; - pipelines: any[] | undefined; - selectedS3File: string | undefined; - selectedPipeline: any | undefined; - - constructor(private backendService: startdatapipeline) {} - - getS3Files() { - this.backendService.getS3Files().subscribe( - (response: { files: string[] }) => { - console.log('API Response:', response); - // ... rest of the code - this.s3Files = response.files || []; - }, - (error) => { - console.error('Error fetching S3 files:', error); - } - ); - } - +export class StartDataPipelineComponent implements OnInit { + public dags$: Observable; + public files$: Observable; + selectedDag: any; + selectedFile: any; + constructor(private dpRunService: DatapipelineRunService, + private airflowService: AirflowService, + private fileService: FileService) {} - getAvailablePipelines() { - this.backendService.getAvailablePipelines().subscribe((pipelines) => { - this.pipelines = pipelines; - }); - } + ngOnInit(): void { + this.dags$ = this.airflowService.getAllDags(); + this.files$ = this.fileService.getAll(); + } startPipeline() { - if (this.selectedS3File && this.selectedPipeline) { - const pipelineId = this.selectedPipeline.id; // Adjust this based on your data structure - this.backendService.startPipeline(this.selectedS3File, pipelineId).subscribe(() => { - console.log('Pipeline started successfully!'); - }); + if (this.selectedFile && this.selectedDag) { + this.dpRunService.create({"datapipelineId": this.selectedDag.dag_id, "fileId": this.selectedFile?.s3_uuid}) + .subscribe((value: any) => { + const executionId = value?.object.executionId; + console.log(executionId); + // TODO dont subscribe in a subscribe q_q but for now it can work + this.dpRunService.startDatapipelineRun(executionId).subscribe(); + } ); + } else { + throw Error("File and/or dag not selected."); } } + + changeDag(dag: any) { + this.selectedDag = dag; + } + + changeFile(file: any) { + this.selectedFile = file; + } } diff --git a/src/frontend/src/environments/environment.development.ts b/src/frontend/src/environments/environment.development.ts index cb2342d..5b89f0f 100644 --- a/src/frontend/src/environments/environment.development.ts +++ b/src/frontend/src/environments/environment.development.ts @@ -1,8 +1,4 @@ export const environment = { production: false, - AWS_ACCESS_KEY: "", - AWS_SECRET_KEY: "", SERVER_URL: "http://localhost:8000", - BACKEND_URL: "http://192.168.1.41:5000/" - }; diff --git a/src/frontend/src/environments/environment.ts b/src/frontend/src/environments/environment.ts index 929b1f2..0e39584 100644 --- a/src/frontend/src/environments/environment.ts +++ b/src/frontend/src/environments/environment.ts @@ -1,7 +1,4 @@ export const environment = { production: false, - AWS_ACCESS_KEY: "", - AWS_SECRET_KEY: "", SERVER_URL: "http://localhost:8000", - BACKEND_URL: "http://192.168.1.41:5000/" }; diff --git a/src/frontend/tsconfig.json b/src/frontend/tsconfig.json index def9c59..ff9e726 100644 --- a/src/frontend/tsconfig.json +++ b/src/frontend/tsconfig.json @@ -2,6 +2,7 @@ { "compileOnSave": false, "compilerOptions": { + "strictPropertyInitialization": false, "baseUrl": "./", "outDir": "./dist/out-tsc", "forceConsistentCasingInFileNames": true,
LastModified Date File Size StorageClassS3 Uuid Action
{{ i + 1 }}{{ file.Key }}{{ file.StorageClass }}{{ file.LastModified}}{{ file.Size}}{{ file.StorageClass}}{{ file.name }}{{ file.mime_type }}{{ file.last_modified}}{{ file.size}}{{ file.storage_class}}{{ file.s3_uuid}}
- - + +