Added test and error handling

.coveralls.yml

@@ -0,0 +1,3 @@
+service_name: travis-pro
+repo_token: QK1j7BHQFmk4IBo8y0JcAbFVByR5gIsgn
+

README.md

@@ -1,5 +1,7 @@
 # DB API for EVA
 
+[![Coverage Status](https://coveralls.io/repos/github/georgia-tech-db/eva-api/badge.svg?branch=test)](https://coveralls.io/github/georgia-tech-db/eva-api?branch=test)
+
 ## Install
 
 Installation of EVA involves setting a virtual environment using miniconda.
@@ -31,6 +33,7 @@ Currently supported APIs:
 - execute / execute_async
 - fetch_one / fetch_one_async
 - fetch_all / fetch_all_async
+- load a video
 
 Usage: check [db_api_example.py](db_api_example.py)
 

demo/config.py

@@ -0,0 +1,9 @@
+import pathlib
+import os
+
+FLASK_HOST="0.0.0.0"
+FLASK_PORT=5001
+
+DATASET_DIR = pathlib.Path().absolute() / 'dataset'
+EVA_HOST = os.getenv('EVA_HOSTNAME')
+EVA_PORT = int(os.getenv('EVA_PORT'))

demo/demo_api.py

@@ -0,0 +1,130 @@
+import asyncio
+import json
+import os
+import random
+import pathlib
+from flask import Flask, request, make_response, send_file, jsonify
+from flask_restful import Resource, Api
+from src.db_api import connect_async
+from utils import create_video_from_frames, delete_old_video_files
+from config import FLASK_HOST, FLASK_PORT, DATASET_DIR, EVA_HOST, EVA_PORT
+
+app = Flask(__name__)
+api = Api(app)
+
+input = []
+
+class SendVideo(Resource):
+
+    def get(self, video_name):
+        try: 
+            video_name = video_name + ".mp4"
+            video_path = DATASET_DIR / video_name
+            return send_file(video_path)
+        except Exception as e:
+            return str(e)
+
+
+class SendName(Resource):
+    request_id = 0
+
+    def get(self):
+        SendName.request_id = SendName.request_id + 1
+        name = generate_video_name(SendName.request_id)
+        return jsonify({"name": name})
+
+
+class RequestFrames(Resource):
+
+    request_id = 0
+
+    def post(self):
+        try: 
+            # TODO: Perform delete operation in a background thread
+            delete_old_video_files()
+
+            RequestFrames.request_id = RequestFrames.request_id + 1
+            params = request.get_json()
+
+            query = create_query(params)
+            #query = 'SELECT id,data FROM MyVideo WHERE id < 5;'
+            query_list = [query]
+            frames = asyncio.run(get_frames(query_list))
+            print("calling create video")
+            video_name = generate_video_name(RequestFrames.request_id)        
+            create_video_from_frames(frames._batch, video_name)
+            return jsonify({"name": video_name})
+
+            #name = video_name + ".mp4"
+            #video_path = DATASET_DIR / name
+            #return send_file(video_path)
+        except Exception as e:
+            return str(e)
+
+
+class LoadFileIntoEva(Resource):
+
+    def get(self, video_name):
+        try:
+            _video_name = video_name + ".mp4"
+            video_path = DATASET_DIR / _video_name
+            query = create_load_query(video_path, video_name)
+
+            query_list = [query]
+            frames = asyncio.run(get_frames(query_list))
+            print("Calling load video into database.")
+            print("status: ", frames._status)
+            return jsonify({"status": frames._status})
+        except Exception as e:
+            return str(e)
+
+
+async def get_frames(query_list):
+    hostname = EVA_HOST
+    port = EVA_PORT
+
+    connection = await connect_async(hostname, port)
+    cursor = connection.cursor()
+    for query in query_list:
+        print('Query: %s' % query)
+        await cursor.execute_async(query)
+        print("executed")
+        response = await cursor.fetch_one_async()
+        print("got response")
+    return response
+
+
+def create_load_query(file_name, table_name):
+    query = "LOAD DATA INFILE '{}' INTO {};".format(file_name, table_name)
+    #print(query)
+    return query
+
+
+def create_query(req):
+    #print(req)
+    query = "SELECT "
+    for s in req['select']:
+        query = query + s['text'] + ", "
+    query = query[:len(query) - 2] + " FROM " + req['from']
+    if req['where']:
+        query = query + " WHERE "
+        for s in req['where']:
+            query = query + s['text'] + " OR "
+        query = query[:len(query) - 4]
+    query = query + ";"
+    return query
+
+
+def generate_video_name(num: int):
+    n = random.randrange(0,100000,1)
+    name = "result" + str(num) + "_" + str(n)
+    return name
+
+
+api.add_resource(RequestFrames, '/api/queryeva')
+api.add_resource(SendVideo, '/api/send_video/<string:video_name>')
+api.add_resource(SendName, '/api/send_name')
+api.add_resource(LoadFileIntoEva, '/api/load_file/<string:video_name>')
+
+if __name__ == '__main__':
+    app.run(host=FLASK_HOST, port=FLASK_PORT)

demo/requirements.txt

@@ -0,0 +1,6 @@
+Flask==1.1.2
+Flask-RESTful==0.3.8
+moviepy==1.0.3
+numpy==1.19.5
+pandas
+opencv-contrib-python

demo/run_before_startup.py

@@ -0,0 +1,23 @@
+import asyncio
+import sys
+
+from typing import List
+from src.db_api import connect, connect_async
+from config import EVA_HOST, EVA_PORT
+async def run_async(query: List[str]):
+    hostname = EVA_HOST
+    port = EVA_PORT
+
+    connection = await connect_async(hostname, port)
+    cursor = connection.cursor()
+    for onequery in query:
+        await cursor.execute_async(onequery)
+        response = await cursor.fetch_one_async()
+        print('Query: %s' % onequery)
+        print(response)
+
+if __name__ == '__main__':
+    asyncio.run(run_async(['LOAD DATA INFILE "data/mnist/mnist.mp4" INTO mnist;',
+                           'CREATE UDF MnistCNN INPUT (Frame_Array NDARRAY (3, 28, 28)) OUTPUT (label TEXT(2)) TYPE Classificatio IMPL "udfs/digit_recognition.py";']))
+
+

create_video.py → demo/utils.py

@@ -1,9 +1,11 @@
 import cv2
-import os
+import os, time, sys
 import numpy as np
 from src.response import Response
 from src.batch import Batch
 from moviepy.editor import *
+import pathlib
+from config import DATASET_DIR
 
 def create_video_from_frames(batch, name):
 
@@ -32,9 +34,20 @@ def create_video_from_frames(batch, name):
 
 def edit_video(name, duration):
     video = VideoFileClip(name).set_duration(duration)
-    if not os.path.exists('data'):
-        os.makedirs('data')
-    video.write_videofile("data/"+name, fps = 30)
+    filename = str(DATASET_DIR / name )
+    video.write_videofile(filename, fps = 30)
     video.close()
 
+
+def delete_old_video_files():
+    now = time.time()
+    try:
+        for f in os.listdir(DATASET_DIR):
+            f = os.path.join(DATASET_DIR, f)
+            if os.stat(f).st_mtime < now - 900:
+                if os.path.isfile(f):
+                    os.remove(f)
+    except:
+        if not os.path.exists(DATASET_DIR):
+            os.makedirs(DATASET_DIR)
 

requirements.txt

@@ -0,0 +1,78 @@
+absl-py==0.10.0
+aniso8601==9.0.1
+astunparse==1.6.3
+attrs==20.3.0
+cachetools==4.2.1
+certifi==2020.12.5
+chardet==4.0.0
+click==7.1.2
+coverage==5.5
+decorator==4.4.2
+dill==0.3.3
+flake8 @ file:///tmp/build/80754af9/flake8_1615834841867/work
+Flask==1.1.2
+Flask-RESTful==0.3.8
+flatbuffers==1.12
+future==0.18.2
+gast==0.3.3
+google-auth==1.27.1
+google-auth-oauthlib==0.4.3
+google-pasta==0.2.0
+googleapis-common-protos==1.53.0
+grpcio==1.32.0
+h5py==2.10.0
+idna==2.10
+imageio==2.9.0
+imageio-ffmpeg==0.4.3
+importlib-metadata @ file:///opt/concourse/worker/volumes/live/ae7e927e-57e2-4966-7b89-25dc6a80092c/volume/importlib-metadata_1615900433179/work
+importlib-resources==5.1.2
+iniconfig==1.1.1
+itsdangerous==1.1.0
+Jinja2==2.11.3
+Keras-Preprocessing==1.1.2
+Markdown==3.3.4
+MarkupSafe==1.1.1
+mccabe==0.6.1
+mkl-fft==1.3.0
+mkl-random==1.1.1
+mkl-service==2.3.0
+moviepy==1.0.3
+nose==1.3.7
+numpy==1.19.5
+oauthlib==3.1.0
+opencv-python==4.5.1.48
+opt-einsum==3.3.0
+packaging==20.9
+pandas==1.2.3
+Pillow==8.1.1
+pluggy==0.13.1
+proglog==0.1.9
+promise==2.3
+protobuf==3.15.4
+py==1.10.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle @ file:///home/ktietz/src/ci_mi/pycodestyle_1612807597675/work
+pyflakes @ file:///home/ktietz/src/ci_ipy2/pyflakes_1612551159640/work
+pyparsing==2.4.7
+pytest==6.2.2
+python-dateutil @ file:///home/ktietz/src/ci/python-dateutil_1611928101742/work
+pytz @ file:///tmp/build/80754af9/pytz_1612215392582/work
+requests==2.25.1
+requests-oauthlib==1.3.0
+rsa==4.7.2
+six @ file:///opt/concourse/worker/volumes/live/f983ba11-c9fe-4dff-7ce7-d89b95b09771/volume/six_1605205318156/work
+tensorboard==2.4.1
+tensorboard-plugin-wit==1.8.0
+tensorflow==2.4.1
+tensorflow-datasets==4.2.0
+tensorflow-estimator==2.4.0
+tensorflow-metadata==0.28.0
+termcolor==1.1.0
+toml==0.10.2
+tqdm==4.58.0
+typing-extensions @ file:///home/ktietz/src/ci_mi/typing_extensions_1612808209620/work
+urllib3==1.26.3
+Werkzeug==1.0.1
+wrapt==1.12.1
+zipp @ file:///tmp/build/80754af9/zipp_1615904174917/work

src/batch.py

@@ -16,7 +16,7 @@
 import numpy as np
 import pandas as pd
 
-from typing import List
+# from typing import List
 from pandas import DataFrame
 from src.logging_manager import LoggingManager, LoggingLevel