Merge pull request #12 from runboj/prefect

Prefect

.env.example

@@ -0,0 +1,19 @@
+DEFAULT_ALGORITHM_DESCRIPTION=/path/to/PCA_v1.0.0.json
+
+PREFECT_API_URL=http://prefect:4200/api
+FLOW_NAME="Parent flow/launch_parent_flow"
+TIMEZONE="US/Pacific"
+PREFECT_TAGS='["latent-space-explorer"]'
+
+# MLEx Content Registry API
+CONTENT_API_URL="http://content-api:8000/api/v0/models"
+
+TILED_API_KEY=<api key>
+
+READ_DIR=/path/to/read/data
+WRITE_DIR=/path/to/write/results
+
+# Slurm jobs
+PARTITIONS='["p1", "p2"]'
+RESERVATIONS='["r1", "r2"]'
+MAX_TIME="1:00:00"

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# 127 is width of the Github code viewer,
+# black default is 88 so this will only warn about comments >127
+max-line-length = 127
+# Ignore errors due to incompatibility with black
+#https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html
+extend-ignore = E203,E701

.gitignore

@@ -5,14 +5,13 @@ __pycache__/
 test.py
 
 # output dir
+results/
 data/output/
 data/upload/
 data/.file_manager_vars.pkl
 data/mlexchange_store/
 .DS_Store
 
-src/dash_component_editor.py
-
 # C extensions
 *.so
 

.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+default_language_version:
+    python: python3
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-ast
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: debug-statements
+  - repo: https://github.com/gitguardian/ggshield
+    rev: v1.25.0
+    hooks:
+      - id: ggshield
+        language_version: python3
+        stages: [commit]
+  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.2.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]

docker-compose.yml

@@ -1,6 +1,7 @@
 version: "3"
 
 services:
+
   front-end:
     restart: "unless-stopped"
     container_name: "latentxp"
@@ -9,18 +10,27 @@ services:
       dockerfile: "docker/Dockerfile"
     mem_limit: 2g
     environment:
-      DATA_DIR: "${PWD}/data/"
-      # USER: "$USER"
+      READ_DIR: "${READ_DIR}"
+      WRITE_DIR: "${WRITE_DIR}"
+      PREFECT_TAGS: "${PREFECT_TAGS}"
+      PREFECT_API_URL: '${PREFECT_API_URL}'
+      CONTENT_API_URL: '${CONTENT_API_URL}'
+      TILED_API_KEY: '${TILED_API_KEY}'
+      FLOW_NAME: '${FLOW_NAME}'
+      TIMEZONE: "${TIMEZONE}"
+      PARTITIONS: "${PARTITIONS}"
+      RESERVATIONS: "${RESERVATIONS}"
+      MAX_TIME: "${MAX_TIME}"
     volumes:
-      - ./data:/app/work/data
-      - ./src:/app/work/src
+      - $READ_DIR:/app/work/data
+      - $WRITE_DIR:/app/work/mlex_store
+      # - ./src:/app/work/src
+      - ../mlex_file_manager/file_manager:/app/work/src/file_manager
     ports:
-      - "8070:8070"
+      - "127.0.0.1:8070:8070"
     networks:
-      - computing_api_default  
+      mlex_mle_net:
 
 networks:
-  computing_api_default:
+  mlex_mle_net:
     external: true
-
-# env file: set up pwd

docker/Dockerfile

@@ -1,61 +1,17 @@
-# FROM python:3.9
-# LABEL maintainer="THE MLEXCHANGE TEAM"
-
-# RUN ls
-# COPY docker/requirements.txt requirements.txt
-
-# RUN apt-get update && apt-get install -y \
-#     build-essential \
-#     wget \
-#     python3-pip\
-#     ffmpeg\
-#     libsm6\
-#     libxext6 
-
-# RUN pip3 install --upgrade pip &&\
-#     pip3 install --timeout=2000 -r requirements.txt\
-#     pip install git+https://github.com/taxe10/mlex_file_manager
-
-# RUN git clone https://github.com/mlexchange/mlex_dash_component_editor
-
-# # EXPOSE 8000
-
-# WORKDIR /app/work
-# ENV HOME /app/work
-# COPY src src
-# # ENV PYTHONUNBUFFERED=1
-# RUN mv /mlex_dash_component_editor/src/dash_component_editor.py /app/work/src/dash_component_editor.py
-
-# CMD ["bash"]
-# #CMD python3 src/frontend.py
-# CMD sleep 3600
-
 FROM python:3.9
 LABEL maintainer="THE MLEXCHANGE TEAM"
 
 RUN ls
 COPY docker/requirements.txt requirements.txt
 
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    wget \
-    python3-pip\
-    ffmpeg\
-    libsm6\
-    libxext6 
-
 RUN pip3 install --upgrade pip &&\
-    pip3 install --timeout=2000 -r requirements.txt\
-    pip install git+https://github.com/taxe10/mlex_file_manager
+    pip3 install -r requirements.txt\
+    pip install git+https://github.com/mlexchange/mlex_file_manager\
+    pip install git+https://github.com/mlexchange/mlex_dash_component_editor
 
-RUN git clone https://github.com/mlexchange/mlex_dash_component_editor
 WORKDIR /app/work
 ENV HOME /app/work
 COPY src src
-RUN mv /mlex_dash_component_editor/src/dash_component_editor.py /app/work/src/dash_component_editor.py
 
 CMD ["bash"]
-#CMD sleep 3600 
 CMD python3 src/frontend.py
-
-

docker/requirements.txt

@@ -1,5 +1,4 @@
 dash==2.9.3
-# dash_component_editor==0.0.7
 dash-core-components==2.0.0
 dash-bootstrap-components==1.0.2
 dash-html-components==2.0.0
@@ -11,4 +10,7 @@ requests==2.26.0
 pyarrow==11.0.0
 diskcache==5.6.3
 pandas
-numpy
+numpy
+Pillow
+python-dotenv
+prefect-client==2.14.21