053 ecs stack (#62)

* runs from docker container

- store unpinned dependencies in requirements-bare.txt
- update sample data to supply missing field

* a setup.py is not needed here

* remove dokku-specific files

* build both docker images

* fix syntax for workflow_dispatch

* try running docker build workflow on push

* use include: in matrix definition

* temporarily comment out pipeline image

* try to set GIT_DESC

* use separate step to set GIT_DESC

* don't escape $ in GH action

* set GIT_DESC after checkout step

* fetch all repo history in checkout action

* tag image with GIT_DESC, add image cache

* rename github workflow for creating docker image for pipeline

* separate github workflow for application docker image

* decouple github actions for docker images

* increase verbosity of a logging message

* update names of image build actions

* extend gunicorn timeout to allow time to load data

* only add :latest tag to ya16sdb-app image

* remove old deployment for dash app

* define VERSION variable in Docker image

* update readme

* use a single guincorn worker

* Added version and link to repo in footer

* modified the footer

* boto3 is in requirements.txt so it is required

* moving data load into first callback

* I think this should be ==

* load data in genus options callback on first load

* Download date at top will load global data when app is initialized

* Added Loading... message at the top

* some extra container logging

* disable timeouts to see if that fixes things

* Using S3.Client.head_object to read timestamp

* Should be able to use default --timeout setting

* check if state is None before attempting to set_global_data()

---------

Co-authored-by: Noah Hoffman <[email protected]>

.github/workflows/docker-app.yml

@@ -0,0 +1,36 @@
+name: Build and push Docker image for Dash application
+
+on:
+  - push
+
+jobs:
+  build_app_image:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  #fetch all history, required for 'git describe'
+      - name: Set GIT_DESC
+        run: echo "GIT_DESC=$(git describe --tags)" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          cache-from: type=gha
+          context: ./dash
+          file: ./dash/Dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/ya16sdb-app:latest
+          build-args: |
+            "VERSION=${{ env.GIT_DESC }}"

.github/workflows/docker-pipeline.yml

@@ -0,0 +1,38 @@
+name: Build and push docker image for pipeline
+
+on:
+  push:
+    tags: [ '*.*.*' ]
+
+jobs:
+  build_pipeline_image:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+    steps:
+      # - name: Set RELEASE_VERSION
+      #   run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  #fetch all history, required for 'git describe'
+      - name: Set GIT_DESC
+        run: echo "GIT_DESC=$(git describe --tags)" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          cache-from: type=gha
+          context: .
+          file: "Dockerfile"
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/ya16sdb:latest
+            ghcr.io/${{ github.repository_owner }}/ya16sdb:${{ env.GIT_DESC }}

dash/Dockerfile

@@ -1,7 +1,18 @@
-FROM ubuntu:18.04
-MAINTAINER [email protected]
-RUN apt-get update && apt-get install --assume-yes python3 python3-pip
-COPY ./ /dash/
-RUN pip3 install --requirement /dash/requirements.txt
-CMD ["gunicorn", "app:app.server", "--config", "/dash/config.py"]
-EXPOSE 443
+FROM python:3.11-slim-bullseye
+
+RUN apt-get -y update && \
+    apt-get -y upgrade
+
+WORKDIR /opt/run
+COPY requirements.txt .
+RUN python3 -m venv /usr/local/share/venv
+RUN /usr/local/share/venv/bin/python -m pip install -r requirements.txt
+
+COPY app.py data.py entrypoint.sh filter_details.feather.gz .
+
+ARG VERSION
+ENV VERSION ${VERSION}
+
+EXPOSE 8000
+ENTRYPOINT ["/opt/run/entrypoint.sh"]
+CMD ["run"]

dash/README.rst

@@ -12,10 +12,10 @@ Authors: Noah Hoffman, Chris Rosenthal
 summary of features
 ===================
 
-Users can browse Bacteria and Archea species using the Genus and 
-Species dropdown menus.  Users can also search for sequences by 
-seqname, species taxonomy id, or species taxonomy name using the 
-search box or by url arguments.  Users can modify scatter plot markers 
+Users can browse Bacteria and Archea species using the Genus and
+Species dropdown menus.  Users can also search for sequences by
+seqname, species taxonomy id, or species taxonomy name using the
+search box or by url arguments.  Users can modify scatter plot markers
 by color and shape and use the Selection and Visibility drop down menus
 to increase the size or visibility of the markers respectively.  Both
 the Selection menu and plot selection tool will update the data table
@@ -25,7 +25,7 @@ standalone dev environment
 ==========================
 
 Note that the dev application uses data in
-``sq_order_tallies.feather.gz`` by default.
+``filter_details.feather.gz`` by default.
 
 Set up::
 
@@ -40,105 +40,18 @@ Run the app::
 
 The application will be served at ``http://127.0.0.1:8050``
 
-dokku on local Vagrant instance
-===============================
+Docker
+======
 
-before starting
----------------
+Building a Docker image::
 
-Be aware that certain network configurations including an active VPN
-can interfer with the setup process.  We recommend disconnecting 
-and disabling any VPN or manual network configuration before proceeding.
+  docker build --platform=linux/amd64 . -t ya16sdb --build-arg VERSION=$(git describe --tags)
 
-We'll mostly follow instructions here
-(https://github.com/dokku/dokku/blob/master/docs/getting-started/install/vagrant.md),
-with a few modifications.
+Run the application from Docker::
 
-After Vagrant is installed on your system (eg, ``brew install vagrant``),
-clone the dokku repository and create a Vagrant instance,
-for example (plan on getting a cup of coffee after the last command)::
+  docker run --platform=linux/amd64 --rm -p 8000:8000 ya16sdb
 
-  mkdir -p ~/src
-  cd ~/src
-  git clone https://github.com/dokku/dokku.git
-  cd dokku
-  vargrant up
+Or from an image hosted on GitHub Container Registry::
 
-Follow instructions at the url above to install your public key to the
-dokku instance. I used ``dokku.me`` as the host name in the dokku
-configuration form.
+  docker run --platform=linux/amd64 --rm -p 8000:8000 ghcr.io/nhoffman/ya16sdb-app:latest
 
-Add the following to ``~/.ssh/config``::
-
-  Host dokku.me
-    Port 22
-
-There appears to be two users available for login: ``root``, which
-provides an interactive prompt, and ``dokku`` which executes the
-``dokku`` command instead of a shell. The latter user is used to issue
-commands.
-
-Now you should be able to ssh into the dokku instance and create the application::
-
-  ssh [email protected] apps:create ya16sdb
-
-Confirm that the application was created::
-
-  % ssh [email protected] apps:list
-  =====> My Apps
-  ya16sdb
-
-Define a git remote on the dokku host::
-
-  git remote add dokku [email protected]:ya16sdb
-
-Finally, we can deploy the application. Dokku expects the application
-to be at the top level of the project repository, and because
-``/dash`` is a subdirectory, we will use ``git subtree`` to push only
-this directory (command must be executed in the root of the repo)::
-
-  cd ya16sdb
-  git subtree push --prefix dash dokku master
-
-Now you should be able to access the application at
-``http://ya16sdb.dokku.me``
-
-mounting data
--------------
-
-Details about the storage mounting plugin are here 
-(https://github.com/dokku/dokku/blob/master/docs/advanced-usage/persistent-storage.md).
-
-In summary, you will need to setup a folder with correct permissions on the
-dokku instance and transfer your data before mounting::
-
-  ssh [email protected]
-  mkdir /var/lib/dokku/data/storage/ya16sdb
-
-Exit and transfer your data file up::
-
-  scp file.feather [email protected]:/var/lib/dokku/data/storage/ya16sdb
-
-And, finally, mount::
-
-  ssh [email protected] storage:mount ya16sdb /var/lib/dokku/data/storage/ya16sdb:/storage
-
-Be aware there could be permission issues you may need to update with your
-data file depending on many factors. Applying a `chmod a+r` to the file should
-fix most issues.
-
-Let's Encrypt plugin
---------------------
-
-https://github.com/dokku/dokku-letsencrypt
-
-troubleshooting dokku
----------------------
-
-Within the dokku container, the application is served using
-``gunicorn``. Confirm that the configuration in ``Procfile`` for
-running ``gunicorn`` is working as expected like this::
-
-  exec $(sed 's/web: //' Procfile)
-
-This time the application is served at ``http://127.0.0.1:8000``

dash/app.py

@@ -7,14 +7,18 @@
 import os
 import pandas
 import urllib
+import logging
 
 from dash.dependencies import Input, State, Output
 
+log = logging.getLogger(__name__)
+
 COLORS = ['blue', 'red', 'black', 'yellow',
           'gray', 'green', 'violet', 'silver']
 DEFAULT_COLOR = 'is_out'
 DEFAULT_SHAPE = 'confidence'
 DEFAULT_GENUS = '1350'  # Enterococcus
+DEFAULT_GENUS_NAME = 'Enterococcus'
 DEFAULT_SPECIES = '1351'  # Enterococcus faecalis
 DEFAULT_Y = 'y'
 DEFAULT_X = 'x'
@@ -79,9 +83,6 @@ def write_layout():
     This is done using a function so Dash will not cache it allowing us to
     update the underlining data.
     '''
-    set_global_data()
-    genus_opts = tax[['genus', 'genus_name']].drop_duplicates().values
-    genus_opts = [{'label': gn, 'value': gi} for gi, gn in genus_opts]
     axes = ['confidence', 'dist_pct', 'x', 'y',
             'type_classification', 'rank_order']
     return dash.html.Div(
@@ -90,9 +91,8 @@ def write_layout():
             dash.dcc.Store(id='state'),
             dash.dcc.Location(id='url', refresh=False),
             dash.dcc.Markdown(
-                children=[
-                    str(df['download_date'].max().strftime('%A, %B %d, %Y'))
-                ],
+                children='Loading...',
+                id='download-date',
                 style={
                     'textAlign': 'right',
                     'fontStyle': 'italic',
@@ -116,7 +116,6 @@ def write_layout():
                 children=[
                     dash.dcc.Dropdown(
                         id='genus-column',
-                        options=genus_opts,
                         clearable=False)],
                 style={
                     'display': 'inline-block',
@@ -299,7 +298,21 @@ def write_layout():
                     'margin': 5,
                     'overflowY': 'scroll',
                     'padding': 10,
-                    'width': '97%'})])
+                    'width': '97%'}),
+            dash.html.A(
+                os.environ.get(
+                    'VERSION',
+                    'github.com/nhoffman/ya16sdb/dash'),
+                href='https://github.com/nhoffman/'
+                     'ya16sdb/pkgs/container/ya16sdb-app',
+                style={
+                    'display': 'inline-block',
+                    'textAlign': 'center',
+                    'verticalAlign': 'middle',
+                    'width': '100%',
+                    'textDecoration': 'none'},
+                target='_blank')
+                ])
 
 
 app.layout = write_layout
@@ -367,13 +380,29 @@ def update_xaxis_value(search):
 #     return '?' + urllib.parse.urlencode({'species_to_id': value})
 
 
+@app.callback(Output('download-date', 'children'), Input('state', 'data'))
+def update_download_date(state):
+    if state is None:
+        set_global_data()
+    return str(df['download_date'].max().strftime('%A, %B %d, %Y'))
+
+
+@app.callback(Output('genus-column', 'options'), Input('state', 'data'))
+def update_genus_options(state):
+    opts = tax[['genus', 'genus_name']].drop_duplicates().values
+    opts = [{'label': gn, 'value': gi} for gi, gn in opts]
+    return opts
+
+
 @app.callback(
     Output('genus-column', 'value'),
     [Input('url', 'search'),
      Input('submit-button', 'n_clicks')],
     [State('text-input', 'value'),
      State('state', 'data')])
 def update_genus_value(search, n_clicks, text, state):
+    if state is None:
+        set_global_data()
     request, data = parse_search_input(df, state, search, n_clicks, text)
     if request is None:
         value = DEFAULT_GENUS