From 6cc80d891c0b3b10f7a515a4ade827ec538643ac Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 10:01:05 -0400 Subject: [PATCH 01/11] Update VM --- README.md | 2 +- cytominer_ami.json | 4 ++-- efs.sh | 2 +- init.sh | 1 + python.sh | 39 +++++++++++++++++++++++++++++++----- r.sh | 24 ++++++++++++++++------- s3.sh | 30 ++++++++++++++++++++++++---- tools.sh | 49 ++++++++++++++++++++++++++++++++++++++++++---- 8 files changed, 127 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index f578f09..9ab676f 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ aws configure ### Edit EFS and S3 configurations -Edit `EFS_ID` in `efs.sh`, and `S3_ROLE` and `BUCKET_ID` appropriately. +Edit `EFS_ID` in `efs.sh`, and `S3_ROLE` and `BUCKET_ID` is `s3.sh` appropriately. ### Validate and build diff --git a/cytominer_ami.json b/cytominer_ami.json index dfe4031..865d080 100644 --- a/cytominer_ami.json +++ b/cytominer_ami.json @@ -8,7 +8,7 @@ }, "builders": [{ "access_key": "{{user `aws_access_key`}}", - "ami_name": "cytomining/images/hvm-ssd/cytominer-ubuntu-trusty-14.04-amd64-server-{{timestamp}}", + "ami_name": "cytomining/images/hvm-ssd/cytominer-ubuntu-xenial-18.04-amd64-server-{{timestamp}}", "associate_public_ip_address" : true, "instance_type": "c4.xlarge", "region": "us-east-1", @@ -16,7 +16,7 @@ "security_group_ids": "{{user `security_group_ids`}}", "source_ami_filter": { "filters": { - "name": "ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-*", + "name": "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*", "root-device-type": "ebs", "virtualization-type": "hvm" }, diff --git a/efs.sh b/efs.sh index b50c6da..942dc03 100644 --- a/efs.sh +++ b/efs.sh @@ -1,6 +1,6 @@ #!/bin/bash -EFS_ID="fs-d1824199" +EFS_ID="fs-3609f37f" #----------------------------- # EFS diff --git a/init.sh b/init.sh index ccc4375..b0f3c5f 100644 --- a/init.sh +++ b/init.sh @@ -3,4 +3,5 @@ sleep 90 sudo apt-get update + sudo apt-get upgrade -y diff --git a/python.sh b/python.sh index 8a2bb87..ed76eea 100644 --- a/python.sh +++ b/python.sh @@ -7,27 +7,48 @@ sudo easy_install pip mkdir -p ~/work/software/archives + cd ~/work/software/archives + git clone https://github.com/yyuu/pyenv.git ~/.pyenv + echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc + echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc + echo 'eval "$(pyenv init -)"' >> ~/.bashrc + export PYENV_ROOT="$HOME/.pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + eval "$(pyenv init -)" -pyenv install 3.5.1 + +pyenv install 3.8.2 + pyenv install 2.7.12 -pyenv shell 3.5.1 + +pyenv shell 3.8.2 + pip install --upgrade pip + pip install --upgrade setuptools + pip install IPython pyyaml + +# this is for pe2loaddata + pyenv shell 2.7.12 + pip install --upgrade pip + pip install --upgrade setuptools + pip install IPython==5.0 + pip install pyyaml # DCP and cytominer are now configured within this script because @@ -37,10 +58,15 @@ pip install pyyaml # DCP #----------------------------- cd + git clone https://github.com/CellProfiler/Distributed-CellProfiler.git + cd Distributed-CellProfiler/files -pyenv local 2.7.12 -pyenv shell 2.7.12 + +pyenv local 3.8.2 + +pyenv shell 3.8.2 + pip install -r requirements.txt #optional packages to be able to run ancillary DCP functions @@ -49,9 +75,12 @@ sudo apt-get install -y cloud-image-utils parallel python-pandas #----------------------------- # cytominer #----------------------------- -pyenv shell 3.5.1 +pyenv shell 3.8.2 + pip install --upgrade pip + pip install git+git://github.com/cytomining/cytominer-database.git + pip install IPython Rscript -e 'devtools::install_github("cytomining/cytominer", dependencies=TRUE)' diff --git a/r.sh b/r.sh index 18d0368..b88a13e 100644 --- a/r.sh +++ b/r.sh @@ -2,13 +2,23 @@ #----------------------------- # R -# https://www.digitalocean.com/community/tutorials/how-to-set-up-r-on-ubuntu-14-04 +# https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04-quickstart #----------------------------- -sudo sh -c 'echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list' -gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9 -gpg -a --export E084DAB9 | sudo apt-key add - -sudo apt-get update -sudo apt-get -y install r-base + +sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 + +sudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' + +sudo apt -y update + +sudo apt -y install r-base=3.6.3-1bionic + mkdir -p ~/R/library + echo "R_LIBS=~/R/library" > ~/.Renviron -Rscript -e 'install.packages(c("devtools", "docopt", "dplyr", "feather", "ggplot2", "knitr", "magrittr", "readr", "rmarkdown", "RSQLite", "stringr", "testthat", "tidyr", "tidyverse"), Ncpus=4, repos=c("http://cran.us.r-project.org", "https://cran.cnr.berkeley.edu/", "https://cran.revolutionanalytics.com/"), lib="~/R/library/")' + +# Here are options for installing specific versions +# - use conda to specify +# - to create a cytominer-vm dummy package on github and then install that + +Rscript -e 'install.packages(c("devtools", "docopt", "dplyr", "feather", "ggplot2", "knitr", "magrittr", "readr", "rmarkdown", "RSQLite", "stringr", "testthat", "tidyr", "tidyverse"), Ncpus=4, repos=c("http://cran.us.r-project.org", "https://cran.rstudio.com/", "https://cran.revolutionanalytics.com/"), lib="~/R/library/")' diff --git a/s3.sh b/s3.sh index bb4b7d8..5b44dbb 100644 --- a/s3.sh +++ b/s3.sh @@ -1,25 +1,47 @@ #!/bin/bash # IAM role that allows access to S3 bucket -S3_ROLE="ec2-iam-role-s3" +S3_ROLE="s3-imaging-platform-role" # Name of S3 bucket to be mounted -BUCKET_ID="imaging-platform-dev" +BUCKET_ID="imaging-platform" #----------------------------- # s3fs # https://github.com/s3fs-fuse/s3fs-fuse #----------------------------- -sudo apt-get install -y automake autotools-dev g++ git libcurl4-gnutls-dev libfuse-dev libssl-dev libxml2-dev make pkg-config +sudo apt install -y \ + automake \ + autotools-dev \ + g++ \ + git \ + libcurl4-gnutls-dev \ + libfuse-dev \ + libssl-dev \ + libxml2-dev \ + make \ + pkg-config + mkdir -p ~/work/software/archives + cd ~/work/software/archives + git clone https://github.com/s3fs-fuse/s3fs-fuse.git + cd s3fs-fuse + ./autogen.sh -./configure + +./configure --prefix=/usr --with-openssl + make + sudo make install + cd ~ + mkdir ~/bucket + echo "${BUCKET_ID} /home/ubuntu/bucket fuse.s3fs _netdev,allow_other,iam_role=${S3_ROLE},uid=1000,gid=1000,umask=0022 0 0" | sudo tee --append /etc/fstab + sudo mount -a diff --git a/tools.sh b/tools.sh index dc53b34..b28b4a6 100644 --- a/tools.sh +++ b/tools.sh @@ -3,8 +3,49 @@ #----------------------------- # Tools #----------------------------- -sudo apt-get install -y build-essential bzip2 checkinstall emacs htop jq mysql-client-core-5.5 pandoc parallel postgresql postgresql-client-common sqlite sqlite3 tree zip libbz2-dev -sudo apt-get install -y libffi-dev libfftw3-dev libldap2-dev libpq-dev libreadline6 libreadline6-dev libsasl2-dev libsqlite3-dev libssh2-1-dev libtiff5 libtiff5-dev libxslt1-dev -sudo apt-get install -y python python-dev python-pip python-pip python-setuptools -sudo pip install awscli +# Do this first +# See https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai +sudo apt install -y \ + tzdata + +sudo apt install -y \ + build-essential \ + bzip2 \ + checkinstall \ + emacs \ + htop \ + jq \ + mysql-client-core-5.7 \ + pandoc \ + parallel \ + postgresql \ + postgresql-client-common \ + sqlite \ + sqlite3 \ + tree \ + zip + +sudo apt install -y \ + libbz2-dev \ + libffi-dev \ + libfftw3-dev \ + libldap2-dev \ + libpq-dev \ + libreadline7 \ + libreadline-dev \ + libsasl2-dev \ + libsqlite3-dev \ + libssh2-1-dev \ + libtiff5 \ + libtiff5-dev \ + libxslt1-dev + +sudo apt install -y \ + python \ + python-dev \ + python-pip \ + python-setuptools + +# removed sudo below +pip install awscli From 07a18669eebc8882b7c1b77e61a3db3e509cedab Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 10:55:49 -0400 Subject: [PATCH 02/11] efs --- efs.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/efs.sh b/efs.sh index 942dc03..432a2f2 100644 --- a/efs.sh +++ b/efs.sh @@ -7,7 +7,10 @@ EFS_ID="fs-3609f37f" # https://console.aws.amazon.com/efs/home?region=us-east-1#/filesystems #----------------------------- -sudo apt-get install -y nfs-common +sudo apt install -y nfs-common + sudo mkdir ~/efs + sudo chown ubuntu ~/efs/ + echo "sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone).${EFS_ID}.efs.us-east-1.amazonaws.com:/ ~/efs" >> ~/.bashrc From cc02ae050faed509cd22207ff9aaa7be919db801 Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 16:44:55 -0400 Subject: [PATCH 03/11] bionic-18.04 --- cytominer_ami.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cytominer_ami.json b/cytominer_ami.json index 865d080..dd05d98 100644 --- a/cytominer_ami.json +++ b/cytominer_ami.json @@ -8,7 +8,7 @@ }, "builders": [{ "access_key": "{{user `aws_access_key`}}", - "ami_name": "cytomining/images/hvm-ssd/cytominer-ubuntu-xenial-18.04-amd64-server-{{timestamp}}", + "ami_name": "cytomining/images/hvm-ssd/cytominer-ubuntu-bionic-18.04-amd64-server-{{timestamp}}", "associate_public_ip_address" : true, "instance_type": "c4.xlarge", "region": "us-east-1", @@ -16,7 +16,7 @@ "security_group_ids": "{{user `security_group_ids`}}", "source_ami_filter": { "filters": { - "name": "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*", + "name": "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*", "root-device-type": "ebs", "virtualization-type": "hvm" }, From 14170cb1b19cada5ac077bd23f315b6b8d28afad Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 16:45:46 -0400 Subject: [PATCH 04/11] deps are installed via cytominer and cytotools --- r.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/r.sh b/r.sh index b88a13e..990a2c4 100644 --- a/r.sh +++ b/r.sh @@ -17,8 +17,13 @@ mkdir -p ~/R/library echo "R_LIBS=~/R/library" > ~/.Renviron -# Here are options for installing specific versions -# - use conda to specify -# - to create a cytominer-vm dummy package on github and then install that +Rscript -e 'install.packages(c("devtools"), repos = "http://cran.us.r-project.org", Ncpus=4)' + +Rscript -e 'devtools::install_version("cytominer", version = "0.2.1", repos = "http://cran.us.r-project.org", dependencies=TRUE, Ncpus=4)' + +# cytotools doesn't current have tidyr as a dep but will eventually +# (but cytominer_script needs it) +Rscript -e 'install.packages(c("tidyr"), repos = "http://cran.us.r-project.org", Ncpus=4)' + +Rscript -e 'devtools::install_github("cytomining/cytotools", ref = "2fc846c", dependencies=TRUE)' -Rscript -e 'install.packages(c("devtools", "docopt", "dplyr", "feather", "ggplot2", "knitr", "magrittr", "readr", "rmarkdown", "RSQLite", "stringr", "testthat", "tidyr", "tidyverse"), Ncpus=4, repos=c("http://cran.us.r-project.org", "https://cran.rstudio.com/", "https://cran.revolutionanalytics.com/"), lib="~/R/library/")' From 0a55da2a843b14871840a652d06747f29d5313de Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 17:26:09 -0400 Subject: [PATCH 05/11] cleanup python --- python.sh | 84 +++++++++++++++++++++++++++++++++++-------------------- tools.sh | 8 ------ 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/python.sh b/python.sh index ed76eea..53b16a5 100644 --- a/python.sh +++ b/python.sh @@ -2,9 +2,21 @@ #----------------------------- # Python +#----------------------------- + +sudo apt install -y \ + python \ + python-dev \ + python-pip \ + python-setuptools + +#----------------------------- +# pyenv # https://github.com/yyuu/pyenv #----------------------------- -sudo easy_install pip + +# TODO: remove this commented-out line +# sudo easy_install pip mkdir -p ~/work/software/archives @@ -12,11 +24,15 @@ cd ~/work/software/archives git clone https://github.com/yyuu/pyenv.git ~/.pyenv -echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc +echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bash_profile + +echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bash_profile -echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc +echo -e 'if command -v pyenv 1>/dev/null 2>&1; then\n eval "$(pyenv init -)"\nfi' >> ~/.bash_profile -echo 'eval "$(pyenv init -)"' >> ~/.bashrc +# In a non-interactive install, you can't do this: +# exec "$SHELL" +# and so you need to mock this: export PYENV_ROOT="$HOME/.pyenv" @@ -24,41 +40,52 @@ export PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init -)" - pyenv install 3.8.2 -pyenv install 2.7.12 - - pyenv shell 3.8.2 pip install --upgrade pip pip install --upgrade setuptools -pip install IPython pyyaml +#----------------------------- +# pe2loaddata +#----------------------------- +# This section will go once pe2loaddata is packaged -# this is for pe2loaddata +pip install IPython pyyaml -pyenv shell 2.7.12 +#----------------------------- +# NOTE: +# DCP and cytominer are configured within this script because +# pyenv wouldn't load when they were moved into their own scripts +#----------------------------- -pip install --upgrade pip +#----------------------------- +# DCP Control Node +# https://github.com/CellProfiler/Distributed-CellProfiler/wiki/Before-you-get-started%3A-setting-up +#----------------------------- -pip install --upgrade setuptools +cd -pip install IPython==5.0 +# This is done in tools.sh +# sudo apt install build-essential checkinstall -pip install pyyaml +# This is done above +# sudo apt install python python-dev python-setuptools python-dev build-essential -# DCP and cytominer are now configured within this script because -# pyenv wouldn't load when they were moved into their own scripts +# I doubt this is needed +# sudo easy_install pip -#----------------------------- -# DCP -#----------------------------- -cd +# This is done above +# pip install --upgrade setuptools + +sudo apt -y install \ + cloud-image-utils \ + fabric +# TODO Maybe specify git hash? git clone https://github.com/CellProfiler/Distributed-CellProfiler.git cd Distributed-CellProfiler/files @@ -69,18 +96,15 @@ pyenv shell 3.8.2 pip install -r requirements.txt -#optional packages to be able to run ancillary DCP functions -sudo apt-get install -y cloud-image-utils parallel python-pandas +# TODO: is this required? +# or is this sufficient: pip install awscli +pip install awscli --ignore-installed six + +pip install --upgrade awscli #----------------------------- # cytominer #----------------------------- -pyenv shell 3.8.2 - -pip install --upgrade pip - -pip install git+git://github.com/cytomining/cytominer-database.git -pip install IPython +pip install cytominer-database==0.3.3 -Rscript -e 'devtools::install_github("cytomining/cytominer", dependencies=TRUE)' diff --git a/tools.sh b/tools.sh index b28b4a6..b09cf74 100644 --- a/tools.sh +++ b/tools.sh @@ -41,11 +41,3 @@ sudo apt install -y \ libtiff5-dev \ libxslt1-dev -sudo apt install -y \ - python \ - python-dev \ - python-pip \ - python-setuptools - -# removed sudo below -pip install awscli From 629dc71c5cd0a4e7da38ac16c911e360196ae538 Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 17:30:57 -0400 Subject: [PATCH 06/11] wheels --- python.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python.sh b/python.sh index 53b16a5..06008a4 100644 --- a/python.sh +++ b/python.sh @@ -44,9 +44,10 @@ pyenv install 3.8.2 pyenv shell 3.8.2 -pip install --upgrade pip - -pip install --upgrade setuptools +pip install --upgrade \ + pip \ + setuptools \ + wheel #----------------------------- # pe2loaddata @@ -67,8 +68,6 @@ pip install IPython pyyaml # https://github.com/CellProfiler/Distributed-CellProfiler/wiki/Before-you-get-started%3A-setting-up #----------------------------- -cd - # This is done in tools.sh # sudo apt install build-essential checkinstall @@ -85,6 +84,8 @@ sudo apt -y install \ cloud-image-utils \ fabric +cd ~ + # TODO Maybe specify git hash? git clone https://github.com/CellProfiler/Distributed-CellProfiler.git From 7486d551c2a3987ab7b4fd2382df2ba6dbead726 Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 17:32:06 -0400 Subject: [PATCH 07/11] apt-get -> apt --- docker.sh | 18 +++++++++++++----- python.sh | 4 +++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docker.sh b/docker.sh index ea9e0f7..fb60cb0 100644 --- a/docker.sh +++ b/docker.sh @@ -4,14 +4,22 @@ # docker # https://docs.docker.com/engine/installation/linux/ubuntulinux/ #----------------------------- -sudo apt-get install -y apt-transport-https ca-certificates +sudo apt install -y apt-transport-https ca-certificates + sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D + sudo sh -c 'echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" > /etc/apt/sources.list.d/docker.list' -sudo apt-get update -sudo apt-get purge lxc-docker + +sudo apt update + +sudo apt purge lxc-docker + apt-cache policy docker-engine # Verify that right repository. -sudo apt-get install -y linux-image-extra-$(uname -r) -sudo apt-get install -y docker-engine + +sudo apt install -y linux-image-extra-$(uname -r) + +sudo apt install -y docker-engine sudo groupadd docker + sudo usermod -aG docker ubuntu diff --git a/python.sh b/python.sh index 06008a4..3880ca1 100644 --- a/python.sh +++ b/python.sh @@ -55,7 +55,9 @@ pip install --upgrade \ # This section will go once pe2loaddata is packaged -pip install IPython pyyaml +pip install --upgrade \ + IPython \ + pyyaml #----------------------------- # NOTE: From ea1aed9483d2efb422f481c6e0aac104b2423f07 Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 17:43:50 -0400 Subject: [PATCH 08/11] docker update --- docker.sh | 33 ++++++++++++++++++++++++--------- python.sh | 7 +++---- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/docker.sh b/docker.sh index fb60cb0..f387104 100644 --- a/docker.sh +++ b/docker.sh @@ -2,23 +2,38 @@ #----------------------------- # docker -# https://docs.docker.com/engine/installation/linux/ubuntulinux/ +# https://docs.docker.com/engine/install/ubuntu/#prerequisites #----------------------------- -sudo apt install -y apt-transport-https ca-certificates -sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D - -sudo sh -c 'echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" > /etc/apt/sources.list.d/docker.list' +sudo apt remove -y \ + docker \ + docker-engine \ + docker.io \ + containerd \ + runc sudo apt update -sudo apt purge lxc-docker +sudo apt install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -apt-cache policy docker-engine # Verify that right repository. +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" -sudo apt install -y linux-image-extra-$(uname -r) +sudo apt update -sudo apt install -y docker-engine +sudo apt install -y \ + docker-ce \ + docker-ce-cli \ + containerd.io sudo groupadd docker diff --git a/python.sh b/python.sh index 3880ca1..3da1add 100644 --- a/python.sh +++ b/python.sh @@ -72,16 +72,15 @@ pip install --upgrade \ # This is done in tools.sh # sudo apt install build-essential checkinstall +# sudo apt-get install parallel # This is done above # sudo apt install python python-dev python-setuptools python-dev build-essential +# pip install --upgrade setuptools -# I doubt this is needed +# TODO: I doubt this is needed any longer # sudo easy_install pip -# This is done above -# pip install --upgrade setuptools - sudo apt -y install \ cloud-image-utils \ fabric From 7b78fd956fbe76d7cad3121c69dd7f269ae478c2 Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 17:47:50 -0400 Subject: [PATCH 09/11] new line --- tools.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools.sh b/tools.sh index b09cf74..0429bba 100644 --- a/tools.sh +++ b/tools.sh @@ -40,4 +40,3 @@ sudo apt install -y \ libtiff5 \ libtiff5-dev \ libxslt1-dev - From cbc29318fa4b4850384030e30e1a53a1facab47b Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 22:59:41 -0400 Subject: [PATCH 10/11] cleanup readme --- README.md | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9ab676f..541d6f2 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,17 @@ -# VM for profiling -This packer configuration creates a VM provisioned with tools need for profiling analysis. +# VM for profiling +This packer configuration creates a VM provisioned with tools need for +profiling analysis. - Python and R packages -- cytominer and dependencies +- cytominer, cytotools, cytominer-database, and dependencies - Distributed-CellProfiler and dependencies - Docker +- Mount S3 (s3fs) so that CellProfiler can access files +- Mount EFS -In addition, it mounts an S3 bucket and an Amazon Elastic File System (EFS). +In addition, it mounts +- an S3 bucket so that CellProfiler can access files +- an NFS (Amazon Elastic File System (EFS)) ## Install packer @@ -29,20 +34,31 @@ aws configure ### Edit EFS and S3 configurations -Edit `EFS_ID` in `efs.sh`, and `S3_ROLE` and `BUCKET_ID` is `s3.sh` appropriately. +Edit `EFS_ID` in `efs.sh`, and `S3_ROLE` and `BUCKET_ID` is `s3.sh` +appropriately. ### Validate and build -Note that although instance specifics are included in `cytominer_ami.json` (`c4.large`), and an instance will be created, the instance only exists to create the AMI and will be destroyed upon completion, `delete_on_termination: true`, leaving only the newly created AMI. +Note that although instance specifics are included in `cytominer_ami.json` +(`c4.large`), and an instance will be created, the instance only exists to +create the AMI and will be destroyed upon completion, +`delete_on_termination: true`, leaving only the newly created AMI. ``` packer validate cytominer_ami.json ``` -Specify `security_group_ids`, `subnet_id` and `vpc_id` on which the builder should be run (get these from AWS console) +Specify `security_group_ids`, `subnet_id` and `vpc_id` on which the builder +should be run (get these from AWS console) ``` packer build -var 'security_group_ids=sg-NNNNNNNN,sg-NNNNNNNN' -var 'subnet_id=subnet-NNNNNNNN' -var 'vpc_id=vpc-NNNNNNNN' cytominer_ami.json ``` +The security groups should be such that together they allow +- SSH access on port 22 +- NFS access on part 2049 (for EFS) + + + From 78e3ac33279b7e9616017bfeb3746480e126f5dc Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Sun, 3 May 2020 23:00:54 -0400 Subject: [PATCH 11/11] revert to using apt-get --- docker.sh | 10 +++++----- efs.sh | 2 +- python.sh | 18 ++++++------------ r.sh | 5 +++-- s3.sh | 4 ++-- tools.sh | 6 +++--- 6 files changed, 20 insertions(+), 25 deletions(-) diff --git a/docker.sh b/docker.sh index f387104..6d1c8c1 100644 --- a/docker.sh +++ b/docker.sh @@ -5,16 +5,16 @@ # https://docs.docker.com/engine/install/ubuntu/#prerequisites #----------------------------- -sudo apt remove -y \ +sudo apt-get remove -y \ docker \ docker-engine \ docker.io \ containerd \ runc -sudo apt update +sudo apt-get update -sudo apt install -y \ +sudo apt-get install -y \ apt-transport-https \ ca-certificates \ curl \ @@ -28,9 +28,9 @@ sudo add-apt-repository \ $(lsb_release -cs) \ stable" -sudo apt update +sudo apt-get update -sudo apt install -y \ +sudo apt-get install -y \ docker-ce \ docker-ce-cli \ containerd.io diff --git a/efs.sh b/efs.sh index 432a2f2..5e70a10 100644 --- a/efs.sh +++ b/efs.sh @@ -7,7 +7,7 @@ EFS_ID="fs-3609f37f" # https://console.aws.amazon.com/efs/home?region=us-east-1#/filesystems #----------------------------- -sudo apt install -y nfs-common +sudo apt-get install -y nfs-common sudo mkdir ~/efs diff --git a/python.sh b/python.sh index 3da1add..1274e18 100644 --- a/python.sh +++ b/python.sh @@ -4,7 +4,7 @@ # Python #----------------------------- -sudo apt install -y \ +sudo apt-get install -y \ python \ python-dev \ python-pip \ @@ -15,9 +15,6 @@ sudo apt install -y \ # https://github.com/yyuu/pyenv #----------------------------- -# TODO: remove this commented-out line -# sudo easy_install pip - mkdir -p ~/work/software/archives cd ~/work/software/archives @@ -78,16 +75,16 @@ pip install --upgrade \ # sudo apt install python python-dev python-setuptools python-dev build-essential # pip install --upgrade setuptools -# TODO: I doubt this is needed any longer +# This is no longer needed +# See https://pip.pypa.io/en/stable/installing/ # sudo easy_install pip -sudo apt -y install \ +sudo apt-get install -y \ cloud-image-utils \ fabric cd ~ -# TODO Maybe specify git hash? git clone https://github.com/CellProfiler/Distributed-CellProfiler.git cd Distributed-CellProfiler/files @@ -98,11 +95,8 @@ pyenv shell 3.8.2 pip install -r requirements.txt -# TODO: is this required? -# or is this sufficient: pip install awscli -pip install awscli --ignore-installed six - -pip install --upgrade awscli +pip install --upgrade \ + awscli #----------------------------- # cytominer diff --git a/r.sh b/r.sh index 990a2c4..f11deb6 100644 --- a/r.sh +++ b/r.sh @@ -9,9 +9,10 @@ sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD sudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' -sudo apt -y update +sudo apt-get update -y -sudo apt -y install r-base=3.6.3-1bionic +sudo apt-get install -y \ + r-base=3.6.3-1bionic mkdir -p ~/R/library diff --git a/s3.sh b/s3.sh index 5b44dbb..4f0aa96 100644 --- a/s3.sh +++ b/s3.sh @@ -10,7 +10,7 @@ BUCKET_ID="imaging-platform" # s3fs # https://github.com/s3fs-fuse/s3fs-fuse #----------------------------- -sudo apt install -y \ +sudo apt-get install -y \ automake \ autotools-dev \ g++ \ @@ -32,7 +32,7 @@ cd s3fs-fuse ./autogen.sh -./configure --prefix=/usr --with-openssl +./configure --prefix=/usr --with-openssl make diff --git a/tools.sh b/tools.sh index 0429bba..ffe32c3 100644 --- a/tools.sh +++ b/tools.sh @@ -6,10 +6,10 @@ # Do this first # See https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai -sudo apt install -y \ +sudo apt-get install -y \ tzdata -sudo apt install -y \ +sudo apt-get install -y \ build-essential \ bzip2 \ checkinstall \ @@ -26,7 +26,7 @@ sudo apt install -y \ tree \ zip -sudo apt install -y \ +sudo apt-get install -y \ libbz2-dev \ libffi-dev \ libfftw3-dev \