From 7a1c458fce60e68cb41a1ae180b891b6a9cffc07 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 29 Feb 2024 22:07:42 +0000 Subject: [PATCH] Add changes for 55ade0afef61720729a49f2b9487ccb7c33909ad --- .../asset_config_files.md.txt} | 0 .../asset_server_config.md.txt} | 0 .../asset_server_setup.md.txt} | 0 .../hybrid_asset_server_setup.md.txt} | 0 _sources/aws/AWS Infrastructure Notes.md.txt | 0 _sources/aws/Client Migration Notes.md.txt | 18 -- _sources/aws/aws_infrastructure_notes.md.txt | 3 + ... => aws_specify_asset_server_setup.md.txt} | 9 +- ....md.txt => aws_specify_cloud_setup.md.txt} | 74 ++++--- _sources/aws/client_migration_notes.md.txt | 33 +++ ...p.md.txt => ec2_rds_specify7_setup.md.txt} | 95 ++++++-- ...xt => specify_cloud_graviton_setup.md.txt} | 0 .../dev_process/specify_processes.rst.txt | 91 ++++++++ _sources/index.rst.txt | 32 ++- .../add_new_instance_to_specify_cloud.md.txt | 4 +- .../kuit_notes.md.txt} | 9 +- .../specify7_ecs_most-in-one.md.txt} | 7 +- .../useful_bash_cmds.md.txt} | 29 +-- ...e_django_unit_test_debugging_notes.md.txt} | 2 +- _sources/security/fix_exposed_secret.rst.txt | 41 ++++ .../asset_config_files.html | 37 ++-- .../asset_server_config.html | 29 ++- .../asset_server_setup.html | 30 ++- .../hybrid_asset_server_setup.html | 29 ++- aws/AWS Infrastructure Notes.html | 102 --------- aws/Client Migration Notes.html | 114 ---------- aws/aws_infrastructure_notes.html | 132 +++++++++++ ...ml => aws_specify_asset_server_setup.html} | 67 ++++-- ...loud.html => aws_specify_cloud_setup.html} | 180 +++++++++++---- aws/client_migration_notes.html | 155 +++++++++++++ ...Setup.html => ec2_rds_specify7_setup.html} | 170 +++++++++++---- ...html => specify_cloud_graviton_setup.html} | 38 +++- dev_process/specify_processes.html | 206 ++++++++++++++++++ genindex.html | 27 ++- index.html | 66 +++++- .../add_new_instance_to_specify_cloud.html | 51 +++-- aws/KU IT Notes.html => misc/kuit_notes.html | 62 ++++-- .../specify7_ecs_most-in-one.html | 57 +++-- .../useful_bash_cmds.html | 106 +++++++-- ...code_django_unit_test_debugging_notes.html | 44 +++- objects.inv | Bin 759 -> 787 bytes search.html | 27 ++- searchindex.js | 2 +- security/fix_exposed_secret.html | 170 +++++++++++++++ 44 files changed, 1815 insertions(+), 533 deletions(-) rename _sources/{aws/AWS Config Files.md.txt => asset_server/asset_config_files.md.txt} (100%) rename _sources/{aws/Asset Server Config.md.txt => asset_server/asset_server_config.md.txt} (100%) rename _sources/{aws/Asset Server Setup.md.txt => asset_server/asset_server_setup.md.txt} (100%) rename _sources/{aws/Hybrid Asset Server Setup Instructions.md.txt => asset_server/hybrid_asset_server_setup.md.txt} (100%) delete mode 100644 _sources/aws/AWS Infrastructure Notes.md.txt delete mode 100644 _sources/aws/Client Migration Notes.md.txt create mode 100644 _sources/aws/aws_infrastructure_notes.md.txt rename _sources/aws/{AWS Specify Asset Server Setup.md.txt => aws_specify_asset_server_setup.md.txt} (99%) rename _sources/aws/{Setup AWS Specify Cloud.md.txt => aws_specify_cloud_setup.md.txt} (94%) create mode 100644 _sources/aws/client_migration_notes.md.txt rename _sources/aws/{EC2 & RDS Specify7 Setup.md.txt => ec2_rds_specify7_setup.md.txt} (65%) rename _sources/aws/{Specify Cloud Graviton Setup.md.txt => specify_cloud_graviton_setup.md.txt} (100%) create mode 100644 _sources/dev_process/specify_processes.rst.txt rename _sources/{aws => misc}/add_new_instance_to_specify_cloud.md.txt (98%) rename _sources/{aws/KU IT Notes.md.txt => misc/kuit_notes.md.txt} (88%) rename _sources/{aws/Specify7 ECS most-in-one.md.txt => misc/specify7_ecs_most-in-one.md.txt} (97%) rename _sources/{aws/Useful Bash Commands.md.txt => misc/useful_bash_cmds.md.txt} (85%) rename _sources/{aws/VS Code Django Unit Test Debugging Notes.md.txt => misc/vs_code_django_unit_test_debugging_notes.md.txt} (98%) create mode 100644 _sources/security/fix_exposed_secret.rst.txt rename aws/AWS Config Files.html => asset_server/asset_config_files.html (96%) rename aws/Asset Server Config.html => asset_server/asset_server_config.html (92%) rename aws/Asset Server Setup.html => asset_server/asset_server_setup.html (94%) rename aws/Hybrid Asset Server Setup Instructions.html => asset_server/hybrid_asset_server_setup.html (82%) delete mode 100644 aws/AWS Infrastructure Notes.html delete mode 100644 aws/Client Migration Notes.html create mode 100644 aws/aws_infrastructure_notes.html rename aws/{AWS Specify Asset Server Setup.html => aws_specify_asset_server_setup.html} (90%) rename aws/{Setup AWS Specify Cloud.html => aws_specify_cloud_setup.html} (84%) create mode 100644 aws/client_migration_notes.html rename aws/{EC2 & RDS Specify7 Setup.html => ec2_rds_specify7_setup.html} (66%) rename aws/{Specify Cloud Graviton Setup.html => specify_cloud_graviton_setup.html} (93%) create mode 100644 dev_process/specify_processes.html rename {aws => misc}/add_new_instance_to_specify_cloud.html (73%) rename aws/KU IT Notes.html => misc/kuit_notes.html (79%) rename aws/Specify7 ECS most-in-one.html => misc/specify7_ecs_most-in-one.html (89%) rename aws/Useful Bash Commands.html => misc/useful_bash_cmds.html (68%) rename aws/VS Code Django Unit Test Debugging Notes.html => misc/vs_code_django_unit_test_debugging_notes.html (82%) create mode 100644 security/fix_exposed_secret.html diff --git a/_sources/aws/AWS Config Files.md.txt b/_sources/asset_server/asset_config_files.md.txt similarity index 100% rename from _sources/aws/AWS Config Files.md.txt rename to _sources/asset_server/asset_config_files.md.txt diff --git a/_sources/aws/Asset Server Config.md.txt b/_sources/asset_server/asset_server_config.md.txt similarity index 100% rename from _sources/aws/Asset Server Config.md.txt rename to _sources/asset_server/asset_server_config.md.txt diff --git a/_sources/aws/Asset Server Setup.md.txt b/_sources/asset_server/asset_server_setup.md.txt similarity index 100% rename from _sources/aws/Asset Server Setup.md.txt rename to _sources/asset_server/asset_server_setup.md.txt diff --git a/_sources/aws/Hybrid Asset Server Setup Instructions.md.txt b/_sources/asset_server/hybrid_asset_server_setup.md.txt similarity index 100% rename from _sources/aws/Hybrid Asset Server Setup Instructions.md.txt rename to _sources/asset_server/hybrid_asset_server_setup.md.txt diff --git a/_sources/aws/AWS Infrastructure Notes.md.txt b/_sources/aws/AWS Infrastructure Notes.md.txt deleted file mode 100644 index e69de29..0000000 diff --git a/_sources/aws/Client Migration Notes.md.txt b/_sources/aws/Client Migration Notes.md.txt deleted file mode 100644 index 39311d1..0000000 --- a/_sources/aws/Client Migration Notes.md.txt +++ /dev/null @@ -1,18 +0,0 @@ -The migration from Digital Ocean to Amazon Web Services has our hosting provider will increase Specify's reliability and security. - -Since MySQL 5.7 is now deprecated, we are now using MariaDB v10.11. In the future we hope to upgrade to PostgreSQL - -The database backups will be further improved by storing daily backups for a month - -For connecting Specify6 to the database via ssh, two things have changed, there will be no root login to the server, and the IP address for the database. The Linux user name will be the same as in your institution's url, but with underscore `_` replacing dashes `-` - -Here are the new database IPs (they have been updated in the wiki as well https://github.com/specify/specify7/wiki/Specify-6-Remote-Access): -NA: `172.31.96.36` -EU: `172.31.16.73` -CA: `172.31.35.249` - -Here is an example -On Linux/Mac`ssh -N -L3307:172.31.96.36:3306 institution_id@na-db-1.specifycloud.org` -On Windows PuTTY target `C:\Program Files\PuTTY\putty.exe" -ssh -i C:\users\your_user\private_key_.ppk institution_id@eu-db-1.specifycloud.org -L 3307:172.31.16.73:3306 -N` - -For now, you will log into the database as `master` with the same previous passwords, but we will soon be creating database user for each institution. \ No newline at end of file diff --git a/_sources/aws/aws_infrastructure_notes.md.txt b/_sources/aws/aws_infrastructure_notes.md.txt new file mode 100644 index 0000000..e435f0f --- /dev/null +++ b/_sources/aws/aws_infrastructure_notes.md.txt @@ -0,0 +1,3 @@ +# AWS Infrastructure Notes + +TODO \ No newline at end of file diff --git a/_sources/aws/AWS Specify Asset Server Setup.md.txt b/_sources/aws/aws_specify_asset_server_setup.md.txt similarity index 99% rename from _sources/aws/AWS Specify Asset Server Setup.md.txt rename to _sources/aws/aws_specify_asset_server_setup.md.txt index 09c12e4..61e3943 100644 --- a/_sources/aws/AWS Specify Asset Server Setup.md.txt +++ b/_sources/aws/aws_specify_asset_server_setup.md.txt @@ -1,3 +1,5 @@ +# AWS Specify Asset Server Setup + ## EC2 Non-Dockerized Build ```bash #!/bin/bash @@ -132,6 +134,7 @@ sudo openssl dhparam -dsaparam -out /etc/ssl/certs/dhparam.pem 1024; # Edit ``` +## Config files /etc/systemd/system/web-asset-server.service -> ``` [Unit] @@ -352,10 +355,6 @@ DaFwAJUrqwEqrQP5fEQdOMdh522RwuD2/fPeXTukQHI8gUuMjk652aeLOcn1Ufhy -----END DH PARAMETERS----- ``` - -swiss asset server password: xD5dakesktkxceb - - ## EC2 Non-docker build shell script ```bash @@ -393,3 +392,5 @@ pip install --no-cache-dir -r requirements.txt; ``` ## Docker Build + +TODO \ No newline at end of file diff --git a/_sources/aws/Setup AWS Specify Cloud.md.txt b/_sources/aws/aws_specify_cloud_setup.md.txt similarity index 94% rename from _sources/aws/Setup AWS Specify Cloud.md.txt rename to _sources/aws/aws_specify_cloud_setup.md.txt index 6c04b76..444e39b 100644 --- a/_sources/aws/Setup AWS Specify Cloud.md.txt +++ b/_sources/aws/aws_specify_cloud_setup.md.txt @@ -1,10 +1,14 @@ -### Setup Aurora MySQL Database +# Specify Cloud Setup -### Setup EC2 Server +## Setup Aurora MySQL Database +TODO + +## Setup EC2 Server EC2 Parameters: - ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-arm64-server-20220131 -ami-0770bf1d6ae61c858 -Initial Commands: +ami-0770bf1d6ae61c858 + +## Initial Commands ```bash #!/bin/bash @@ -66,19 +70,27 @@ sudo systemctl reload sshd; docker-compose up -d ``` -SSH Client: vim ~/.ssh/config -``` -Host * - ServerAliveInterval 20 - #TCPKeepAlive no -``` -SSH Server: sudo vim /etc/ssh/sshd_config -``` -ClientAliveInterval 1200 -ClientAliveCountMax 3 -``` -Then run `sudo systemctl reload sshd` +## SSH Configuration + +* Client + + * config file: ~/.ssh/config + ``` + Host * + ServerAliveInterval 20 + #TCPKeepAlive no + ``` + +* Server + + * config file: /etc/ssh/sshd_config + ``` + ClientAliveInterval 1200 + ClientAliveCountMax 3 + ``` + * Then run `sudo systemctl reload sshd` +## Config files spcloudservers.json -> ```json { @@ -120,16 +132,17 @@ SP7_DEBUG=false ``` -### Info Misc. -aws credentials: +## Info Misc. + +### aws credentials: - username: `specify.user` -- password: `Specify-Cloud-aws-user` +- password: SPECIFY_USER_PASSWORD - access key: ACCESS_KEY - secret access key: ACCESS_KEY_SECRET - default region: us-east-1 - default output format: json -AWS EC2 User data: +### AWS EC2 User data: ```bash # Avoid services restarting during apt upgrade sudo sed -i "s/#\$nrconf{kernelhints} = -1;/\$nrconf{kernelhints} = -1;/g" /etc/needrestart/needrestart.conf; @@ -210,9 +223,10 @@ sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-arm64/jre/bin/ja # Build without docker cd specify6; ant compile-nonmac; - ``` +### AWS Pricing + Database Prices: - db.r5.large - 2 vCPUs - 16 gb ram - $0.24 per hour = $173.00 per month - db.m5.large - 4vCPUs - 16 gb ram - $0.171 per hour = $123.10 per month @@ -221,10 +235,13 @@ Database Prices: - db.t3.xlarge - 4vCPUs - 16 gb ram - $0.272 per hour = $195.80 per month - +db.t4g.medium - 2vCPUs - 4 gb ram - $0.065 per hour = $46.80 per month - db.t4g.large - 2vCPUs - 8 gb ram - $0.129 per hour = $92.88 per month + Aurora v2 Prices: - 1 ACU - 2 vCPUs - 2 gb ram - $0.12 per ACU hour = $86.40 per ACU month + Aurora v1 Prices: - 1 ACU - 2 vCPUs - 2 gb ram - $0.06 per ACU hour = $43.29 per ACU month + EC2 Prices: - t4g.nano - 2vCPUs - 0.5 gb ram - $0.0042 per hour = $3.02 per month - t4g.micro - 2vCPUs - 1 gb ram - $0.0084 per hour = $6.05 per month @@ -235,6 +252,7 @@ EC2 Prices: - m7g.medium - 1vCPUs - 4 gb ram - $0.0408 per hour = $29.38 per month - m7g.large - 2vCPUs - 8 gb ram - $0.0816 per hour = $58.75 per month - m7g.xlarge - 4vCPUs - 16 gb ram - $0.2232 per hour = $160.70 per month + Fargate Prices (Linux/ARM): - On Demand - $0.03238 per vCPU per hour and $0.00356 per GB per hour - Spot - $0.01279585 per vCPU per hour and $0.00140508 per GB per hour @@ -250,6 +268,7 @@ Fargate Prices (Linux/ARM): - ex. 2 cpus and 8 gb = $26.52 per month - ex. 8 cpus and 16 gb = $89.89 per month - ex. 16 cpus and 32 gb = $179.78 per month + Notes: - m7g is general purpose using graviton 3 - t4g is general purpose using graviton 2 @@ -265,12 +284,14 @@ NA Server: - 10 containers per task definition - So 9 task definitions needed for django - vimsfish might need more than 0.5 GB + CA Server: - 8 clients - digital ocean 1vCPUs 2 GB memory - cpu usage nominal at 8% with spikes to 80% - memory usage nominal at 85% - beaty might need more than 0.5 GB + EU Server: - 9 clients - digital ocean 1vCPUs 2 GB memory @@ -278,7 +299,9 @@ EU Server: - memory usage nominal at 80% - herb_rbge might need more than 0.5 GB -So maybe 1vCPU and 0.5 GB of memory will be enough to handle each django container. Most are fine with 0.5 GB, only a few will go over with the django and worker containers combined. +So maybe 1vCPU and 0.5 GB of memory will be enough to handle each django container. +Most are fine with 0.5 GB, only a few will go over with the django and worker containers +combined. Price Option Comparison t4g.medium @@ -328,10 +351,9 @@ PLGLgQtim77M68m+XNWLAAAAFGFjd2hpdGUyMTFAZ21haWwuY29tAQ== ``` MariaDB version: 10.3.38-MariaDB-0ubuntu0.20.04.1-log -AWS DB password: dancing-taco-magic-rainbow-vibes -AWS DB password: dance-taco-magic-rainbow-vibe +AWS DB password: DB_PASSWORD -Install Ubuntu EC2 instance all in one with no docker +### Install Ubuntu EC2 instance with no docker ```bash #!/bin/bash @@ -412,7 +434,7 @@ sudo ufw allow 'Nginx HTTP'; sudo ufw status; ``` -Using the Amazon arm54 centos image: +### Using the Amazon arm54 centos image: ```bash #!/bin/bash diff --git a/_sources/aws/client_migration_notes.md.txt b/_sources/aws/client_migration_notes.md.txt new file mode 100644 index 0000000..512d6fd --- /dev/null +++ b/_sources/aws/client_migration_notes.md.txt @@ -0,0 +1,33 @@ +# Digital Ocean to AWS migration + +The migration from Digital Ocean to Amazon Web Services has our hosting provider will +increase Specify's reliability and security. + +Since MySQL 5.7 is now deprecated, we are now using MariaDB v10.11. In the future we +hope to upgrade to PostgreSQL + +The database backups will be further improved by storing daily backups for a month + +For connecting Specify6 to the database via ssh, two things have changed, there will be +no root login to the server, and the IP address for the database. The Linux user name +will be the same as in your institution's url, but with +underscore `_` replacing dashes `-` + +Here are the new database IPs (they have been updated in the wiki [here] +(https://github.com/specify/specify7/wiki/Specify-6-Remote-Access): +NA: `172.31.96.36` +EU: `172.31.16.73` +CA: `172.31.35.249` + +Here is an example +On Linux/Mac`ssh -N -L3307:xxx.xx.xx.xx:3306 institution_id@na-db-1.specifycloud.org` +On Windows PuTTY target + +```commandline +C:\Program Files\PuTTY\putty.exe -ssh -i C:\users\your_user\private_key_.ppk \ + institution_id@eu-db-1.specifycloud.org -L 3307:xxx.xx.xx.xx:3306 -N` + +``` + +For now, you will log into the database as `master` with the same previous passwords, +but we will soon be creating database user for each institution. \ No newline at end of file diff --git a/_sources/aws/EC2 & RDS Specify7 Setup.md.txt b/_sources/aws/ec2_rds_specify7_setup.md.txt similarity index 65% rename from _sources/aws/EC2 & RDS Specify7 Setup.md.txt rename to _sources/aws/ec2_rds_specify7_setup.md.txt index dfac7b4..7918659 100644 --- a/_sources/aws/EC2 & RDS Specify7 Setup.md.txt +++ b/_sources/aws/ec2_rds_specify7_setup.md.txt @@ -1,11 +1,26 @@ -These are the detailed instructions to get Specify7 up and running on an EC2 instance. This is a temporary solution for deployment, with an ECS solution coming in the future. These instruction require access to the `specify/docker-compositions` private repo. For this kind of AWS deployment, it is easiest to use the `specifycloud` deployment, even with just one client instance. I used the `specifycloud` deployment for my AWS EC2 instance, but if you want to include things like the asset server, you can create a separate instance of the asset server deployment (instructions here https://github.com/specify/web-asset-server). You could also try to use the `all-in-one` deployment instead. - -Specify7 was not originally designed to be cloud native, but since I have joined the team and picked up the project, I have been working on developing a modern cloud native solution. I hope to soon have a set of CDK scripts that will make deployment to ECS, S3, and RDS simple and scalable. +# Deploy Specify7 to an EC2 instance + +These are the detailed instructions to get Specify7 up and running on an EC2 instance. +This is a temporary solution for deployment, with an ECS solution coming in the future. +These instruction require access to the `specify/docker-compositions` private repo. +For this kind of AWS deployment, it is easiest to use the `specifycloud` deployment, +even with just one client instance. I used the `specifycloud` deployment for my AWS +EC2 instance, but if you want to include things like the asset server, you can create a +separate instance of the asset server deployment (instructions `here +`). You could also try to use the +`all-in-one` deployment instead. + +Specify7 was not originally designed to be cloud native, but since I have joined the +team and picked up the project, I have been working on developing a modern cloud native +solution. I hope to soon have a set of CDK scripts that will make deployment to ECS, +S3, and RDS simple and scalable. ## Spin Up EC2 Instance -- For the Amazon Machine Image (AMI), choose the default Ubuntu Server (Ubuntu Server 22.04 LTS) with the 64-bit x86 architecture. -- For the instance type, start with the t3.small or t3.medium. Upgrade to a better instance if needed. +- For the Amazon Machine Image (AMI), choose the default Ubuntu Server (Ubuntu Server + 22.04 LTS) with the 64-bit x86 architecture. +- For the instance type, start with the t3.small or t3.medium. Upgrade to a better + instance if needed. - In Network Settings, make sure to allow HTTP and HTTPS traffic. - Setup your Key Pair for logging in. - Launch Instance. @@ -14,7 +29,12 @@ Specify7 was not originally designed to be cloud native, but since I have joined Create an S3 bucket for storing some static files. -You can either upload the GitHub repo to s3, or you can clone the repo directly in the EC2 instance. If you are cloning inside the EC2 instance, note that you will need a GitHub access token to clone the private repo, instructions here https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token. Make sure when configuring the token that you allow it to read from private repos, otherwise cloning will not work. +You can either upload the GitHub repo to s3, or you can clone the repo directly in the +EC2 instance. If you are cloning inside the EC2 instance, note that you will need a +GitHub access token to clone the private repo, instructions `here +`_. +Make sure when configuring the token that you allow it to read from private repos, +otherwise cloning will not work. Upload your SQL file to the S3 bucket for the initial database upload. @@ -37,8 +57,13 @@ spcloudservers.json -> } } ``` -Your need to edit the version of Specify6 for your database's schema version (latest is 6.8.03). For sp7, input the GitHub tag that you want to use in the repo (latest is always edge). For your first run, keep https set to false. Change it to true only after you have setup SSL/TLS. The database value is the name of your database. +Your need to edit the version of Specify6 for your database's schema version +(latest is 6.8.03). For sp7, input the GitHub tag that you want to use in the repo +(latest is always edge). For your first run, keep https set to false. Change it to +true only after you have setup SSL/TLS. The database value is the name of your +database. +## Config file defaults.env -> ``` DATABASE_HOST= @@ -55,17 +80,22 @@ CELERY_RESULT_BACKEND=redis://redis/1 LOG_LEVEL=WARNING SP7_DEBUG=false ``` -You will only need to edit the DATABASE_HOST, MASTER_PASSWORD, ASSET_SERVER_URL, and ASSET_SERVER_KEY +You will only need to edit the DATABASE_HOST, MASTER_PASSWORD, ASSET_SERVER_URL, and +ASSET_SERVER_KEY ## Configure Specify7 Once the EC2 instance is up and running, ssh into the EC2 instance. - Download your AWS access key - `chmod 600 YOUR-AWS-ACCESS-KEY.pem` -- On the EC2 Instance webpage, click connect, choose ssh, and copy the ssh command. Make sure you are in the directory with the access key. +- On the EC2 Instance webpage, click connect, choose ssh, and copy the ssh command. + Make sure you are in the directory with the access key. - You can try using AWS CloudShell if you don't want to ssh from your local machine. -Here are the commands to run to setup Specify7. Note that for the AWS access key, you can use the AWS Secrets manager if you prefer. Also note that after some of these commands, SystemD services will restart. When that happens, just press enter once or twice to you get back to the shell. +Here are the commands to run to setup Specify7. Note that for the AWS access key, +you can use the AWS Secrets manager if you prefer. Also note that after some of these +commands, SystemD services will restart. When that happens, just press enter once or +twice to you get back to the shell. ```bash # Avoid services restarting during apt updates @@ -112,31 +142,42 @@ gh repo clone https://github.com/specify/docker-compositions.git; ## Spin Up RDS Instance -- For choosing the RDS engine type, I have been experimenting with using Aurora MySQL 5.7 to handle dynamic scaling, but the default option for Specify7 on AWS RDS is MariaDB 10.6.10. +- For choosing the RDS engine type, I have been experimenting with using Aurora + MySQL 5.7 to handle dynamic scaling, but the default option for Specify7 on AWS RDS is + MariaDB 10.6.10. - For Templates, I would suggest starting with Dev/Test instead of production. - Setup Master password. -- For testing, I would suggest using either the `db.t3.medium` or `db.t3.large` instances. You might feel the need to upgrade to the `db.m5.large` instance or something similar. -- Decide on storage size to accommodate your data size. The general purpose SSD storage option should suffice, but you can try the provisioned IOPS SSD option if needed. -- Important, make sure in the connectivity options, select "Connect to an EC2 compute resource" and add your EC2 instance. +- For testing, I would suggest using either the `db.t3.medium` or `db.t3.large` + instances. You might feel the need to upgrade to the `db.m5.large` instance or + something similar. +- Decide on storage size to accommodate your data size. The general purpose SSD storage + option should suffice, but you can try the provisioned IOPS SSD option if needed. +- Important, make sure in the connectivity options, select "Connect to an EC2 compute + resource" and add your EC2 instance. - Create database. ## Upload Data -Get the database host from the AWS RDS webpage. In the connectivity tab of your RDS instance, copy the test marked 'Endpoint'. Then run the following commands to upload your data to the database. +Get the database host from the AWS RDS webpage. In the connectivity tab of your RDS +instance, copy the test marked 'Endpoint'. Then run the following commands to upload +your data to the database. ```bash # Database setup cd specifycloud; mkdir seed-databases; aws s3 cp s3://specify-cloud/seed-database/archireef.sql ./seed-databases/; -mysql --host specify-cloud-swiss-demo-database-1.c9qlnkmf2lfl.eu-central-2.rds.amazonaws.com --port 3306 -u master -p -e "create database archireef;"; -mysql --host specify-cloud-swiss-demo-database-1.c9qlnkmf2lfl.eu-central-2.rds.amazonaws.com --port 3306 -u master -p specify < ./seed-databases/archireef.sql; +mysql --host specify-cloud-database-1...rds.amazonaws.com \ + --port 3306 -u master -p -e "create database archireef;"; +mysql --host specify-cloud-database-1...rds.amazonaws.com \ + --port 3306 -u master -p specify < ./seed-databases/archireef.sql; rm -f ./seed-databases/archireef.sql; ``` ## Deploy Specify7 -Here are the remaining commands on the EC2 instance to setup the database connection and start running Specify7. Replace the S3 and RDS urls with your own. +Here are the remaining commands on the EC2 instance to setup the database connection +and start running Specify7. Replace the S3 and RDS urls with your own. ```bash # Run Specify Network @@ -145,14 +186,18 @@ make; sudo docker compose up -d; ``` -Go to the AWS console webpage and navigate to the EC2 instance page. For your instance, select the networking tab, then copy the the public IPv4 DNS address and open if in you r browser. Note to change the url from https to http if you haven't setup SSL/TLS yet (https notes in the Concluding notes section). +Go to the AWS console webpage and navigate to the EC2 instance page. For your instance, +select the networking tab, then copy the the public IPv4 DNS address and open if in your +browser. Note to change the url from https to http if you haven't setup SSL/TLS yet +(https notes in the Concluding notes section). ## Docker Container Dependencies Containers: - specify7 - django application of specify7 - - depends on connections to the webpack, specify7-worker, redis, asset-server, nginx, and report-runner containers + - depends on connections to the webpack, specify7-worker, redis, asset-server, + nginx, and report-runner containers - depends on files being created by the specify6 container - webpack - holds static files for the front-end @@ -172,7 +217,8 @@ Containers: - generates files and database schema version for specify7 - the conainter stops after the files are created - independent container - - this container can be removed if the static files are moved to S3 and then copied into the specify7 container on startup + - this container can be removed if the static files are moved to S3 and then copied + into the specify7 container on startup - nginx - webserver for specify7 django application - depends on connection to specify7 and webpack containers @@ -183,8 +229,11 @@ Containers: ## Concluding Notes -You can use 'AWS Certificate Manager' or the tool 'CertBot' for setting up HTTPS SSL/TLS certificates. Who will need to setup a domain name through AWS Route 53. You will probably want to setup an elastic ip address for the EC2 instance through AWS as well. -Here are the instructions for using certbot: +You can use 'AWS Certificate Manager' or the tool 'CertBot' for setting up HTTPS +SSL/TLS certificates. Who will need to setup a domain name through AWS Route 53. +You will probably want to setup an elastic ip address for the EC2 instance through AWS +as well. Here are the instructions for using certbot: + ```bash sudo apt install -y certbot python3-certbot-apache; sudo mkdir /var/www/archireef; diff --git a/_sources/aws/Specify Cloud Graviton Setup.md.txt b/_sources/aws/specify_cloud_graviton_setup.md.txt similarity index 100% rename from _sources/aws/Specify Cloud Graviton Setup.md.txt rename to _sources/aws/specify_cloud_graviton_setup.md.txt diff --git a/_sources/dev_process/specify_processes.rst.txt b/_sources/dev_process/specify_processes.rst.txt new file mode 100644 index 0000000..45f7814 --- /dev/null +++ b/_sources/dev_process/specify_processes.rst.txt @@ -0,0 +1,91 @@ +Specify Development Process +################################ + +Overview +************** +At Specify, we follow a structured process when creating a new product or feature to +ensure that we deliver high-quality solutions that meet user needs. Here's the step-by- +step process we follow: + +1. Vision: We start by defining the vision for the new product or feature. This includes +understanding the problem we're trying to solve and the goals we want to achieve. + +2. User Stories: Based on the vision, we create user stories that describe the specific +functionality from the user's perspective. This helps us stay focused on delivering value +to our users. + +3. Detailed Requirements: Once we have the user stories, we break them down into +detailed requirements. This includes defining the data model, UI/UX design, and any +other technical specifications. + +4. Priorities: We then prioritize the requirements based on factors such as user impact, +value, and technical complexity. + +5. Estimates Time: After prioritizing, we estimate the time required to implement each +requirement. This process enables effective planning of the project timeline and +allocation of resources. + +6. Issue Creation on GitHub: For each requirement, we create an issue on GitHub to +break it down into smaller tasks. This allows us to track progress and collaborate +effectively. + +7. Project Creation on GitHub: Once all the issues are created, we create a project on +GitHub to assemble all the small issues related to the feature. This provides a +centralized view of the project and helps us manage it more efficiently. + +8. Build: With the project plan in place, we start building the feature. This involves +writing code, designing interfaces, and implementing the required functionality. + +9. Unit Testing: After the initial build, the feature is tested by at least three different +team members on a unit level. This ensures that each component works as expected +and meets the requirements. + +10. Integration Testing: Once the feature is integrated into the larger system, we +perform testing on a global level to ensure that it works seamlessly with the existing +features. This global testing will have two phases: the first phase will be conducted by +the internal specification team, and the second phase will involve an open testing week +during which clients can test the new release version and provide feedback on their +findings. + +11. Launch: Finally, once the feature has been thoroughly tested and meets our quality +standards, we launch it to our users. This includes deploying the feature to production +and communicating the new feature to users. + +By following this process, we ensure that we deliver high-quality products and features +that meet user needs and align with our vision. + +Detailed process for new feature requests +********************************************* + +1. In the case a request comes from a client, a detailed requirement document will be +requested to be sent to the support team. + +2. The support team will create a feature request on GitHub with a detailed +explanation of what is needed. They will rephrase the request from the client and +use Specify’s terms to describe the needs. They will consider anything that needs +to be added on top of what the client has requested or what should be removed +from the request. The feature request will have a second part: a clear checklist of all +items that need to be present in the feature. For example, the new feature will do X +when doing Y in the component Z. The new feature will do A when doing B in the +component C. This list needs to include all places and all actions that will be +comprised in the feature. + +3. In the case the feature request comes from a client, the GitHub feature request link +will be sent to the client, and approval will be requested. In other cases, the link will +be posted to the specified engineering channel. + +4. The GitHub feature request will be reviewed by the lead software developer. It will +be assigned to a release and to a developer to work on. + +5. Once development has started ; The pull request created by the developer +responsible for the new feature will include a detailed checklist on how to test the +branch. This list should be in total compliance with the feature request checklist but +with added detailed explanations on how to proceed with the workflow. This list +might be updated in the case the developer brings modifications to their code. + +6. The testing team, when testing the branch, needs to write a comment on the PR +confirming that each point on the checklist has passed. In the case some of the +items fail, the full checklist will have to be tested again and added with a check in a +new comment during the second testing phase. This is a process that will need to +be followed for each iteration of testing. Each pull request needs 3 approvals before +being merged. \ No newline at end of file diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt index 16e85ff..f097169 100644 --- a/_sources/index.rst.txt +++ b/_sources/index.rst.txt @@ -2,10 +2,38 @@ Welcome to Specify Developer documentation! =========================================================== .. toctree:: - :maxdepth: 2 + :maxdepth: 1 + :caption: Internal Processes: + + dev_process/specify_processes + +.. toctree:: + :maxdepth: 1 :caption: Amazon Web Services: - aws/* + aws/aws_infrastructure_notes + aws/aws_specify_asset_server_setup + aws/aws_specify_cloud_setup + aws/client_migration_notes + aws/ec2_rds_specify7_setup + aws/specify7_ecs_most-in-one + aws/specify_cloud_graviton_setup + +.. toctree:: + :maxdepth: 1 + :caption: Security: + + security/fix_exposed_secret + +.. toctree:: + :maxdepth: 1 + :caption: Misc: + + misc/add_new_instance_to_specify_cloud + misc/kuit_notes + misc/specify7_ecs_most-in-one + misc/useful_bash_cmds + misc/vs_code_django_unit_test_debugging_notes Indices and tables diff --git a/_sources/aws/add_new_instance_to_specify_cloud.md.txt b/_sources/misc/add_new_instance_to_specify_cloud.md.txt similarity index 98% rename from _sources/aws/add_new_instance_to_specify_cloud.md.txt rename to _sources/misc/add_new_instance_to_specify_cloud.md.txt index a5d1e91..4a2f766 100644 --- a/_sources/aws/add_new_instance_to_specify_cloud.md.txt +++ b/_sources/misc/add_new_instance_to_specify_cloud.md.txt @@ -47,7 +47,7 @@ 1. Add url: unsm-vp.specifycloud.org/context/system_info.json 2. Add alias: unsm-vp -# Misc +## Misc * Add ssh key: @@ -56,7 +56,7 @@ vim .ssh/authorized_keys sudo systemctl reload sshd ``` -# Troubleshooting +## Troubleshooting * Handle mariadb failing to restart after restarting the Database droplet: diff --git a/_sources/aws/KU IT Notes.md.txt b/_sources/misc/kuit_notes.md.txt similarity index 88% rename from _sources/aws/KU IT Notes.md.txt rename to _sources/misc/kuit_notes.md.txt index 378f4c7..4b42625 100644 --- a/_sources/aws/KU IT Notes.md.txt +++ b/_sources/misc/kuit_notes.md.txt @@ -1,6 +1,9 @@ -## New certificate on biprdsp7wbdb.cc.ku.edu server +# Working with central KU IT -Form to request new certificate: https://kuit.service-now.com/nav_to.do?uri=%2Fcom.glideapp.servicecatalog_cat_item_view.do%3Fv%3D1%26sysparm_id%3D78fee42fdb2a8850162673e1ba96195b%26sysparm_link_parent%3D322911f41bec6490cf2d337e034bcb23%26sysparm_catalog%3De0d08b13c3330100c8b837659bba8fb4%26sysparm_catalog_view%3Dcatalog_default%26sysparm_view%3Dcatalog_default +## New certificate on ku.edu server + +Form to request new certificate: +https://kuit.service-now.com/nav_to.do?uri=%2Fcom.glideapp.servicecatalog_cat_item_view.do%3Fv%3D1%26sysparm_id%3D78fee42fdb2a8850162673e1ba96195b%26sysparm_link_parent%3D322911f41bec6490cf2d337e034bcb23%26sysparm_catalog%3De0d08b13c3330100c8b837659bba8fb4%26sysparm_catalog_view%3Dcatalog_default%26sysparm_view%3Dcatalog_default generate CSR string: ```bash @@ -20,7 +23,7 @@ A challenge password []: An optional company name []: ``` -verify configuration with `openssl req -new -newkey rsa:2048 -nodes -keyout server.key -out server.csr` with output +Verify configuration with `openssl req -new -newkey rsa:2048 -nodes -keyout server.key -out server.csr` with output ``` C = US, ST = Kansas, L = Lawrence, O = University of Kansas, OU = Specify, CN = biimages.biodiversity.ku.edu, emailAddress = alec.white@ku.edu ``` diff --git a/_sources/aws/Specify7 ECS most-in-one.md.txt b/_sources/misc/specify7_ecs_most-in-one.md.txt similarity index 97% rename from _sources/aws/Specify7 ECS most-in-one.md.txt rename to _sources/misc/specify7_ecs_most-in-one.md.txt index 0d2c59c..3c2c9ad 100644 --- a/_sources/aws/Specify7 ECS most-in-one.md.txt +++ b/_sources/misc/specify7_ecs_most-in-one.md.txt @@ -1,6 +1,9 @@ +# Create a Most-In-One Specify Docker image + ## Image -create a one client pod with specify7, specify7-worker, webpack, nginx, and maybe specify6 -excludes mariadb, redis, asset-server +Create a one client pod with specify7, specify7-worker, webpack, nginx, and maybe +specify6. Excludes mariadb, redis, asset-server. + ```Dockerfile FROM arm64v8/ubuntu:20.04 diff --git a/_sources/aws/Useful Bash Commands.md.txt b/_sources/misc/useful_bash_cmds.md.txt similarity index 85% rename from _sources/aws/Useful Bash Commands.md.txt rename to _sources/misc/useful_bash_cmds.md.txt index 4309386..47bc941 100644 --- a/_sources/aws/Useful Bash Commands.md.txt +++ b/_sources/misc/useful_bash_cmds.md.txt @@ -1,4 +1,6 @@ -sftp +# Useful bash commands + +## sftp ```bash sftp -i alec_specify_ssh_key ubuntu@ec2-52-206-2-67.compute-1.amazonaws.com; pwd; @@ -12,42 +14,43 @@ background & foreground tasks ``` -check storage +## check storage ```bash df -H du -sh * ``` -rsync +## rsync ```bash -rsync -avz -e "ssh -i ~/specify/keys/specify-aws-ssh.pem" ~/git/specify-aws/specify7-cluster/ ubuntu@ec2-52-206-2-67.compute-1.amazonaws.com:/home/ubuntu/specify7-cluster/ - +rsync -avz -e "ssh -i ~/specify/keys/specify-aws-ssh.pem" \ + ~/git/specify-aws/specify7-cluster/ \ + ubuntu@ec2-52-206-2-67.compute-1.amazonaws.com:/home/ubuntu/specify7-cluster/ ``` -docker images view architecture and OS +## docker images view architecture and OS ```bash for img in $(docker image ls -q); do echo $img; docker image inspect $img | jq '.[0] | {image: .RepoTags[0], os: .Os, arch: .Architecture}'; done ``` -run a django unit test through docker +## run a django unit test through docker ```bash sudo docker exec -it specify7-specify7-1 bash -c "ve/bin/python3 manage.py test specifyweb.notifications.tests.NotificationsTests" ``` -git stash specify files +## git stash specify files ```bash git stash push -m "stash-name" file1.txt file2.txt git stash list git stash apply stash^(0) ``` -add user +## add user ```bash adduser --disabled-password --gecos "" specify; su - specify; ``` -docker build and push for multiple architectures +## docker build and push for multiple architectures ```bash export DOCKER_CLI_EXPERIMENTAL=enabled docker buildx create --name mybuilder --use # only do once @@ -58,7 +61,7 @@ docker buildx build --platform linux/amd64,linux/arm64 -t specifyconsortium/spec docker buildx use default # don't think needs to be done ``` -create linux user for ssh login and database access +## create linux user for ssh login and database access ```bash #!/bin/bash @@ -99,14 +102,14 @@ EXIT; ``` -view live formatted nginx logs example +## view live formatted nginx logs example ```bash docker logs specifycloud-nginx-1 --tail 1000 --since 10m --follow | \ grep -v updown | grep -v notification | grep specifycloud.org | \ awk '{ split($4,time,"["); print time[2], "-", $6, $7, $8, $9, $10, $11; }' ``` -Add swap memory +## Add swap memory ```bash sudo fallocate -l 4G /swapfile; # sudo dd if=/dev/zero of=/swapfile bs=1024 count=4096k; # if fallocate is not available diff --git a/_sources/aws/VS Code Django Unit Test Debugging Notes.md.txt b/_sources/misc/vs_code_django_unit_test_debugging_notes.md.txt similarity index 98% rename from _sources/aws/VS Code Django Unit Test Debugging Notes.md.txt rename to _sources/misc/vs_code_django_unit_test_debugging_notes.md.txt index 99879b7..1013c82 100644 --- a/_sources/aws/VS Code Django Unit Test Debugging Notes.md.txt +++ b/_sources/misc/vs_code_django_unit_test_debugging_notes.md.txt @@ -1,4 +1,4 @@ -VS Code Django Unit Test Debugging Notes: +# VS Code Django Unit Test Debugging Might need to add permissions to the test db for the master user: `mysql --host 127.0.0.1 -u root -p'root' -e "GRANT ALL PRIVILEGES ON test_cuic.* TO 'master'@'%';"` diff --git a/_sources/security/fix_exposed_secret.rst.txt b/_sources/security/fix_exposed_secret.rst.txt new file mode 100644 index 0000000..7d5a9a3 --- /dev/null +++ b/_sources/security/fix_exposed_secret.rst.txt @@ -0,0 +1,41 @@ +Fix Exposed Secrets +############################ + + +If a secret or password is inadvertently exposed through Github or some other site +on the web, not only must it be deleted as soon as possible, but the password must +be changed or the secret revoked. + +Secrets +*************** + +You must understand the implications of revoking this secret by investigating where it +is used in your code and on any virtual or physical machines or processes. + +* Create another secret and replace all instances of the old one with the new +* Make your secret unusable by revoking it +* Store the new secret safely. `GitGuardian best practices + `_ + + + +Github +----------------- + +Making your repository private is not sufficient + +* **Do not** commit on top of the current source code + +* If the secret is in the last commit + + * Remove from file + * Edit previous commit + * Force push changes + +.. code-block:: + + git add + git commit --amend + git push --all --force + git push --tags --force + diff --git a/aws/AWS Config Files.html b/asset_server/asset_config_files.html similarity index 96% rename from aws/AWS Config Files.html rename to asset_server/asset_config_files.html index ffc2686..1353405 100644 --- a/aws/AWS Config Files.html +++ b/asset_server/asset_config_files.html @@ -42,22 +42,31 @@ @@ -74,7 +83,7 @@
  • Specify 7 Docker Config Example
  • - View page source + View page source
    • diff --git a/aws/Asset Server Config.html b/asset_server/asset_server_config.html similarity index 92% rename from aws/Asset Server Config.html rename to asset_server/asset_server_config.html index 95a38a9..78ba179 100644 --- a/aws/Asset Server Config.html +++ b/asset_server/asset_server_config.html @@ -42,10 +42,31 @@ @@ -62,7 +83,7 @@
  • <no title>
  • - View page source + View page source
    • diff --git a/aws/Asset Server Setup.html b/asset_server/asset_server_setup.html similarity index 94% rename from aws/Asset Server Setup.html rename to asset_server/asset_server_setup.html index d791fe9..d4687fd 100644 --- a/aws/Asset Server Setup.html +++ b/asset_server/asset_server_setup.html @@ -42,11 +42,31 @@ @@ -63,7 +83,7 @@
  • EC2 Non-Dockerized Build
  • - View page source + View page source
    • diff --git a/aws/Hybrid Asset Server Setup Instructions.html b/asset_server/hybrid_asset_server_setup.html similarity index 82% rename from aws/Hybrid Asset Server Setup Instructions.html rename to asset_server/hybrid_asset_server_setup.html index ab102b3..cddba01 100644 --- a/aws/Hybrid Asset Server Setup Instructions.html +++ b/asset_server/hybrid_asset_server_setup.html @@ -42,10 +42,31 @@ @@ -62,7 +83,7 @@
  • <no title>
  • - View page source + View page source
    • diff --git a/aws/AWS Infrastructure Notes.html b/aws/AWS Infrastructure Notes.html deleted file mode 100644 index bc27825..0000000 --- a/aws/AWS Infrastructure Notes.html +++ /dev/null @@ -1,102 +0,0 @@ - - - - - - - <no title> — Specify Developer Documentation documentation - - - - - - - - - - - - - - - - - -
    - - -
    - -
    -
    -
    - -
    -
    -
    -
    - - - -
    -
    - -
    -
    -
    -
    - - - - \ No newline at end of file diff --git a/aws/Client Migration Notes.html b/aws/Client Migration Notes.html deleted file mode 100644 index c9226d9..0000000 --- a/aws/Client Migration Notes.html +++ /dev/null @@ -1,114 +0,0 @@ - - - - - - - <no title> — Specify Developer Documentation documentation - - - - - - - - - - - - - - - - - -
    - - -
    - -
    -
    -
    - -
    -
    -
    -
    - -

    The migration from Digital Ocean to Amazon Web Services has our hosting provider will increase Specify’s reliability and security.

    -

    Since MySQL 5.7 is now deprecated, we are now using MariaDB v10.11. In the future we hope to upgrade to PostgreSQL

    -

    The database backups will be further improved by storing daily backups for a month

    -

    For connecting Specify6 to the database via ssh, two things have changed, there will be no root login to the server, and the IP address for the database. The Linux user name will be the same as in your institution’s url, but with underscore _ replacing dashes -

    -

    Here are the new database IPs (they have been updated in the wiki as well https://github.com/specify/specify7/wiki/Specify-6-Remote-Access): -NA: 172.31.96.36 -EU: 172.31.16.73 -CA: 172.31.35.249

    -

    Here is an example -On Linux/Macssh -N -L3307:172.31.96.36:3306 institution_id@na-db-1.specifycloud.org -On Windows PuTTY target C:\Program Files\PuTTY\putty.exe" -ssh -i C:\users\your_user\private_key_.ppk institution_id@eu-db-1.specifycloud.org -L 3307:172.31.16.73:3306 -N

    -

    For now, you will log into the database as master with the same previous passwords, but we will soon be creating database user for each institution.

    - - -
    -
    - -
    -
    -
    -
    - - - - \ No newline at end of file diff --git a/aws/aws_infrastructure_notes.html b/aws/aws_infrastructure_notes.html new file mode 100644 index 0000000..a1ea43f --- /dev/null +++ b/aws/aws_infrastructure_notes.html @@ -0,0 +1,132 @@ + + + + + + + AWS Infrastructure Notes — Specify Developer Documentation documentation + + + + + + + + + + + + + + + + + + + +
    + + +
    + +
    +
    +
    + +
    +
    +
    +
    + +
    +

    AWS Infrastructure Notes

    +

    TODO

    +
    + + +
    +
    + +
    +
    +
    +
    + + + + \ No newline at end of file diff --git a/aws/AWS Specify Asset Server Setup.html b/aws/aws_specify_asset_server_setup.html similarity index 90% rename from aws/AWS Specify Asset Server Setup.html rename to aws/aws_specify_asset_server_setup.html index 64464b2..af4283c 100644 --- a/aws/AWS Specify Asset Server Setup.html +++ b/aws/aws_specify_asset_server_setup.html @@ -4,7 +4,7 @@ - EC2 Non-Dockerized Build — Specify Developer Documentation documentation + AWS Specify Asset Server Setup — Specify Developer Documentation documentation @@ -20,7 +20,9 @@ - + + + @@ -42,13 +44,37 @@ @@ -63,9 +89,9 @@
    @@ -73,8 +99,10 @@
    -
    -

    EC2 Non-Dockerized Build

    +
    +

    AWS Specify Asset Server Setup

    +
    +

    EC2 Non-Dockerized Build

    #!/bin/bash
 
 sudo apt update;
@@ -207,6 +235,9 @@ 
    EC2 Non-Dockerized Build# Edit
    +
    +
    +

    Config files

    /etc/systemd/system/web-asset-server.service ->

    [Unit]
 Description=Specify Web Asset Server
@@ -422,10 +453,9 @@ 
    EC2 Non-Dockerized Build-----END DH PARAMETERS-----
    -

    swiss asset server password: xD5dakesktkxceb

    -

    EC2 Non-docker build shell script

    +

    EC2 Non-docker build shell script

    #!/bin/bash
 
 sudo apt update;
@@ -460,13 +490,18 @@ 
    EC2 Non-docker build shell script
-
    Docker Build
    
+
    Docker Build
    
+
    TODO
    
+
    -
    +
    diff --git a/aws/Setup AWS Specify Cloud.html b/aws/aws_specify_cloud_setup.html similarity index 84% rename from aws/Setup AWS Specify Cloud.html rename to aws/aws_specify_cloud_setup.html index bc605bb..488df32 100644 --- a/aws/Setup AWS Specify Cloud.html +++ b/aws/aws_specify_cloud_setup.html @@ -4,7 +4,7 @@ - Setup Aurora MySQL Database — Specify Developer Documentation documentation + Specify Cloud Setup — Specify Developer Documentation documentation @@ -20,7 +20,9 @@ - + + + @@ -42,14 +44,47 @@
    @@ -64,9 +99,9 @@
    @@ -74,17 +109,22 @@
    -
    -

    Setup Aurora MySQL Database

    +
    +

    Specify Cloud Setup

    +
    +

    Setup Aurora MySQL Database

    +

    TODO

    -

    Setup EC2 Server

    +

    Setup EC2 Server

    EC2 Parameters:

    • ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-arm64-server-20220131
      -ami-0770bf1d6ae61c858 -Initial Commands:

      • +ami-0770bf1d6ae61c858

    +
    +
    +

    Initial Commands

    #!/bin/bash
 
 sudo apt-get update;
@@ -145,18 +185,36 @@ 
    Setup EC2 Server up -d
    -

    SSH Client: vim ~/.ssh/config

    +
    +
    +

    SSH Configuration

    +
      +

    • Client

      +
        +

      • config file: ~/.ssh/config

        Host *
-    ServerAliveInterval 20
-    #TCPKeepAlive no
+	ServerAliveInterval 20
+	#TCPKeepAlive no
        -

        SSH Server: sudo vim /etc/ssh/sshd_config

        +
        • +
      +
      • +

    • Server

      +
        +

      • config file: /etc/ssh/sshd_config

        ClientAliveInterval 1200
 ClientAliveCountMax 3
        -

        Then run sudo systemctl reload sshd

        +
        • +

      • Then run sudo systemctl reload sshd

        • +
      +
      • +
    +
    +
    +

    Config files

    spcloudservers.json ->

    {
 	"servers": {
@@ -198,17 +256,20 @@ 
    Setup EC2 Server
-
    Info Misc.
    
-
    aws credentials:
    
+
    Info Misc.
    
+
    
+
    aws credentials:
    
 
      
 
    • username: specify.user
      • 
-
    • password: Specify-Cloud-aws-user
      • 
+
    • password: SPECIFY_USER_PASSWORD
      • 
 
    • access key: ACCESS_KEY
      • 
 
    • secret access key: ACCESS_KEY_SECRET
      • 
 
    • default region: us-east-1
      • 
 
    • default output format: json
      • 
 
    
-
    AWS EC2 User data:
    
+
    
+
    
+
    AWS EC2 User data:
    
 
    # Avoid services restarting during apt upgrade
 sudo sed -i "s/#\$nrconf{kernelhints} = -1;/\$nrconf{kernelhints} = -1;/g" /etc/needrestart/needrestart.conf;
 sudo sed -i "s/#\$nrconf{restart} = 'i';/\$nrconf{restart} = 'a';/g" /etc/needrestart/needrestart.conf;
@@ -290,6 +351,9 @@ 
    Info Misc. compile-nonmac;
 
    
 
    
+
    
+
    
+
    AWS Pricing
    
 
    Database Prices:
    
 
+
    Aurora v1 Prices:
    
+
      
+
    • 1 ACU - 2 vCPUs - 2 gb ram - $0.06 per ACU hour = $43.29 per ACU month
      • 
+
    
+
    EC2 Prices:
    
+
    
+
    
+
    Install Ubuntu EC2 instance with no docker
    
 
    #!/bin/bash
 
 # Avoid services restarting during apt upgrade
@@ -496,7 +577,9 @@ 
    Specify Network Extract ufw status;
 
    
 
    
-
    Using the Amazon arm54 centos image:
    
+
    
+
    
+
    Using the Amazon arm54 centos image:
    
 
    #!/bin/bash
 
 sudo yum upgrade;
@@ -522,12 +605,17 @@ 
    Specify Network Extract install --upgrade -r specify7/requirements.txt
 
    
 
    
+
    
+
    -
    +
    diff --git a/aws/client_migration_notes.html b/aws/client_migration_notes.html new file mode 100644 index 0000000..45af2fb --- /dev/null +++ b/aws/client_migration_notes.html @@ -0,0 +1,155 @@ + + + + + + + Digital Ocean to AWS migration — Specify Developer Documentation documentation + + + + + + + + + + + + + + + + + + + +
    + + +
    + +
    +
    +
    + +
    +
    +
    +
    + +
    +

    Digital Ocean to AWS migration

    +

    The migration from Digital Ocean to Amazon Web Services has our hosting provider will +increase Specify’s reliability and security.

    +

    Since MySQL 5.7 is now deprecated, we are now using MariaDB v10.11. In the future we +hope to upgrade to PostgreSQL

    +

    The database backups will be further improved by storing daily backups for a month

    +

    For connecting Specify6 to the database via ssh, two things have changed, there will be +no root login to the server, and the IP address for the database. The Linux user name +will be the same as in your institution’s url, but with +underscore _ replacing dashes -

    +

    Here are the new database IPs (they have been updated in the wiki [here] +(https://github.com/specify/specify7/wiki/Specify-6-Remote-Access): +NA: 172.31.96.36 +EU: 172.31.16.73 +CA: 172.31.35.249

    +

    Here is an example +On Linux/Macssh -N -L3307:xxx.xx.xx.xx:3306 institution_id@na-db-1.specifycloud.org +On Windows PuTTY target

    +
    C:\Program Files\PuTTY\putty.exe  -ssh -i C:\users\your_user\private_key_.ppk \
+    institution_id@eu-db-1.specifycloud.org -L 3307:xxx.xx.xx.xx:3306 -N`
+
+
    +
    +

    For now, you will log into the database as master with the same previous passwords, +but we will soon be creating database user for each institution.

    +
    + + +
    +
    + +
    +
    +
    +
    + + + + \ No newline at end of file diff --git a/aws/EC2 & RDS Specify7 Setup.html b/aws/ec2_rds_specify7_setup.html similarity index 66% rename from aws/EC2 & RDS Specify7 Setup.html rename to aws/ec2_rds_specify7_setup.html index ff3f083..cae9f25 100644 --- a/aws/EC2 & RDS Specify7 Setup.html +++ b/aws/ec2_rds_specify7_setup.html @@ -4,7 +4,7 @@ - Spin Up EC2 Instance — Specify Developer Documentation documentation + Deploy Specify7 to an EC2 instance — Specify Developer Documentation documentation @@ -20,7 +20,9 @@ - + + + @@ -42,18 +44,42 @@
    @@ -68,9 +94,9 @@
    @@ -78,22 +104,40 @@
    -

    These are the detailed instructions to get Specify7 up and running on an EC2 instance. This is a temporary solution for deployment, with an ECS solution coming in the future. These instruction require access to the specify/docker-compositions private repo. For this kind of AWS deployment, it is easiest to use the specifycloud deployment, even with just one client instance. I used the specifycloud deployment for my AWS EC2 instance, but if you want to include things like the asset server, you can create a separate instance of the asset server deployment (instructions here https://github.com/specify/web-asset-server). You could also try to use the all-in-one deployment instead.

    -

    Specify7 was not originally designed to be cloud native, but since I have joined the team and picked up the project, I have been working on developing a modern cloud native solution. I hope to soon have a set of CDK scripts that will make deployment to ECS, S3, and RDS simple and scalable.

    +
    +

    Deploy Specify7 to an EC2 instance

    +

    These are the detailed instructions to get Specify7 up and running on an EC2 instance.
    +This is a temporary solution for deployment, with an ECS solution coming in the future.
    +These instruction require access to the specify/docker-compositions private repo.
    +For this kind of AWS deployment, it is easiest to use the specifycloud deployment, +even with just one client instance. I used the specifycloud deployment for my AWS +EC2 instance, but if you want to include things like the asset server, you can create a +separate instance of the asset server deployment (instructions here  <https://github.com/specify/web-asset-server>). You could also try to use the +all-in-one deployment instead.

    +

    Specify7 was not originally designed to be cloud native, but since I have joined the +team and picked up the project, I have been working on developing a modern cloud native +solution. I hope to soon have a set of CDK scripts that will make deployment to ECS, +S3, and RDS simple and scalable.

    -

    Spin Up EC2 Instance

    +

    Spin Up EC2 Instance

      -

    • For the Amazon Machine Image (AMI), choose the default Ubuntu Server (Ubuntu Server 22.04 LTS) with the 64-bit x86 architecture.

      • -

    • For the instance type, start with the t3.small or t3.medium. Upgrade to a better instance if needed.

      • +

    • For the Amazon Machine Image (AMI), choose the default Ubuntu Server (Ubuntu Server +22.04 LTS) with the 64-bit x86 architecture.

      • +

    • For the instance type, start with the t3.small or t3.medium. Upgrade to a better +instance if needed.

    • In Network Settings, make sure to allow HTTP and HTTPS traffic.

    • Setup your Key Pair for logging in.

    • Launch Instance.

    -

    S3 Setup

    +

    S3 Setup

    Create an S3 bucket for storing some static files.

    -

    You can either upload the GitHub repo to s3, or you can clone the repo directly in the EC2 instance. If you are cloning inside the EC2 instance, note that you will need a GitHub access token to clone the private repo, instructions here https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token. Make sure when configuring the token that you allow it to read from private repos, otherwise cloning will not work.

    +

    You can either upload the GitHub repo to s3, or you can clone the repo directly in the +EC2 instance. If you are cloning inside the EC2 instance, note that you will need a +GitHub access token to clone the private repo, instructions here  <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token>_.
    +Make sure when configuring the token that you allow it to read from private repos, +otherwise cloning will not work.

    Upload your SQL file to the S3 bucket for the initial database upload.

    There are two config files that you need to create for specifycloud:

    spcloudservers.json ->

    @@ -113,7 +157,14 @@

    S3 Setup }

    -

    Your need to edit the version of Specify6 for your database’s schema version (latest is 6.8.03). For sp7, input the GitHub tag that you want to use in the repo (latest is always edge). For your first run, keep https set to false. Change it to true only after you have setup SSL/TLS. The database value is the name of your database.

    +

    Your need to edit the version of Specify6 for your database’s schema version +(latest is 6.8.03). For sp7, input the GitHub tag that you want to use in the repo +(latest is always edge). For your first run, keep https set to false. Change it to +true only after you have setup SSL/TLS. The database value is the name of your +database.

    + +
    +

    Config file

    defaults.env ->

    DATABASE_HOST=<YOUR_DATABASE_HOST>
 DATABASE_PORT=3306
@@ -130,18 +181,23 @@ 
    S3 Setup
 SP7_DEBUG=false
    -

    You will only need to edit the DATABASE_HOST, MASTER_PASSWORD, ASSET_SERVER_URL, and ASSET_SERVER_KEY

    +

    You will only need to edit the DATABASE_HOST, MASTER_PASSWORD, ASSET_SERVER_URL, and +ASSET_SERVER_KEY

    -

    Configure Specify7

    +

    Configure Specify7

    Once the EC2 instance is up and running, ssh into the EC2 instance.

    • Download your AWS access key

    • chmod 600 YOUR-AWS-ACCESS-KEY.pem

      • -

    • On the EC2 Instance webpage, click connect, choose ssh, and copy the ssh command. Make sure you are in the directory with the access key.

      • +

    • On the EC2 Instance webpage, click connect, choose ssh, and copy the ssh command.
      +Make sure you are in the directory with the access key.

    • You can try using AWS CloudShell if you don’t want to ssh from your local machine.

    -

    Here are the commands to run to setup Specify7. Note that for the AWS access key, you can use the AWS Secrets manager if you prefer. Also note that after some of these commands, SystemD services will restart. When that happens, just press enter once or twice to you get back to the shell.

    +

    Here are the commands to run to setup Specify7. Note that for the AWS access key, +you can use the AWS Secrets manager if you prefer. Also note that after some of these +commands, SystemD services will restart. When that happens, just press enter once or +twice to you get back to the shell.

    # Avoid services restarting during apt updates
 sudo sed -i "s/#\$nrconf{kernelhints} = -1;/\$nrconf{kernelhints} = -1;/g" /etc/needrestart/needrestart.conf;
 
@@ -186,49 +242,64 @@ 
    Configure Specify7
-
    Spin Up RDS Instance
    
+
    Spin Up RDS Instance
    
 
      
-
    • For choosing the RDS engine type, I have been experimenting with using Aurora MySQL 5.7 to handle dynamic scaling, but the default option for Specify7 on AWS RDS is MariaDB 10.6.10.
      • 
+
    • For choosing the RDS engine type, I have been experimenting with using Aurora
+MySQL 5.7 to handle dynamic scaling, but the default option for Specify7 on AWS RDS is
+MariaDB 10.6.10.
      • 
 
    • For Templates, I would suggest starting with Dev/Test instead of production.
      • 
 
    • Setup Master password.
      • 
-
    • For testing, I would suggest using either the db.t3.medium or db.t3.large instances. You might feel the need to upgrade to the db.m5.large instance or something similar.
      • 
-
    • Decide on storage size to accommodate your data size. The general purpose SSD storage option should suffice, but you can try the provisioned IOPS SSD option if needed.
      • 
-
    • Important, make sure in the connectivity options, select “Connect to an EC2 compute resource” and add your EC2 instance.
      • 
+
    • For testing, I would suggest using either the db.t3.medium or db.t3.large
+instances. You might feel the need to upgrade to the db.m5.large instance or
+something similar.
      • 
+
    • Decide on storage size to accommodate your data size. The general purpose SSD storage
+option should suffice, but you can try the provisioned IOPS SSD option if needed.
      • 
+
    • Important, make sure in the connectivity options, select “Connect to an EC2 compute
+resource” and add your EC2 instance.
      • 
 
    • Create database.
      •
    -

    Upload Data

    -

    Get the database host from the AWS RDS webpage. In the connectivity tab of your RDS instance, copy the test marked ‘Endpoint’. Then run the following commands to upload your data to the database.

    +

    Upload Data

    +

    Get the database host from the AWS RDS webpage. In the connectivity tab of your RDS +instance, copy the test marked ‘Endpoint’. Then run the following commands to upload +your data to the database.

    # Database setup
 cd specifycloud;
 mkdir seed-databases;
 aws s3 cp s3://specify-cloud/seed-database/archireef.sql ./seed-databases/;
-mysql --host specify-cloud-swiss-demo-database-1.c9qlnkmf2lfl.eu-central-2.rds.amazonaws.com --port 3306 -u master -p -e "create database archireef;";
-mysql --host specify-cloud-swiss-demo-database-1.c9qlnkmf2lfl.eu-central-2.rds.amazonaws.com --port 3306 -u master -p specify < ./seed-databases/archireef.sql;
+mysql --host specify-cloud-database-1.<identifier>.<region>.rds.amazonaws.com \
+      --port 3306 -u master -p -e "create database archireef;";
+mysql --host specify-cloud-database-1.<identifier>.<region>.rds.amazonaws.com \
+	  --port 3306 -u master -p specify < ./seed-databases/archireef.sql;
 rm -f ./seed-databases/archireef.sql;
    -

    Deploy Specify7

    -

    Here are the remaining commands on the EC2 instance to setup the database connection and start running Specify7. Replace the S3 and RDS urls with your own.

    +

    Deploy Specify7

    +

    Here are the remaining commands on the EC2 instance to setup the database connection +and start running Specify7. Replace the S3 and RDS urls with your own.

    # Run Specify Network
 cd specifycloud;
 make;
 sudo docker compose up -d;
    -

    Go to the AWS console webpage and navigate to the EC2 instance page. For your instance, select the networking tab, then copy the the public IPv4 DNS address and open if in you r browser. Note to change the url from https to http if you haven’t setup SSL/TLS yet (https notes in the Concluding notes section).

    +

    Go to the AWS console webpage and navigate to the EC2 instance page. For your instance, +select the networking tab, then copy the the public IPv4 DNS address and open if in your +browser. Note to change the url from https to http if you haven’t setup SSL/TLS yet +(https notes in the Concluding notes section).

    -

    Docker Container Dependencies

    +

    Docker Container Dependencies

    Containers:

    • specify7

      • django application of specify7

        • -

      • depends on connections to the webpack, specify7-worker, redis, asset-server, nginx, and report-runner containers

        • +

      • depends on connections to the webpack, specify7-worker, redis, asset-server, +nginx, and report-runner containers

      • depends on files being created by the specify6 container

      • @@ -263,7 +334,8 @@

      Docker Container Dependencies -

      Concluding Notes

      -

      You can use ‘AWS Certificate Manager’ or the tool ‘CertBot’ for setting up HTTPS SSL/TLS certificates. Who will need to setup a domain name through AWS Route 53. You will probably want to setup an elastic ip address for the EC2 instance through AWS as well. -Here are the instructions for using certbot:

      +

      Concluding Notes

      +

      You can use ‘AWS Certificate Manager’ or the tool ‘CertBot’ for setting up HTTPS +SSL/TLS certificates. Who will need to setup a domain name through AWS Route 53.
      +You will probably want to setup an elastic ip address for the EC2 instance through AWS +as well. Here are the instructions for using certbot:

      sudo apt install -y certbot python3-certbot-apache;
 sudo mkdir /var/www/archireef;
 sudo certbot --webroot -w /var/www/archireef -d <YOUR_EC2_DOMAIN_NAME> certonly;
@@ -308,12 +382,16 @@ 
      Concluding Notessudo systemctl reload sshd
      
+
    -