Skip to content

Commit f59d0fd

Browse files
actions-useractions-user
committed
Add changes for 69a372a 
1 parent b720074 commit f59d0fd

10 files changed

+298
-866
lines changed

‎_downloads/6ec29de37be126c3f696e22d6d1a96df/build_non_docker_asset_server.sh

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/bin/bash
2+
3+
## Set these environment variables before running the script
4+
# export DOMAIN_NAME <domain name>
5+
# export SUBDOMAIN_PREFIX <first section of subdomain>
6+
7+
sudo apt update;
8+
sudo apt upgrade -y;
9+
sudo apt-get -y install --no-install-recommends \
10+
python3-venv \
11+
python3.8 \
12+
python3.8-dev \
13+
python3-pip \
14+
imagemagick \
15+
ghostscript \
16+
git \
17+
nginx \
18+
certbot \
19+
authbind \
20+
s3fs \
21+
awscli;
22+
23+
# python 3.6
24+
#sudo apt update
25+
#sudo apt install build-essential checkinstall zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev libsqlite3-dev wget libbz2-dev;
26+
#wget https://www.python.org/ftp/python/3.6.15/Python-3.6.15.tgz;
27+
#tar -xf Python-3.6.15.tgz;
28+
#cd Python-3.6.15 && ./configure --enable-optimizations;
29+
#make -j$(nproc);
30+
#sudo make altinstall;
31+
#python3.6 --version;
32+
33+
# python 3.6 install with apt
34+
sudo apt install software-properties-common;
35+
sudo add-apt-repository ppa:deadsnakes/ppa;
36+
sudo apt update;
37+
sudo apt install python3.6;
38+
sudo apt-get install python3.6-distutils;
39+
40+
# install pip3.6
41+
#wget https://bootstrap.pypa.io/pip/3.6/get-pip.py;
42+
python3.6 -m venv --without-pip ve;
43+
source ve/bin/activate;
44+
wget https://bootstrap.pypa.io/get-pip.py;
45+
#wget https://bootstrap.pypa.io/pip/3.5/get-pip.py
46+
#deactivate;
47+
48+
# activate python3.6 venv
49+
sudo apt install -y python3-virtualenv;
50+
python3.6 -m venv myenv;
51+
source myenv/bin/activate;
52+
pip install --no-cache-dir -r requirements.txt;
53+
#deactivate;
54+
55+
# TLS dependencies
56+
sudo apt-get -y install --no-install-recommends \
57+
certbot \
58+
python3-certbot-nginx \
59+
software-properties-common;
60+
61+
# Configure AWS
62+
aws configure set aws_access_key_id "ACCESS_KEY";
63+
aws configure set aws_secret_access_key "ACCESS_KEY_SECRET";
64+
aws configure set default.region us-east-1;
65+
aws configure set default.output json;
66+
67+
# Import attachment files
68+
#mkdir attachments;
69+
#aws s3 cp s3://specify-cloud/assets-server/attachments/ ~/attachments --recursive;
70+
71+
# S3 Mounting
72+
mkdir attachments;
73+
s3fs specify-cloud /assets-server/attachments/;
74+
75+
# Clone asset server repo
76+
git clone https://github.com/specify/web-asset-server.git;
77+
cd ~/web-asset-server;
78+
git checkout arm-build;
79+
80+
# Build python web asset server
81+
python3.8 -m venv ve;
82+
sudo ve/bin/pip install --no-cache-dir -r requirements.txt
83+
#sudo pip install -r requirements.txt;
84+
85+
# Port config
86+
# not needed when running with nginx
87+
#sudo apt-get install authbind;
88+
#touch 80;
89+
#chmod u+x 80;
90+
#sudo mv 80 /etc/authbind/byport;
91+
92+
# Create SystemD service
93+
sudo cat > /etc/systemd/system/web-asset-server.service << EOF
94+
[Unit]
95+
Description=Specify Web Asset Server
96+
Wants=network.target
97+
98+
[Service]
99+
User=ubuntu
100+
WorkingDirectory=/home/ubuntu/web-asset-server
101+
ExecStart=/home/ubuntu/web-asset-server/ve/bin/python /home/ubuntu/web-asset-server/server.py
102+
Restart=always
103+
104+
[Install]
105+
WantedBy=multi-user.target
106+
107+
EOF
108+
109+
sudo systemctl daemon-reload;
110+
sudo systemctl enable web-asset-server.service;
111+
sudo systemctl start web-asset-server.service;
112+
sudo systemctl status web-asset-server.service;
113+
114+
# nginx
115+
# sudo vim etc/nginx/sites-enabled/assets.conf
116+
sudo rm -f /etc/nginx/sites-enabled/default;
117+
sudo nginx -t;
118+
sudo /etc/init.d/nginx reload;
119+
120+
# S3 Mounting
121+
mount -o discard,defaults,noatime /dev/disk/by-id/scsi-0DO_Volume_volume-nyc1-01 /mnt/volume-nyc1-01
122+
123+
# TODO: EFS Mounting
124+
125+
sudo ls -la /etc/letsencrypt/live/$SUBDOMAIN_PREFIX.$DOMAIN_NAME;
126+
# Certbot TLS config
127+
sudo mkdir /var/www/.well-known;
128+
sudo certbot --nginx -d $SUBDOMAIN_PREFIX.$DOMAIN_NAME -d $SUBDOMAIN_PREFIX.$DOMAIN_NAME;
129+
sudo ls -la /etc/letsencrypt/live/$SUBDOMAIN_PREFIX.$DOMAIN_NAME;
130+
#openssl dhparam -out /etc/nginx/dhparam.pem 4096;
131+
sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096; #2048 or 1024
132+
sudo openssl dhparam -dsaparam -out /etc/ssl/certs/dhparam.pem 1024;
133+
# add https server config to nginx assets.
134+
135+
# Edit

‎_sources/aws/aws_specify_asset_server_setup.md.txt

Lines changed: 25 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -1,140 +1,19 @@
11
# AWS Specify Asset Server Setup
22

33
## EC2 Non-Dockerized Build
4-
```bash
5-
#!/bin/bash
6-
7-
sudo apt update;
8-
sudo apt upgrade -y;
9-
sudo apt-get -y install --no-install-recommends \
10-
python3-venv \
11-
python3.8 \
12-
python3.8-dev \
13-
python3-pip \
14-
imagemagick \
15-
ghostscript \
16-
git \
17-
nginx \
18-
certbot \
19-
authbind \
20-
s3fs \
21-
awscli;
22-
23-
# python 3.6
24-
#sudo apt update
25-
#sudo apt install build-essential checkinstall zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev libsqlite3-dev wget libbz2-dev;
26-
#wget https://www.python.org/ftp/python/3.6.15/Python-3.6.15.tgz;
27-
#tar -xf Python-3.6.15.tgz;
28-
#cd Python-3.6.15 && ./configure --enable-optimizations;
29-
#make -j$(nproc);
30-
#sudo make altinstall;
31-
#python3.6 --version;
32-
33-
# python 3.6 install with apt
34-
sudo apt install software-properties-common;
35-
sudo add-apt-repository ppa:deadsnakes/ppa;
36-
sudo apt update;
37-
sudo apt install python3.6;
38-
sudo apt-get install python3.6-distutils;
39-
40-
# install pip3.6
41-
#wget https://bootstrap.pypa.io/pip/3.6/get-pip.py;
42-
python3.6 -m venv --without-pip ve;
43-
source ve/bin/activate;
44-
wget https://bootstrap.pypa.io/get-pip.py;
45-
#wget https://bootstrap.pypa.io/pip/3.5/get-pip.py
46-
#deactivate;
47-
48-
# activate python3.6 venv
49-
sudo apt install -y python3-virtualenv;
50-
python3.6 -m venv myenv;
51-
source myenv/bin/activate;
52-
pip install --no-cache-dir -r requirements.txt;
53-
#deactivate;
54-
55-
# TLS dependencies
56-
sudo apt-get -y install --no-install-recommends \
57-
certbot \
58-
python3-certbot-nginx \
59-
software-properties-common;
60-
61-
# Configure AWS
62-
aws configure set aws_access_key_id "ACCESS_KEY";
63-
aws configure set aws_secret_access_key "ACCESS_KEY_SECRET";
64-
aws configure set default.region us-east-1;
65-
aws configure set default.output json;
66-
67-
# Import attachment files
68-
#mkdir attachments;
69-
#aws s3 cp s3://specify-cloud/assets-server/attachments/ ~/attachments --recursive;
704

71-
# S3 Mounting
72-
mkdir attachments;
73-
s3fs specify-cloud /assets-server/attachments/;
5+
Make sure to set environment variables before running the bash script
746

75-
# Clone asset server repo
76-
git clone https://github.com/specify/web-asset-server.git;
77-
cd ~/web-asset-server;
78-
git checkout arm-build;
79-
80-
# Build python web asset server
81-
python3.8 -m venv ve;
82-
sudo ve/bin/pip install --no-cache-dir -r requirements.txt
83-
#sudo pip install -r requirements.txt;
84-
85-
# Port config
86-
# not needed when running with nginx
87-
#sudo apt-get install authbind;
88-
#touch 80;
89-
#chmod u+x 80;
90-
#sudo mv 80 /etc/authbind/byport;
91-
92-
# Create SystemD service
93-
sudo cat > /etc/systemd/system/web-asset-server.service << EOF
94-
[Unit]
95-
Description=Specify Web Asset Server
96-
Wants=network.target
97-
98-
[Service]
99-
User=ubuntu
100-
WorkingDirectory=/home/ubuntu/web-asset-server
101-
ExecStart=/home/ubuntu/web-asset-server/ve/bin/python /home/ubuntu/web-asset-server/server.py
102-
Restart=always
103-
104-
[Install]
105-
WantedBy=multi-user.target
106-
107-
EOF
108-
109-
sudo systemctl daemon-reload;
110-
sudo systemctl enable web-asset-server.service;
111-
sudo systemctl start web-asset-server.service;
112-
sudo systemctl status web-asset-server.service;
113-
114-
# nginx
115-
# sudo vim etc/nginx/sites-enabled/assets.conf
116-
sudo rm -f /etc/nginx/sites-enabled/default;
117-
sudo nginx -t;
118-
sudo /etc/init.d/nginx reload;
119-
120-
# S3 Mounting
121-
mount -o discard,defaults,noatime /dev/disk/by-id/scsi-0DO_Volume_volume-nyc1-01 /mnt/volume-nyc1-01
122-
123-
# TODO: EFS Mounting
124-
125-
# Certbot TLS config
126-
sudo mkdir /var/www/.well-known;
127-
sudo certbot --nginx -d assets-test.specifycloud.org -d assets-test.specifycloud.org;
128-
sudo ls -la /etc/letsencrypt/live/assets-test.specifycloud.org;
129-
#openssl dhparam -out /etc/nginx/dhparam.pem 4096;
130-
sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096; #2048 or 1024
131-
sudo openssl dhparam -dsaparam -out /etc/ssl/certs/dhparam.pem 1024;
132-
# add https server config to nginx assets.
133-
134-
# Edit
7+
```command line
8+
export DOMAIN_NAME <domain name>
9+
export SUBDOMAIN_PREFIX <first section of subdomain>
13510
```
13611

12+
[build_non_docker_asset_server.sh](../scripts/build_non_docker_asset_server.sh) to build the instance.
13+
13714
## Config files
15+
16+
Make sure to set environment variables
13817
/etc/systemd/system/web-asset-server.service ->
13918
```
14019
[Unit]
@@ -185,7 +64,7 @@ ALLOW_STATIC_FILE_ACCESS = True
18564
# These values are interpolated into the web_asset_store.xml resource
18665
# so the client knows how to talk to the server.
18766
#HOST = 'localhost'
188-
HOST = 'assets-test.specifycloud.org'
67+
HOST = 'subdomain.domain.name'
18968
PORT = 8080
19069
#PORT = 80
19170

@@ -228,7 +107,7 @@ CAN_THUMBNAIL = {'image/jpeg', 'image/gif', 'image/png', 'image/tiff', 'applicat
228107
SERVER = 'wsgiref' # For testing. Requires no extra packages.
229108
```
230109

231-
/etc/nginx/sites-enabled/assets.conf from the aasets1.specifycloud.org- ->
110+
/etc/nginx/sites-enabled/assets.conf from the subdomain.domain.name- ->
232111
```
233112
# Nginx configuration for supplying an HTTPS end point for the web
234113
# asset server. The asset server is running on the same system
@@ -241,7 +120,7 @@ SERVER = 'wsgiref' # For testing. Requires no extra packages.
241120
server {
242121
# HTTP access is needed for Specify 6. It will not work with HTTPS.
243122
listen 80 default_server;
244-
server_name assets1.specifycloud.org;
123+
server_name subdomain.domain.name;
245124
client_max_body_size 0;
246125

247126
# The LetsEncrypt certificate mechanism places a nonce
@@ -258,7 +137,7 @@ server {
258137
# it defines point to this proxy.
259138
location = /web_asset_store.xml {
260139
proxy_pass http://localhost:8080/web_asset_store.xml;
261-
sub_filter 'http://assets1.specifycloud.org:8080' 'http://assets1.specifycloud.org';
140+
sub_filter 'http://subdomain.domain.name:8080' 'http://subdomain.domain.name';
262141
sub_filter_once off;
263142
sub_filter_types text/xml;
264143
}
@@ -273,11 +152,11 @@ server {
273152
server {
274153
# This stanza defines the HTTPS end point.
275154
listen 443 ssl default_server;
276-
server_name assets1.specifycloud.org;
155+
server_name subdomain.domain.name;
277156
client_max_body_size 0;
278157

279-
ssl_certificate /etc/letsencrypt/live/assets1.specifycloud.org/fullchain.pem;
280-
ssl_certificate_key /etc/letsencrypt/live/assets1.specifycloud.org/privkey.pem;
158+
ssl_certificate /etc/letsencrypt/live/subdomain.domain.name/fullchain.pem;
159+
ssl_certificate_key /etc/letsencrypt/live/subdomain.domain.name/privkey.pem;
281160

282161
# from https://cipherli.st/
283162
# and https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
@@ -311,7 +190,7 @@ server {
311190
# to use HTTPS in addition to changing the port.
312191
location = /web_asset_store.xml {
313192
proxy_pass http://localhost:8080/web_asset_store.xml;
314-
sub_filter 'http://assets1.specifycloud.org:8080' 'https://assets1.specifycloud.org';
193+
sub_filter 'http://subdomain.domain.name:8080' 'https://subdomain.domain.name';
315194
sub_filter_once off;
316195
sub_filter_types text/xml;
317196
}
@@ -323,15 +202,15 @@ server {
323202
}
324203
```
325204

326-
/etc/letsencrypt/renewal/assets1.specifycloud.org.conf ->
205+
/etc/letsencrypt/renewal/subdomain.domain.name.conf ->
327206
```
328207
# renew_before_expiry = 30 days
329-
cert = /etc/letsencrypt/live/assets1.specifycloud.org/cert.pem
330-
privkey = /etc/letsencrypt/live/assets1.specifycloud.org/privkey.pem
331-
chain = /etc/letsencrypt/live/assets1.specifycloud.org/chain.pem
332-
fullchain = /etc/letsencrypt/live/assets1.specifycloud.org/fullchain.pem
208+
cert = /etc/letsencrypt/live/subdomain.domain.name/cert.pem
209+
privkey = /etc/letsencrypt/live/subdomain.domain.name/privkey.pem
210+
chain = /etc/letsencrypt/live/subdomain.domain.name/chain.pem
211+
fullchain = /etc/letsencrypt/live/subdomain.domain.name/fullchain.pem
333212
version = 1.9.0
334-
archive_dir = /etc/letsencrypt/archive/assets1.specifycloud.org
213+
archive_dir = /etc/letsencrypt/archive/subdomain.domain.name
335214

336215
# Options and defaults used in the renewal process
337216
[renewalparams]
@@ -340,18 +219,13 @@ account = a563615cc912ed3d7a3edfede09d6760
340219
post_hook = systemctl reload nginx
341220
server = https://acme-v02.api.letsencrypt.org/directory
342221
[[webroot_map]]
343-
assets1.specifycloud.org = /var/www
222+
subdomain.domain.name = /var/www
344223
```
345224

346-
/etc/ssl/certs/dhparam.pem from assets1.specofycloud.org->
225+
/etc/ssl/certs/dhparam.pem from subdomain.domain.name->
347226
```
348227
-----BEGIN DH PARAMETERS-----
349-
MIIBCAKCAQEAlcFKsIuFylwX47jxqbNT0wSVD6ifznsMcti8f7T+zaQQNr84IYIM
350-
pNTT9E6SrVkkJg2u1nGScNqj5lArXvrda6zL66T8WmkFFrGfNW7RYCQ3vpg6BpGs
351-
dJ3+HtWYDNoMbeCrDyMz1DDfX/3OWblTTZRbjpvn/tEgTAn3DexP/QkE9E2c1AUX
352-
Mf/07vWpZ7giemaNgaME3fHDKyReNhTpfg1eDKypUUhEmr+PJmWQ9LQBc12LyXOP
353-
DaFwAJUrqwEqrQP5fEQdOMdh522RwuD2/fPeXTukQHI8gUuMjk652aeLOcn1Ufhy
354-
/KbbV6TJi7wS5F3HVaNXGOLMsHq+CywOCwIBAg==
228+
...
355229
-----END DH PARAMETERS-----
356230
```
357231

0 commit comments

Comments
 (0)