Skip to content

Commit

Permalink
Merge pull request #737 from IATI/develop
Browse files Browse the repository at this point in the history 
Develop to master for changes requirements, adding prometheus
  • Loading branch information
@Bjwebb
Bjwebb authored Feb 13, 2024
2 parents 4cf7be7 + 6f686be commit fa0783e
Show file tree
Hide file tree
Showing 9 changed files with 741 additions and 21 deletions.
2 changes: 2 additions & 0 deletions .dockerignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
db_www_export.dump
media
92 changes: 90 additions & 2 deletions .github/workflows/workflow.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ jobs:
--resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--size Standard_B2s \
--image UbuntuLTS \
--image `az vm image list --all -p Canonical -f UbuntuServer -s 18.04-LTS --query [].urn -o tsv | sort | tail -n 1` \
--ssh-key-values ${{ secrets.DEV_PUB_KEYS }} && \
export NEW_IP=$(az vm list-ip-addresses --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --query [].virtualMachine.network[].publicIpAddresses[][].ipAddress --output tsv) && \
echo "NEW_IP=$NEW_IP" >> $GITHUB_ENV && \
Expand All @@ -135,6 +135,50 @@ jobs:
--access Allow \
--protocol Tcp \
--destination-port-ranges 5000 && \
az network nsg rule create \
--resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--nsg-name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}NSG" \
--name AllowPrometheusPort9157 \
--priority 1011 \
--access Allow \
--protocol Tcp \
--destination-port-ranges 9157 && \
az vm run-command invoke \
--resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--command-id RunShellScript \
--scripts "\
set -eux
adduser prometheus-client --disabled-password --gecos ''
cd /home/prometheus-client/
PROMETHEUS_VERSION=1.7.0
wget https://github.com/prometheus/node_exporter/releases/download/v\$PROMETHEUS_VERSION/node_exporter-\$PROMETHEUS_VERSION.linux-amd64.tar.gz
tar -xvzf node_exporter-\$PROMETHEUS_VERSION.linux-amd64.tar.gz
echo \"\
[Unit]
Description=Prometheus Node Exporter
Wants=network-online.target
After=network-online.target
[Service]
User=prometheus-client
Group=prometheus-client
Type=simple
ExecStart=/home/prometheus-client/node_exporter-\$PROMETHEUS_VERSION.linux-amd64/node_exporter \\\\
--collector.systemd \\\\
--web.listen-address=:9157 \\\\
--web.config.file /home/prometheus-client/web-config.yaml
[Install]
WantedBy=multi-user.target
\" > /etc/systemd/system/prometheus-node-exporter.service
echo 'basic_auth_users:
# Do not include the dollars in the secret, as escaping is a pain
# Password is generated using htpasswd -nBC 10 "" | tr -d ':'
prom: \"\$2y\$10\$${{ secrets.PROMETHEUS_CLIENT_PASSWORD_HASHED_PARTIAL }}\"
' > /home/prometheus-client/web-config.yaml
systemctl enable --now prometheus-node-exporter.service
" && \
az vm run-command invoke \
--resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
Expand Down Expand Up @@ -283,7 +327,7 @@ jobs:
--resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--size Standard_B2s \
--image UbuntuLTS \
--image `az vm image list --all -p Canonical -f UbuntuServer -s 18.04-LTS --query [].urn -o tsv | sort | tail -n 1` \
--ssh-key-values ${{ secrets.DEV_PUB_KEYS }} && \
export NEW_IP=$(az vm list-ip-addresses --name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" --query [].virtualMachine.network[].publicIpAddresses[][].ipAddress --output tsv) && \
echo "NEW_IP=$NEW_IP" >> $GITHUB_ENV && \
Expand All @@ -301,6 +345,50 @@ jobs:
--access Allow \
--protocol Tcp \
--destination-port-ranges 5000 && \
az network nsg rule create \
--resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--nsg-name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}NSG" \
--name AllowPrometheusPort9157 \
--priority 1011 \
--access Allow \
--protocol Tcp \
--destination-port-ranges 9157 && \
az vm run-command invoke \
--resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--command-id RunShellScript \
--scripts "\
set -eux
adduser prometheus-client --disabled-password --gecos ''
cd /home/prometheus-client/
PROMETHEUS_VERSION=1.7.0
wget https://github.com/prometheus/node_exporter/releases/download/v\$PROMETHEUS_VERSION/node_exporter-\$PROMETHEUS_VERSION.linux-amd64.tar.gz
tar -xvzf node_exporter-\$PROMETHEUS_VERSION.linux-amd64.tar.gz
echo \"\
[Unit]
Description=Prometheus Node Exporter
Wants=network-online.target
After=network-online.target
[Service]
User=prometheus-client
Group=prometheus-client
Type=simple
ExecStart=/home/prometheus-client/node_exporter-\$PROMETHEUS_VERSION.linux-amd64/node_exporter \\\\
--collector.systemd \\\\
--web.listen-address=:9157 \\\\
--web.config.file /home/prometheus-client/web-config.yaml
[Install]
WantedBy=multi-user.target
\" > /etc/systemd/system/prometheus-node-exporter.service
echo 'basic_auth_users:
# Do not include the dollars in the secret, as escaping is a pain
# Password is generated using htpasswd -nBC 10 "" | tr -d ':'
prom: \"\$2y\$10\$${{ secrets.PROMETHEUS_CLIENT_PASSWORD_HASHED_PARTIAL }}\"
' > /home/prometheus-client/web-config.yaml
systemctl enable --now prometheus-node-exporter.service
" && \
az vm run-command invoke \
--resource-group "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
--name "${{ env.STAGE }}-${{ env.NAME }}-${{ env.NEW_COLOUR }}" \
Expand Down
8 changes: 7 additions & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,16 @@ WORKDIR /usr/src/app
COPY requirements.txt /usr/src/app/
COPY requirements_dev.txt /usr/src/app/
COPY entrypoint.sh /usr/src/app/
ENV PATH=$HOME/.cargo/bin:$PATH
ENV VIRTUAL_ENV=/usr/src/venv
ENV PATH=$VIRTUAL_ENV/bin:$HOME/.cargo/bin:$PATH

RUN apk -U upgrade
# Use a virtual env here, because othewise we get conflicats between Alpine's
# packages and pip's. (This has started happening because we switched to pip-
# tools which pins every dependency).
RUN apk add --no-cache gcc musl-dev python3-dev libffi-dev openssl-dev cargo libwebp libwebp-tools &&\
python3 -m venv /usr/src/venv &&\
. /usr/src/venv/bin/activate &&\
pip3 install -r requirements_dev.txt

RUN apk add --no-cache gettext
Expand Down
10 changes: 10 additions & 0 deletions config/nginx/website_dev.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,13 @@ server {
return 302 https://iatiwebsitedev.blob.core.windows.net/dev-iati-website$1;
}
}


# Proxy the prometheus port from the destination server
server {
listen 9158;

location / {
proxy_pass http://XX.XX.XX.XX:9157;
}
}
10 changes: 10 additions & 0 deletions config/nginx/website_prod.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,13 @@ server {
return 302 https://cdn.iatistandard.org/prod-iati-website$1;
}
}


# Proxy the prometheus port from the destination server
server {
listen 9158;

location / {
proxy_pass http://XX.XX.XX.XX:9157;
}
}
26 changes: 26 additions & 0 deletions requirements.in
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Babel==2.9.1
beautifulsoup4==4.8.2
bleach==3.3.0
dj-database-url==0.5
django-compressor==4.1
django-extensions==3.1.3
django-import-export==2.5.0
django-haystack==3.0
django-modeltranslation==0.17.2
django-prettyjson==0.4.1
django-recaptcha3==0.4.0
django-storages[azure]==1.11.1
django-widget-tweaks==1.4.8
Django==3.2.4
elasticsearch==6.8.2
gunicorn==20.1.0
opencensus-ext-azure==1.0.7
opencensus-ext-django==0.7.4
PyGithub==1.55
psycopg2-binary==2.8.6
requests==2.25.1
wagtail-modeltranslation==0.13.0
wagtail>=4.1,<4.2
Wand==0.6.6
whitenoise==5.2.0
wrapt==1.11.*
Loading

0 comments on commit fa0783e

Please sign in to comment.