From 86f77fc9521845682c7f51181f456d1c0320262b Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 6 May 2024 16:22:32 +0200 Subject: [PATCH 01/49] Flatten roles structure --- playbooks/engine/infrastructure.yml | 8 ++++---- playbooks/federated_api/infrastructure.yml | 4 ++-- roles/{infrastructure => }/chromium/README.md | 0 roles/chromium/meta/main.yml | 2 ++ roles/{infrastructure => }/chromium/tasks/main.yml | 0 roles/{infrastructure => }/common/README.md | 0 roles/{infrastructure => }/common/tasks/main.yml | 1 + roles/infrastructure/chromium/meta/main.yml | 2 -- roles/infrastructure/mongo/meta/main.yml | 2 -- roles/infrastructure/nginx/meta/main.yml | 2 -- roles/infrastructure/node/meta/main.yml | 2 -- roles/{infrastructure => }/mongo/README.md | 0 roles/{infrastructure => }/mongo/files/mongod.conf | 0 roles/mongo/meta/main.yml | 2 ++ roles/{infrastructure => }/mongo/tasks/main.yml | 0 roles/{infrastructure => }/nginx/README.md | 0 roles/{infrastructure => }/nginx/handlers/main.yml | 0 roles/nginx/meta/main.yml | 2 ++ roles/{infrastructure => }/nginx/tasks/main.yml | 0 roles/{infrastructure => }/nginx/templates/nginx.conf.j2 | 0 roles/{infrastructure => }/node/README.md | 0 roles/node/meta/main.yml | 2 ++ roles/{infrastructure => }/node/tasks/main.yml | 0 23 files changed, 15 insertions(+), 14 deletions(-) rename roles/{infrastructure => }/chromium/README.md (100%) create mode 100644 roles/chromium/meta/main.yml rename roles/{infrastructure => }/chromium/tasks/main.yml (100%) rename roles/{infrastructure => }/common/README.md (100%) rename roles/{infrastructure => }/common/tasks/main.yml (93%) delete mode 100644 roles/infrastructure/chromium/meta/main.yml delete mode 100644 roles/infrastructure/mongo/meta/main.yml delete mode 100644 roles/infrastructure/nginx/meta/main.yml delete mode 100644 roles/infrastructure/node/meta/main.yml rename roles/{infrastructure => }/mongo/README.md (100%) rename roles/{infrastructure => }/mongo/files/mongod.conf (100%) create mode 100644 roles/mongo/meta/main.yml rename roles/{infrastructure => }/mongo/tasks/main.yml (100%) rename roles/{infrastructure => }/nginx/README.md (100%) rename roles/{infrastructure => }/nginx/handlers/main.yml (100%) create mode 100644 roles/nginx/meta/main.yml rename roles/{infrastructure => }/nginx/tasks/main.yml (100%) rename roles/{infrastructure => }/nginx/templates/nginx.conf.j2 (100%) rename roles/{infrastructure => }/node/README.md (100%) create mode 100644 roles/node/meta/main.yml rename roles/{infrastructure => }/node/tasks/main.yml (100%) diff --git a/playbooks/engine/infrastructure.yml b/playbooks/engine/infrastructure.yml index 5b1c32c..13d9a31 100644 --- a/playbooks/engine/infrastructure.yml +++ b/playbooks/engine/infrastructure.yml @@ -11,7 +11,7 @@ tags: always - ansible.builtin.include_role: - name: infrastructure/mongo + name: mongo when: - (app_config.recorder.versions.storage.type is defined and app_config.recorder.versions.storage.type == 'mongo') or (app_config.recorder.snapshots.storage.type is defined and app_config.recorder.snapshots.storage.type == 'mongo') @@ -20,6 +20,6 @@ - ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64') roles: - - role: infrastructure/node - - role: infrastructure/chromium - - role: infrastructure/nginx + - role: node + - role: chromium + - role: nginx diff --git a/playbooks/federated_api/infrastructure.yml b/playbooks/federated_api/infrastructure.yml index f67c8fe..005b76c 100644 --- a/playbooks/federated_api/infrastructure.yml +++ b/playbooks/federated_api/infrastructure.yml @@ -4,5 +4,5 @@ become: true roles: - - role: infrastructure/node - - role: infrastructure/nginx + - role: node + - role: nginx diff --git a/roles/infrastructure/chromium/README.md b/roles/chromium/README.md similarity index 100% rename from roles/infrastructure/chromium/README.md rename to roles/chromium/README.md diff --git a/roles/chromium/meta/main.yml b/roles/chromium/meta/main.yml new file mode 100644 index 0000000..b1656ff --- /dev/null +++ b/roles/chromium/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - { role: common } diff --git a/roles/infrastructure/chromium/tasks/main.yml b/roles/chromium/tasks/main.yml similarity index 100% rename from roles/infrastructure/chromium/tasks/main.yml rename to roles/chromium/tasks/main.yml diff --git a/roles/infrastructure/common/README.md b/roles/common/README.md similarity index 100% rename from roles/infrastructure/common/README.md rename to roles/common/README.md diff --git a/roles/infrastructure/common/tasks/main.yml b/roles/common/tasks/main.yml similarity index 93% rename from roles/infrastructure/common/tasks/main.yml rename to roles/common/tasks/main.yml index dc39076..b53c389 100644 --- a/roles/infrastructure/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -10,3 +10,4 @@ - zip update_cache: true state: latest + become: true diff --git a/roles/infrastructure/chromium/meta/main.yml b/roles/infrastructure/chromium/meta/main.yml deleted file mode 100644 index e5e65fc..0000000 --- a/roles/infrastructure/chromium/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: - - { role: infrastructure/common } diff --git a/roles/infrastructure/mongo/meta/main.yml b/roles/infrastructure/mongo/meta/main.yml deleted file mode 100644 index e5e65fc..0000000 --- a/roles/infrastructure/mongo/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: - - { role: infrastructure/common } diff --git a/roles/infrastructure/nginx/meta/main.yml b/roles/infrastructure/nginx/meta/main.yml deleted file mode 100644 index e5e65fc..0000000 --- a/roles/infrastructure/nginx/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: - - { role: infrastructure/common } diff --git a/roles/infrastructure/node/meta/main.yml b/roles/infrastructure/node/meta/main.yml deleted file mode 100644 index e5e65fc..0000000 --- a/roles/infrastructure/node/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: - - { role: infrastructure/common } diff --git a/roles/infrastructure/mongo/README.md b/roles/mongo/README.md similarity index 100% rename from roles/infrastructure/mongo/README.md rename to roles/mongo/README.md diff --git a/roles/infrastructure/mongo/files/mongod.conf b/roles/mongo/files/mongod.conf similarity index 100% rename from roles/infrastructure/mongo/files/mongod.conf rename to roles/mongo/files/mongod.conf diff --git a/roles/mongo/meta/main.yml b/roles/mongo/meta/main.yml new file mode 100644 index 0000000..b1656ff --- /dev/null +++ b/roles/mongo/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - { role: common } diff --git a/roles/infrastructure/mongo/tasks/main.yml b/roles/mongo/tasks/main.yml similarity index 100% rename from roles/infrastructure/mongo/tasks/main.yml rename to roles/mongo/tasks/main.yml diff --git a/roles/infrastructure/nginx/README.md b/roles/nginx/README.md similarity index 100% rename from roles/infrastructure/nginx/README.md rename to roles/nginx/README.md diff --git a/roles/infrastructure/nginx/handlers/main.yml b/roles/nginx/handlers/main.yml similarity index 100% rename from roles/infrastructure/nginx/handlers/main.yml rename to roles/nginx/handlers/main.yml diff --git a/roles/nginx/meta/main.yml b/roles/nginx/meta/main.yml new file mode 100644 index 0000000..b1656ff --- /dev/null +++ b/roles/nginx/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - { role: common } diff --git a/roles/infrastructure/nginx/tasks/main.yml b/roles/nginx/tasks/main.yml similarity index 100% rename from roles/infrastructure/nginx/tasks/main.yml rename to roles/nginx/tasks/main.yml diff --git a/roles/infrastructure/nginx/templates/nginx.conf.j2 b/roles/nginx/templates/nginx.conf.j2 similarity index 100% rename from roles/infrastructure/nginx/templates/nginx.conf.j2 rename to roles/nginx/templates/nginx.conf.j2 diff --git a/roles/infrastructure/node/README.md b/roles/node/README.md similarity index 100% rename from roles/infrastructure/node/README.md rename to roles/node/README.md diff --git a/roles/node/meta/main.yml b/roles/node/meta/main.yml new file mode 100644 index 0000000..b1656ff --- /dev/null +++ b/roles/node/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - { role: common } diff --git a/roles/infrastructure/node/tasks/main.yml b/roles/node/tasks/main.yml similarity index 100% rename from roles/infrastructure/node/tasks/main.yml rename to roles/node/tasks/main.yml From fb35d57c2b572435b65db7faec499340cb3e53be Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 6 May 2024 16:27:09 +0200 Subject: [PATCH 02/49] Extract `github` role --- roles/engine/meta/main.yml | 2 ++ roles/engine/tasks/main.yml | 18 ------------------ roles/federated_api/meta/main.yml | 2 ++ roles/federated_api/tasks/main.yml | 18 ------------------ roles/github/README.md | 1 + roles/github/tasks/main.yml | 19 +++++++++++++++++++ 6 files changed, 24 insertions(+), 36 deletions(-) create mode 100644 roles/engine/meta/main.yml create mode 100644 roles/federated_api/meta/main.yml create mode 100644 roles/github/README.md create mode 100644 roles/github/tasks/main.yml diff --git a/roles/engine/meta/main.yml b/roles/engine/meta/main.yml new file mode 100644 index 0000000..4c5a271 --- /dev/null +++ b/roles/engine/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - { role: github } diff --git a/roles/engine/tasks/main.yml b/roles/engine/tasks/main.yml index d52fe15..68cf46b 100644 --- a/roles/engine/tasks/main.yml +++ b/roles/engine/tasks/main.yml @@ -1,21 +1,3 @@ -- name: Add GitHub Bot SSH key - ansible.builtin.copy: - content: '{{ ota_engine_github_bot_private_key }}' - dest: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' - owner: '{{ ansible_user }}' - group: '{{ ansible_user }}' - mode: "600" - -- name: Add GitHub SSH key to known_hosts - ansible.builtin.known_hosts: - name: github.com - key: "{{ item }}" - path: ~/.ssh/known_hosts - with_items: # GitHub's SSH key fingerprints can be found here: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints - - "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" - - "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" - - "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" - - name: Install services declarations ansible.builtin.git: repo: '{{ ota_engine_app_config.services.repository }}' diff --git a/roles/federated_api/meta/main.yml b/roles/federated_api/meta/main.yml new file mode 100644 index 0000000..4c5a271 --- /dev/null +++ b/roles/federated_api/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - { role: github } diff --git a/roles/federated_api/tasks/main.yml b/roles/federated_api/tasks/main.yml index ee40c2f..72957fc 100644 --- a/roles/federated_api/tasks/main.yml +++ b/roles/federated_api/tasks/main.yml @@ -1,21 +1,3 @@ -- name: Add GitHub Bot SSH key - ansible.builtin.copy: - content: '{{ ota_engine_github_bot_private_key }}' - dest: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' - owner: '{{ ansible_user }}' - group: '{{ ansible_user }}' - mode: "600" - -- name: Add GitHub SSH key to known_hosts - ansible.builtin.known_hosts: - name: github.com - key: "{{ item }}" - path: ~/.ssh/known_hosts - with_items: # GitHub's SSH key fingerprints can be found here: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints - - "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" - - "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" - - "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" - - name: Clone federated API repository ansible.builtin.git: repo: '{{ ota_federated_api_repo }}' diff --git a/roles/github/README.md b/roles/github/README.md new file mode 100644 index 0000000..c27f953 --- /dev/null +++ b/roles/github/README.md @@ -0,0 +1 @@ +# GitHub diff --git a/roles/github/tasks/main.yml b/roles/github/tasks/main.yml new file mode 100644 index 0000000..87c0728 --- /dev/null +++ b/roles/github/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Add GitHub Bot SSH key + ansible.builtin.copy: + src: "{{ inventory_dir }}/ota-github-bot-private-key" + dest: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' + owner: '{{ ansible_user }}' + group: '{{ ansible_user }}' + mode: "600" + +- name: Add GitHub SSH key to known_hosts + ansible.builtin.known_hosts: + name: github.com + key: "{{ item }}" + path: ~/.ssh/known_hosts + with_items: # GitHub's SSH key fingerprints can be found here: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints + - "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" + - "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" + - "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" + From ef2cd8cd485c92684e687894747d8c52f7cd59ee Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 6 May 2024 16:32:10 +0200 Subject: [PATCH 03/49] Create pm2 role --- roles/node/tasks/main.yml | 3 --- roles/pm2/README.md | 1 + roles/pm2/defaults/main.yml | 1 + roles/pm2/meta/main.yml | 3 +++ roles/pm2/tasks/main.yml | 22 ++++++++++++++++++++++ 5 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 roles/pm2/README.md create mode 100644 roles/pm2/defaults/main.yml create mode 100644 roles/pm2/meta/main.yml create mode 100644 roles/pm2/tasks/main.yml diff --git a/roles/node/tasks/main.yml b/roles/node/tasks/main.yml index a667e4f..f0df1bb 100644 --- a/roles/node/tasks/main.yml +++ b/roles/node/tasks/main.yml @@ -24,6 +24,3 @@ - name: Update NPM to latest version 10 ansible.builtin.command: npm install -g npm@10 - -- name: Install PM2 to latest version 5 - ansible.builtin.command: npm install -g pm2@5 --production=true diff --git a/roles/pm2/README.md b/roles/pm2/README.md new file mode 100644 index 0000000..3ccafa4 --- /dev/null +++ b/roles/pm2/README.md @@ -0,0 +1 @@ +# PM2 diff --git a/roles/pm2/defaults/main.yml b/roles/pm2/defaults/main.yml new file mode 100644 index 0000000..5b1b8ab --- /dev/null +++ b/roles/pm2/defaults/main.yml @@ -0,0 +1 @@ +ota_pm2_install_only: false diff --git a/roles/pm2/meta/main.yml b/roles/pm2/meta/main.yml new file mode 100644 index 0000000..d6477f8 --- /dev/null +++ b/roles/pm2/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - { role: common } + - { role: node } diff --git a/roles/pm2/tasks/main.yml b/roles/pm2/tasks/main.yml new file mode 100644 index 0000000..9fb2e17 --- /dev/null +++ b/roles/pm2/tasks/main.yml @@ -0,0 +1,22 @@ +- name: Install PM2 to latest version 5 + ansible.builtin.command: npm install -g pm2@5 --production=true + when: ota_pm2_install_only | bool + +- name: Stop Open Terms Archive applications + ansible.builtin.command: + cmd: pm2 stop pm2.config.cjs + chdir: '/home/{{ ansible_user }}/{{ ota_app_directory }}' + tags: + - stop + when: not ota_pm2_install_only | bool + +- name: Start Open Terms Archive applications + ansible.builtin.command: + cmd: pm2 startOrRestart pm2.config.cjs + chdir: '/home/{{ ansible_user }}/{{ ota_app_directory }}' + environment: + NODE_ENV: production + tags: + - restart + - start + when: not ota_pm2_install_only | bool From dd737170009450c3651061a40ba5e9d1328780e8 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 6 May 2024 16:34:21 +0200 Subject: [PATCH 04/49] Rewrite Nginx role --- roles/nginx/defaults/main.yml | 3 +++ roles/nginx/tasks/main.yml | 42 +++++++++++++++++++++-------- roles/nginx/templates/nginx.conf.j2 | 15 ----------- 3 files changed, 34 insertions(+), 26 deletions(-) create mode 100644 roles/nginx/defaults/main.yml delete mode 100644 roles/nginx/templates/nginx.conf.j2 diff --git a/roles/nginx/defaults/main.yml b/roles/nginx/defaults/main.yml new file mode 100644 index 0000000..b4d3bd0 --- /dev/null +++ b/roles/nginx/defaults/main.yml @@ -0,0 +1,3 @@ +ota_nginx_install: true +ota_nginx_configure: false +ota_nginx_configure_reverse_proxy: false diff --git a/roles/nginx/tasks/main.yml b/roles/nginx/tasks/main.yml index df9abd3..a3b71c2 100644 --- a/roles/nginx/tasks/main.yml +++ b/roles/nginx/tasks/main.yml @@ -4,18 +4,38 @@ name: nginx update_cache: true state: latest + when: ota_nginx_install | bool -- name: Clean default NGINX site - ansible.builtin.file: - state: absent - path: /etc/nginx/sites-enabled/default +- block: + - name: Clean default NGINX site + ansible.builtin.file: + state: absent + path: /etc/nginx/sites-enabled/default + + - name: Setup NGINX conf + ansible.builtin.template: + src: "{{ ota_nginx_config_template }}" + dest: /etc/nginx/conf.d/ota.conf + owner: root + group: root + mode: "644" notify: Restart NGINX + when: ota_nginx_configure | bool + +- block: + - name: Add conf in NGINX sites-available + ansible.builtin.template: + src: "{{ ota_nginx_reverse_proxy_config_template }}" + dest: '/etc/nginx/sites-available/ota' + force: true + mode: "644" -- name: Setup NGINX conf - ansible.builtin.template: - src: nginx.conf.j2 - dest: /etc/nginx/conf.d/ota.conf - owner: root - group: root - mode: "644" + - name: Link conf from sites-available to sites-enabled + ansible.builtin.file: + src: '/etc/nginx/sites-available/ota' + dest: '/etc/nginx/sites-enabled/ota' + state: link + force: true + become: true notify: Restart NGINX + when: ota_nginx_configure_reverse_proxy | bool diff --git a/roles/nginx/templates/nginx.conf.j2 b/roles/nginx/templates/nginx.conf.j2 deleted file mode 100644 index f866df4..0000000 --- a/roles/nginx/templates/nginx.conf.j2 +++ /dev/null @@ -1,15 +0,0 @@ -{{ ansible_managed | comment }} - -# By default, NGINX includes its version number in the response header, which -# can give potential attackers information about the software and version being -# used. This information can be useful for attackers to target specific -# vulnerabilities or exploits that are present in specific versions. -# Disabling server_tokens can help to mitigate this risk by removing the NGINX -# version number from the response headers. -server_tokens off; - -# Configure the rate limiting module to prevent DDoS attacks. -limit_req_zone $binary_remote_addr zone=limited:10m rate=10r/s; - -# Define HTTP response status code that is returned to clients when the rate limit is exceeded. -limit_req_status 429; From 4ada4810c40ccaae35f5420011cc5589ea71c42b Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 6 May 2024 16:36:37 +0200 Subject: [PATCH 05/49] Rewrite how OTA applications are deployed --- playbooks/deploy.yml | 72 ++++++++++++++++ playbooks/engine/all.yml | 6 -- playbooks/engine/application.yml | 14 --- playbooks/engine/infrastructure.yml | 25 ------ playbooks/engine_and_federated_api/all.yml | 6 -- .../engine_and_federated_api/application.yml | 43 ---------- .../infrastructure.yml | 7 -- playbooks/federated_api/all.yml | 6 -- playbooks/federated_api/application.yml | 6 -- playbooks/federated_api/infrastructure.yml | 8 -- .../nginx-reverse-proxy-conf.j2} | 16 ++-- playbooks/templates/nginx.conf.j2 | 15 ++++ roles/engine/defaults/main.yml | 5 -- roles/engine/handlers/main.yml | 6 -- roles/engine/tasks/main.yml | 86 ------------------- roles/engine/templates/.env | 11 --- roles/engine/templates/nginx-conf.j2 | 13 --- roles/engine/templates/pm2.config.j2 | 25 ------ roles/federated_api/README.md | 3 - roles/federated_api/defaults/main.yml | 3 - roles/federated_api/files/pm2.config.cjs | 9 -- roles/federated_api/handlers/main.yml | 6 -- roles/federated_api/tasks/main.yml | 69 --------------- roles/federated_api/templates/.env | 5 -- roles/federated_api/templates/nginx-conf.j2 | 13 --- roles/{engine => ota/applications}/README.md | 0 roles/ota/applications/defaults/main.yml | 1 + .../applications}/meta/main.yml | 1 + roles/ota/applications/tasks/main.yml | 30 +++++++ roles/ota/applications/templates/.env | 17 ++++ roles/ota/database/README.md | 2 + roles/ota/database/defaults/main.yml | 2 + roles/{engine => ota/database}/meta/main.yml | 1 + .../database}/tasks/database.yml | 2 +- roles/ota/database/tasks/main.yml | 17 ++++ 35 files changed, 168 insertions(+), 383 deletions(-) create mode 100644 playbooks/deploy.yml delete mode 100644 playbooks/engine/all.yml delete mode 100644 playbooks/engine/application.yml delete mode 100644 playbooks/engine/infrastructure.yml delete mode 100644 playbooks/engine_and_federated_api/all.yml delete mode 100644 playbooks/engine_and_federated_api/application.yml delete mode 100644 playbooks/engine_and_federated_api/infrastructure.yml delete mode 100644 playbooks/federated_api/all.yml delete mode 100644 playbooks/federated_api/application.yml delete mode 100644 playbooks/federated_api/infrastructure.yml rename playbooks/{engine_and_federated_api/templates/nginx-conf.j2 => templates/nginx-reverse-proxy-conf.j2} (53%) create mode 100644 playbooks/templates/nginx.conf.j2 delete mode 100644 roles/engine/defaults/main.yml delete mode 100644 roles/engine/handlers/main.yml delete mode 100644 roles/engine/tasks/main.yml delete mode 100644 roles/engine/templates/.env delete mode 100644 roles/engine/templates/nginx-conf.j2 delete mode 100644 roles/engine/templates/pm2.config.j2 delete mode 100644 roles/federated_api/README.md delete mode 100644 roles/federated_api/defaults/main.yml delete mode 100644 roles/federated_api/files/pm2.config.cjs delete mode 100644 roles/federated_api/handlers/main.yml delete mode 100644 roles/federated_api/tasks/main.yml delete mode 100644 roles/federated_api/templates/.env delete mode 100644 roles/federated_api/templates/nginx-conf.j2 rename roles/{engine => ota/applications}/README.md (100%) create mode 100644 roles/ota/applications/defaults/main.yml rename roles/{federated_api => ota/applications}/meta/main.yml (62%) create mode 100644 roles/ota/applications/tasks/main.yml create mode 100644 roles/ota/applications/templates/.env create mode 100644 roles/ota/database/README.md create mode 100644 roles/ota/database/defaults/main.yml rename roles/{engine => ota/database}/meta/main.yml (62%) rename roles/{engine => ota/database}/tasks/database.yml (93%) create mode 100644 roles/ota/database/tasks/main.yml diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml new file mode 100644 index 0000000..fb07ab9 --- /dev/null +++ b/playbooks/deploy.yml @@ -0,0 +1,72 @@ +--- +- name: Deploy the Open Terms Archive applications + hosts: all + vars_files: "{{ inventory_dir }}/secrets.yml" + tasks: + - name: Load OTA applications configs + ansible.builtin.include_role: + name: ota/applications + vars: + ota_app_read_config_only: true + + - block: + - name: Install Node + ansible.builtin.include_role: + name: node + + - name: Install PM2 + ansible.builtin.include_role: + name: pm2 + vars: + ota_pm2_install_only: true + + - name: Install Chromium + ansible.builtin.include_role: + name: chromium + when: ota_apps_config['@opentermsarchive/engine'] is defined + + - name: Install and setup Nginx + ansible.builtin.include_role: + name: nginx + vars: + ota_nginx_configure: true + ota_nginx_config_template: ./templates/nginx.conf.j2 + + - name: Install and setup Mongo + ansible.builtin.include_role: + name: mongo + when: + - (ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type == 'mongo') or + (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') + # Skip Debian 11 with ARM architecture as it is not currently supported by MongoDB. + # See https://www.mongodb.com/docs/manual/installation/#supported-platforms + - ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64') + become: true + tags: + - infrastructure + + - name: Setup OTA snapshots and versions databases + ansible.builtin.include_role: + name: ota/database + when: + - ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository is defined or + ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository is defined + + - name: Setup OTA applications + ansible.builtin.include_role: + name: ota/applications + + - name: Start OTA applications + ansible.builtin.include_role: + name: pm2 + + - name: Setup reverse proxy + ansible.builtin.include_role: + name: nginx + apply: + become: true + vars: + ota_nginx_install: false + ota_nginx_configure: false + ota_nginx_configure_reverse_proxy: true + ota_nginx_reverse_proxy_config_template: ./templates/nginx-reverse-proxy-conf.j2 diff --git a/playbooks/engine/all.yml b/playbooks/engine/all.yml deleted file mode 100644 index f361d0d..0000000 --- a/playbooks/engine/all.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Set up infrastructure and deploy the Open Terms Archive engine - hosts: all - -- import_playbook: infrastructure.yml -- import_playbook: application.yml diff --git a/playbooks/engine/application.yml b/playbooks/engine/application.yml deleted file mode 100644 index 4bdad1d..0000000 --- a/playbooks/engine/application.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Deploy the Open Terms Archive engine - hosts: all - - tasks: - - name: Load the production config - ansible.builtin.include_vars: - name: ota_engine_app_config - file: "{{ inventory_dir }}/{{ ota_engine_config_path | default('../config/production.json') }}" - tags: always - - - ansible.builtin.include_role: - name: engine - tags: always diff --git a/playbooks/engine/infrastructure.yml b/playbooks/engine/infrastructure.yml deleted file mode 100644 index 13d9a31..0000000 --- a/playbooks/engine/infrastructure.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- name: Set up infrastructure - hosts: all - become: true - - tasks: - - name: Load OTA engine config - ansible.builtin.include_vars: - name: app_config - file: "{{ inventory_dir }}/{{ ota_engine_config_path | default('../config/production.json') }}" - tags: always - - - ansible.builtin.include_role: - name: mongo - when: - - (app_config.recorder.versions.storage.type is defined and app_config.recorder.versions.storage.type == 'mongo') or - (app_config.recorder.snapshots.storage.type is defined and app_config.recorder.snapshots.storage.type == 'mongo') - # Skip Debian 11 with ARM architecture as it is not currently supported by MongoDB. - # See https://www.mongodb.com/docs/manual/installation/#supported-platforms - - ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64') - - roles: - - role: node - - role: chromium - - role: nginx diff --git a/playbooks/engine_and_federated_api/all.yml b/playbooks/engine_and_federated_api/all.yml deleted file mode 100644 index 952cef7..0000000 --- a/playbooks/engine_and_federated_api/all.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Set up infrastructure and deploy the Open Terms Archive engine and federated API - hosts: all - -- import_playbook: infrastructure.yml -- import_playbook: application.yml diff --git a/playbooks/engine_and_federated_api/application.yml b/playbooks/engine_and_federated_api/application.yml deleted file mode 100644 index dc3b46b..0000000 --- a/playbooks/engine_and_federated_api/application.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- name: Deploy the Open Terms Archive engine and federated API - hosts: all - vars: - ota_reverse_proxy_engine_path: "/collection-api" - ota_reverse_proxy_federated_api_path: "/federation-api" - tasks: - - block: - - name: Load the engine production config - ansible.builtin.include_vars: - name: ota_engine_app_config - file: "{{ inventory_dir }}/{{ ota_engine_config_path | default('../config/production.json') }}" - - - ansible.builtin.include_role: - name: engine - - - ansible.builtin.include_role: - name: federated_api - tags: always - - - block: - - name: Add conf in NGINX sites-available - ansible.builtin.template: - src: nginx-conf.j2 - dest: '/etc/nginx/sites-available/ota' - force: true - mode: "644" - - - name: Link conf from sites-available to sites-enabled - ansible.builtin.file: - src: '/etc/nginx/sites-available/ota' - dest: '/etc/nginx/sites-enabled/ota' - state: link - force: true - become: true - notify: Restart NGINX - - handlers: - - name: Restart NGINX - become: true - ansible.builtin.service: - name: nginx - state: restarted diff --git a/playbooks/engine_and_federated_api/infrastructure.yml b/playbooks/engine_and_federated_api/infrastructure.yml deleted file mode 100644 index 52b6fd6..0000000 --- a/playbooks/engine_and_federated_api/infrastructure.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Set up infrastructure - hosts: all - become: true - -- ansible.builtin.import_playbook: ../engine/infrastructure.yml -- ansible.builtin.import_playbook: ../federated_api/infrastructure.yml diff --git a/playbooks/federated_api/all.yml b/playbooks/federated_api/all.yml deleted file mode 100644 index d4a58bc..0000000 --- a/playbooks/federated_api/all.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Set up infrastructure and deploy the Open Terms Archive federated API - hosts: all - -- import_playbook: infrastructure.yml -- import_playbook: application.yml diff --git a/playbooks/federated_api/application.yml b/playbooks/federated_api/application.yml deleted file mode 100644 index 559efdb..0000000 --- a/playbooks/federated_api/application.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Deploy the Open Terms Archive federated API - hosts: all - - roles: - - role: federated_api diff --git a/playbooks/federated_api/infrastructure.yml b/playbooks/federated_api/infrastructure.yml deleted file mode 100644 index 005b76c..0000000 --- a/playbooks/federated_api/infrastructure.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: Set up infrastructure - hosts: all - become: true - - roles: - - role: node - - role: nginx diff --git a/playbooks/engine_and_federated_api/templates/nginx-conf.j2 b/playbooks/templates/nginx-reverse-proxy-conf.j2 similarity index 53% rename from playbooks/engine_and_federated_api/templates/nginx-conf.j2 rename to playbooks/templates/nginx-reverse-proxy-conf.j2 index b356620..2633873 100644 --- a/playbooks/engine_and_federated_api/templates/nginx-conf.j2 +++ b/playbooks/templates/nginx-reverse-proxy-conf.j2 @@ -3,20 +3,22 @@ server { listen 80; server_name {{ inventory_hostname }}; - - location {{ ota_reverse_proxy_engine_path }} { + +{% if ota_apps_config['@opentermsarchive/engine']['collection-api'].basePath is defined and ota_apps_config['@opentermsarchive/engine']['collection-api'].port is defined %} + location {{ ota_apps_config['@opentermsarchive/engine']['collection-api'].basePath }} { # Allowing for a `burst` of up to 5 requests beyond the specified rate limit. The `nodelay` parameter ensures that excessive requests beyond the burst limit are immediately rejected with a 429 error response instead of being queued. See https://www.nginx.com/blog/rate-limiting-nginx/. limit_req zone=limited burst=5 nodelay; - rewrite ^{{ ota_reverse_proxy_engine_path }}/(.*)$ /$1 break; - proxy_pass http://localhost:{{ ota_engine_app_config.api.port }}; + proxy_pass http://localhost:{{ ota_apps_config['@opentermsarchive/engine']['collection-api'].port }}; proxy_redirect off; } +{% endif %} - location {{ ota_reverse_proxy_federated_api_path }} { +{% if ota_apps_config['@opentermsarchive/federated-api'].basePath is defined and ota_apps_config['@opentermsarchive/federated-api'].port is defined %} + location {{ ota_apps_config['@opentermsarchive/federated-api'].basePath }} { # Allowing for a `burst` of up to 5 requests beyond the specified rate limit. The `nodelay` parameter ensures that excessive requests beyond the burst limit are immediately rejected with a 429 error response instead of being queued. See https://www.nginx.com/blog/rate-limiting-nginx/. limit_req zone=limited burst=5 nodelay; - rewrite ^{{ ota_reverse_proxy_federated_api_path }}/(.*)$ /$1 break; - proxy_pass http://localhost:{{ ota_federated_api_app_config.port }}; + proxy_pass http://localhost:{{ ota_apps_config['@opentermsarchive/federated-api'].port }}; proxy_redirect off; } +{% endif %} } diff --git a/playbooks/templates/nginx.conf.j2 b/playbooks/templates/nginx.conf.j2 new file mode 100644 index 0000000..f866df4 --- /dev/null +++ b/playbooks/templates/nginx.conf.j2 @@ -0,0 +1,15 @@ +{{ ansible_managed | comment }} + +# By default, NGINX includes its version number in the response header, which +# can give potential attackers information about the software and version being +# used. This information can be useful for attackers to target specific +# vulnerabilities or exploits that are present in specific versions. +# Disabling server_tokens can help to mitigate this risk by removing the NGINX +# version number from the response headers. +server_tokens off; + +# Configure the rate limiting module to prevent DDoS attacks. +limit_req_zone $binary_remote_addr zone=limited:10m rate=10r/s; + +# Define HTTP response status code that is returned to clients when the rate limit is exceeded. +limit_req_status 429; diff --git a/roles/engine/defaults/main.yml b/roles/engine/defaults/main.yml deleted file mode 100644 index 3b0782b..0000000 --- a/roles/engine/defaults/main.yml +++ /dev/null @@ -1,5 +0,0 @@ -ota_engine_declarations_directory: "{{ ota_engine_app_config.name }}" -ota_engine_declarations_branch: main -ota_engine_snapshots_branch: main -ota_engine_versions_branch: main -ota_engine_restart_delay: 10800000 # 3 hours in ms diff --git a/roles/engine/handlers/main.yml b/roles/engine/handlers/main.yml deleted file mode 100644 index 347479f..0000000 --- a/roles/engine/handlers/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Restart NGINX - become: true - ansible.builtin.service: - name: nginx - state: restarted diff --git a/roles/engine/tasks/main.yml b/roles/engine/tasks/main.yml deleted file mode 100644 index 68cf46b..0000000 --- a/roles/engine/tasks/main.yml +++ /dev/null @@ -1,86 +0,0 @@ -- name: Install services declarations - ansible.builtin.git: - repo: '{{ ota_engine_app_config.services.repository }}' - dest: '/home/{{ ansible_user }}/{{ ota_engine_declarations_directory }}' - version: '{{ ota_engine_declarations_branch }}' - force: true - key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' - depth: 1 - tags: - - update-declarations - -- name: Install services declarations dependencies - ansible.builtin.command: - cmd: npm ci - chdir: '/home/{{ ansible_user }}/{{ ota_engine_declarations_directory }}' - tags: - - update-declarations - -- name: Add .env file - ansible.builtin.template: - src: .env - dest: '/home/{{ ansible_user }}/{{ ota_engine_declarations_directory }}/.env' - force: true - mode: "644" - -- name: Add pm2 config file - ansible.builtin.template: - src: pm2.config.j2 - dest: '/home/{{ ansible_user }}/{{ ota_engine_declarations_directory }}/pm2.config.cjs' - force: true - mode: "644" - -- name: Stop Open Terms Archive schedulers - ansible.builtin.command: - cmd: pm2 stop pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_engine_declarations_directory }}' - tags: - - stop - - update-declarations - -- name: Add conf in NGINX sites-available - become: true - ansible.builtin.template: - src: nginx-conf.j2 - dest: '/etc/nginx/sites-available/ota' - force: true - mode: "644" - notify: Restart NGINX - -- name: Link conf from sites-available to sites-enabled - become: true - ansible.builtin.file: - src: '/etc/nginx/sites-available/ota' - dest: '/etc/nginx/sites-enabled/ota' - state: link - force: true - notify: Restart NGINX - -- name: Setup snapshots git repository - ansible.builtin.include_tasks: database.yml - when: ota_engine_app_config.recorder.snapshots.storage.git.repository is defined - vars: - engine_database_name: snapshots - engine_database_repository: '{{ ota_engine_app_config.recorder.snapshots.storage.git.repository }}' - engine_database_branch: '{{ ota_engine_snapshots_branch }}' - engine_database_directory: '/home/{{ ansible_user }}/{{ ota_engine_declarations_directory }}/{{ ota_engine_app_config.recorder.snapshots.storage.git.path }}' - -- name: Setup versions git repository - ansible.builtin.include_tasks: database.yml - when: ota_engine_app_config.recorder.versions.storage.git.repository is defined - vars: - engine_database_name: versions - engine_database_repository: '{{ ota_engine_app_config.recorder.versions.storage.git.repository }}' - engine_database_branch: '{{ ota_engine_versions_branch }}' - engine_database_directory: '/home/{{ ansible_user }}/{{ ota_engine_declarations_directory }}/{{ ota_engine_app_config.recorder.versions.storage.git.path }}' - -- name: Start Open Terms Archive schedulers - ansible.builtin.command: - cmd: pm2 startOrRestart pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_engine_declarations_directory }}' - environment: - NODE_ENV: production - tags: - - restart - - start - - update-declarations diff --git a/roles/engine/templates/.env b/roles/engine/templates/.env deleted file mode 100644 index 85472f9..0000000 --- a/roles/engine/templates/.env +++ /dev/null @@ -1,11 +0,0 @@ -{{ ansible_managed | comment }} - -{% if ota_engine_sendinblue_api_key is defined %} -SENDINBLUE_API_KEY={{ ota_engine_sendinblue_api_key }} -{% endif %} -{% if ota_engine_smtp_password is defined %} -SMTP_PASSWORD={{ ota_engine_smtp_password }} -{% endif %} -{% if ota_engine_github_token is defined %} -GITHUB_TOKEN={{ ota_engine_github_token }} -{% endif %} diff --git a/roles/engine/templates/nginx-conf.j2 b/roles/engine/templates/nginx-conf.j2 deleted file mode 100644 index 654911d..0000000 --- a/roles/engine/templates/nginx-conf.j2 +++ /dev/null @@ -1,13 +0,0 @@ -{{ ansible_managed | comment }} - -server { - listen 80; - server_name {{ inventory_hostname }}; - - location / { - # Allowing for a `burst` of up to 5 requests beyond the specified rate limit. The `nodelay` parameter ensures that excessive requests beyond the burst limit are immediately rejected with a 429 error response instead of being queued. See https://www.nginx.com/blog/rate-limiting-nginx/. - limit_req zone=limited burst=5 nodelay; - proxy_pass http://localhost:{{ ota_engine_app_config.api.port }}; - proxy_redirect off; - } -} diff --git a/roles/engine/templates/pm2.config.j2 b/roles/engine/templates/pm2.config.j2 deleted file mode 100644 index cc994e0..0000000 --- a/roles/engine/templates/pm2.config.j2 +++ /dev/null @@ -1,25 +0,0 @@ -module.exports = { - apps: [ - { - name: 'ota', - script: 'npm', - args: 'run start:schedule', - max_restarts: 2, - min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. - restart_delay: {{ ota_engine_restart_delay }}, - }, - { - name: 'ota-api', - script: 'npm', - args: 'run start:api', - }, - { - name: 'ota-release', - script: 'npm', - args: 'run dataset:schedule', - max_restarts: 2, - min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. - restart_delay: 15 * 60 * 1000, - } - ], -}; diff --git a/roles/federated_api/README.md b/roles/federated_api/README.md deleted file mode 100644 index 4320fc0..0000000 --- a/roles/federated_api/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Federated API - -See https://github.com/OpenTermsArchive/federated-api/ diff --git a/roles/federated_api/defaults/main.yml b/roles/federated_api/defaults/main.yml deleted file mode 100644 index 28f3bea..0000000 --- a/roles/federated_api/defaults/main.yml +++ /dev/null @@ -1,3 +0,0 @@ -ota_federated_api_repo: https://github.com/OpenTermsArchive/federated-api -ota_federated_api_directory: federated-api -ota_federated_api_branch: main diff --git a/roles/federated_api/files/pm2.config.cjs b/roles/federated_api/files/pm2.config.cjs deleted file mode 100644 index 65de2c1..0000000 --- a/roles/federated_api/files/pm2.config.cjs +++ /dev/null @@ -1,9 +0,0 @@ -module.exports = { - apps: [ - { - name: 'ota-federated-api', - script: 'npm', - args: 'run start', - } - ], -}; diff --git a/roles/federated_api/handlers/main.yml b/roles/federated_api/handlers/main.yml deleted file mode 100644 index 347479f..0000000 --- a/roles/federated_api/handlers/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Restart NGINX - become: true - ansible.builtin.service: - name: nginx - state: restarted diff --git a/roles/federated_api/tasks/main.yml b/roles/federated_api/tasks/main.yml deleted file mode 100644 index 72957fc..0000000 --- a/roles/federated_api/tasks/main.yml +++ /dev/null @@ -1,69 +0,0 @@ -- name: Clone federated API repository - ansible.builtin.git: - repo: '{{ ota_federated_api_repo }}' - dest: '/home/{{ ansible_user }}/{{ ota_federated_api_directory }}' - version: '{{ ota_federated_api_branch }}' - force: true - key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' - depth: 1 - -- name: Read the production config - shell: cat '/home/{{ ansible_user }}/{{ ota_federated_api_directory }}/config/production.json' - register: read_config - -- name: Save production config data into a variable - set_fact: - ota_federated_api_app_config: "{{ read_config.stdout | from_json }}" - -- name: Install dependencies - ansible.builtin.command: - cmd: npm ci - chdir: '/home/{{ ansible_user }}/{{ ota_federated_api_directory }}' - -- name: Add .env file - ansible.builtin.template: - src: .env - dest: '/home/{{ ansible_user }}/{{ ota_federated_api_directory }}/.env' - force: true - mode: "644" - -- name: Add pm2 config file - ansible.builtin.copy: - src: pm2.config.cjs - dest: '/home/{{ ansible_user }}/{{ ota_federated_api_directory }}/pm2.config.cjs' - mode: "644" - -- name: Stop federated API - ansible.builtin.command: - cmd: pm2 stop pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_federated_api_directory }}' - tags: - - stop - -- name: Add conf in NGINX sites-available - become: true - ansible.builtin.template: - src: nginx-conf.j2 - dest: '/etc/nginx/sites-available/ota' - force: true - mode: "644" - notify: Restart NGINX - -- name: Link conf from sites-available to sites-enabled - become: true - ansible.builtin.file: - src: '/etc/nginx/sites-available/ota' - dest: '/etc/nginx/sites-enabled/ota' - state: link - force: true - notify: Restart NGINX - -- name: Start federated API - ansible.builtin.command: - cmd: pm2 startOrRestart pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_federated_api_directory }}' - environment: - NODE_ENV: production - tags: - - restart - - start diff --git a/roles/federated_api/templates/.env b/roles/federated_api/templates/.env deleted file mode 100644 index 9daa89c..0000000 --- a/roles/federated_api/templates/.env +++ /dev/null @@ -1,5 +0,0 @@ -{{ ansible_managed | comment }} - -{% if ota_federated_api_smtp_password is defined %} -SMTP_PASSWORD={{ ota_federated_api_smtp_password }} -{% endif %} diff --git a/roles/federated_api/templates/nginx-conf.j2 b/roles/federated_api/templates/nginx-conf.j2 deleted file mode 100644 index 8ac6285..0000000 --- a/roles/federated_api/templates/nginx-conf.j2 +++ /dev/null @@ -1,13 +0,0 @@ -{{ ansible_managed | comment }} - -server { - listen 80; - server_name {{ inventory_hostname }}; - - location / { - # Allowing for a `burst` of up to 5 requests beyond the specified rate limit. The `nodelay` parameter ensures that excessive requests beyond the burst limit are immediately rejected with a 429 error response instead of being queued. See https://www.nginx.com/blog/rate-limiting-nginx/. - limit_req zone=limited burst=5 nodelay; - proxy_pass http://localhost:{{ ota_federated_api_app_config.port }}; - proxy_redirect off; - } -} diff --git a/roles/engine/README.md b/roles/ota/applications/README.md similarity index 100% rename from roles/engine/README.md rename to roles/ota/applications/README.md diff --git a/roles/ota/applications/defaults/main.yml b/roles/ota/applications/defaults/main.yml new file mode 100644 index 0000000..fc329a3 --- /dev/null +++ b/roles/ota/applications/defaults/main.yml @@ -0,0 +1 @@ +ota_app_read_config_only: false diff --git a/roles/federated_api/meta/main.yml b/roles/ota/applications/meta/main.yml similarity index 62% rename from roles/federated_api/meta/main.yml rename to roles/ota/applications/meta/main.yml index 4c5a271..d0c083a 100644 --- a/roles/federated_api/meta/main.yml +++ b/roles/ota/applications/meta/main.yml @@ -1,2 +1,3 @@ dependencies: + - { role: common } - { role: github } diff --git a/roles/ota/applications/tasks/main.yml b/roles/ota/applications/tasks/main.yml new file mode 100644 index 0000000..ad61a87 --- /dev/null +++ b/roles/ota/applications/tasks/main.yml @@ -0,0 +1,30 @@ +- name: Clone repository + ansible.builtin.git: + repo: '{{ ota_app_repository }}' + dest: '/home/{{ ansible_user }}/{{ ota_app_directory }}' + version: '{{ ota_app_repository_branch }}' + force: true + key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' + depth: 1 + +- name: Read the production config + shell: cat '/home/{{ ansible_user }}/{{ ota_app_directory }}/config/production.json' + register: read_config + +- name: Save production config data into a variable + set_fact: + ota_apps_config: "{{ read_config.stdout | from_json }}" + +- block: + - name: Install dependencies + ansible.builtin.command: + cmd: npm install ci + chdir: '/home/{{ ansible_user }}/{{ ota_app_directory }}' + + - name: Add .env file + ansible.builtin.template: + src: .env + dest: '/home/{{ ansible_user }}/{{ ota_app_directory }}/.env' + force: true + mode: "644" + when: not ota_app_read_config_only | bool diff --git a/roles/ota/applications/templates/.env b/roles/ota/applications/templates/.env new file mode 100644 index 0000000..824c8ad --- /dev/null +++ b/roles/ota/applications/templates/.env @@ -0,0 +1,17 @@ +{{ ansible_managed | comment }} + +{% if OTA_ENGINE_SENDINBLUE_API_KEY is defined %} +OTA_ENGINE_SENDINBLUE_API_KEY={{ OTA_ENGINE_SENDINBLUE_API_KEY }} +{% endif %} + +{% if OTA_ENGINE_SMTP_PASSWORD is defined %} +OTA_ENGINE_SMTP_PASSWORD={{ OTA_ENGINE_SMTP_PASSWORD }} +{% endif %} + +{% if OTA_ENGINE_GITHUB_TOKEN is defined %} +OTA_ENGINE_GITHUB_TOKEN={{ OTA_ENGINE_GITHUB_TOKEN }} +{% endif %} + +{% if OTA_FEDERATED_API_SMTP_PASSWORD is defined %} +OTA_FEDERATED_API_SMTP_PASSWORD={{ OTA_FEDERATED_API_SMTP_PASSWORD }} +{% endif %} diff --git a/roles/ota/database/README.md b/roles/ota/database/README.md new file mode 100644 index 0000000..efebbdb --- /dev/null +++ b/roles/ota/database/README.md @@ -0,0 +1,2 @@ +# Engine + diff --git a/roles/ota/database/defaults/main.yml b/roles/ota/database/defaults/main.yml new file mode 100644 index 0000000..9b0f865 --- /dev/null +++ b/roles/ota/database/defaults/main.yml @@ -0,0 +1,2 @@ +ota_database_snapshots_branch: main +ota_database_versions_branch: main diff --git a/roles/engine/meta/main.yml b/roles/ota/database/meta/main.yml similarity index 62% rename from roles/engine/meta/main.yml rename to roles/ota/database/meta/main.yml index 4c5a271..d0c083a 100644 --- a/roles/engine/meta/main.yml +++ b/roles/ota/database/meta/main.yml @@ -1,2 +1,3 @@ dependencies: + - { role: common } - { role: github } diff --git a/roles/engine/tasks/database.yml b/roles/ota/database/tasks/database.yml similarity index 93% rename from roles/engine/tasks/database.yml rename to roles/ota/database/tasks/database.yml index d9047a3..a83a01b 100644 --- a/roles/engine/tasks/database.yml +++ b/roles/ota/database/tasks/database.yml @@ -1,4 +1,4 @@ -- name: Check if {{ engine_database_name }} base data has already been obtained from {{ engine_database_repository }} +- name: "Check if {{ engine_database_name }} base data has already been obtained from {{ engine_database_repository }}" ansible.builtin.git: repo: '{{ engine_database_repository }}' version: '{{ engine_database_branch }}' diff --git a/roles/ota/database/tasks/main.yml b/roles/ota/database/tasks/main.yml new file mode 100644 index 0000000..fdf51cd --- /dev/null +++ b/roles/ota/database/tasks/main.yml @@ -0,0 +1,17 @@ +- name: Setup snapshots git repository + ansible.builtin.include_tasks: database.yml + when: ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository is defined + vars: + engine_database_name: snapshots + engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository }}" + engine_database_branch: "{{ ota_database_snapshots_branch }}" + engine_database_directory: "/home/{{ ansible_user }}/{{ ota_app_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }}" + +- name: Setup versions git repository + ansible.builtin.include_tasks: database.yml + when: ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository is defined + vars: + engine_database_name: versions + engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository }}" + engine_database_branch: "{{ ota_database_versions_branch }}" + engine_database_directory: "/home/{{ ansible_user }}/{{ ota_app_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }}" From aabe4d5f0f70974d78c861cc5076f239d12c595f Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 6 May 2024 17:14:15 +0200 Subject: [PATCH 06/49] Update tests --- tests/engine_config.json | 32 ------------------------------- tests/inventory.yml | 31 +++--------------------------- tests/ota-github-bot-private-key | 26 +++++++++++++++++++++++++ tests/secrets.yml | 33 ++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 60 deletions(-) delete mode 100644 tests/engine_config.json create mode 100644 tests/ota-github-bot-private-key create mode 100644 tests/secrets.yml diff --git a/tests/engine_config.json b/tests/engine_config.json deleted file mode 100644 index f78d16d..0000000 --- a/tests/engine_config.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "vagrant", - "services": { - "repository": "https://github.com/OpenTermsArchive/sandbox-declarations.git" - }, - "recorder": { - "versions": { - "storage": { - "git": { - "repository": "git@github.com:OpenTermsArchive/sandbox-versions.git", - "path": "./data/versions", - "publish": false - } - } - }, - "snapshots": { - "storage": { - "git": { - "repository": "git@github.com:OpenTermsArchive/sandbox-snapshots.git", - "path": "./data/snapshots", - "publish": false - } - } - } - }, - "logger": { - "sendMailOnError": false - }, - "api": { - "port": 3000 - } -} diff --git a/tests/inventory.yml b/tests/inventory.yml index 61caf9b..e7c98ca 100644 --- a/tests/inventory.yml +++ b/tests/inventory.yml @@ -5,31 +5,6 @@ vagrant: ansible_port: 2222 ansible_python_interpreter: /usr/bin/python3 ansible_ssh_private_key_file: ~/.ssh/ota-vagrant - ota_engine_config_path: ./engine_config.json - ota_engine_github_bot_private_key: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 63366161663933343964333231666663356335616231393562366266353139653032376635373036 - 3233356266633764663963666564366633346561633633300a333865393538363665383936323336 - 63326432396664643962303737656430363534326466336238613432343231626333303637616137 - 3065333331626234370a653962326564646534616238626538383436396234653234333838633064 - 30326166376664356437386366313162396634313562383564316166663038306361396365303833 - 38316161326432643735376236333532386162383662656130313430313338323236626132646136 - 30643232353765376664303265343836363330373062373535333665383230656662376362636436 - 37393135393162396665616234666137356539363663373130306136346135666133666162333463 - 31303133386663366662303137326435383432303932353863636131373732313737306232306330 - 35303732316338343035316332353262386533333335653732343230393562623766356263653562 - 61343131343532346633323538636366323633396534663735333333653139386335636632663664 - 62626166613130396163373039666630383231653461323138396366373065346335313837643563 - 35353139373564343165623063386338343639333363336466356261396436376266323861303938 - 38333339386164343261316161633131303833363762333733393865626163373236313364656464 - 33346335656233333064653733666332656337626665646161306566356235616438363265316136 - 39663363393332396264323831643037363237303938353235636165633937663964346139633364 - 38626632343139366464613734313334646335386364666438303336306562333337306639333637 - 61396564353430653932643535616461393562346633643830303766376332323333643235343135 - 34323333616466663736323164313036623038376134623662386431306163666430393132623361 - 66346265383435383164313335303262316238623530613737346661373837623939636262653233 - 64363764613537633538316531303063633132393362323034346166306361653661653236633133 - 33316437376364656130323936356662326162323565663433326530343064643866373331616534 - 63313139393761646335366464386638666430346136373561643761633130663736633132633333 - 65636336376633393030333133633733306231666638613034313639666462653563656634633739 - 66383964306235666233363665633333373766353663653237636332306330393130 + + ota_app_repository: https://github.com/OpenTermsArchive/demo-declarations.git + ota_app_directory: demo diff --git a/tests/ota-github-bot-private-key b/tests/ota-github-bot-private-key new file mode 100644 index 0000000..160a9e4 --- /dev/null +++ b/tests/ota-github-bot-private-key @@ -0,0 +1,26 @@ +$ANSIBLE_VAULT;1.1;AES256 +39326264316331366661316164613539613733613336363930356335636434383532336136363266 +6431383230663031663139333533376664393135353231620a633632326139353439636530653966 +35326135336534323532323237393736613239323239623963333937353263376437613635356334 +3133616165646236640a363331656333336464663563303932313937313764633036363133306630 +66646637326564323931653363383130383233656634636330303262386264643439613132626439 +63653935353834626332333665616462633135653863663431633434386530636536656162396532 +64393862366537336366353639623236616434323139376563353932623765386432613463313034 +36353634666436383635333865363835346136613762373461316666386138666331636338646638 +38316465663332333737313465306330373262363761373766366232613233313063666262643930 +39343535386265333832303465376263336436663362623930666264383836396633376535643538 +31346434613635383930643737323265346539663461316636393137636236336135303833643166 +39636132393235333961643262373130376537653036393731633736613864306430646639313366 +32363166653731343830363666653963666266613162396539353965323336373439336231346131 +64653338366166336130366661653163623636623732336431643064363731333236356162663832 +35656339623335303262386633636336393661626564336339376537666630336432663062636339 +61383563653235393761313633346337343830333837386265343037396532303961616166666530 +62653634613962313539633331376464306461376131363366663038663535373139613936626136 +63363439643163326564363233316430643837323262626639353933333833646531353130646563 +38323465323434383566616263386265316665306536353663633438383761613033316161623638 +34373763663134653564663462363663643334643137333737323362623036663032313364636634 +36326330306366386464653639616664383036326433663564353834393431363433396165643166 +32396263303839613362643836353061333164623635663266666566393239346532323137383737 +61613362656161346130333362373766626263393734303262346466653764306432336165323239 +35383837383234393966666361316631353436636665646231663531653033316637386134333030 +37386534366561303734316632333936346331626634306335366665636333656433 diff --git a/tests/secrets.yml b/tests/secrets.yml new file mode 100644 index 0000000..e39bd48 --- /dev/null +++ b/tests/secrets.yml @@ -0,0 +1,33 @@ +OTA_ENGINE_SMTP_PASSWORD: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 31336232636137366331316132356539623061306138323465613662633336393536363932373831 + 3633346231366163353166396364333135623462393939340a356636346433623963633837323239 + 63653665616230383965346663363732386665663463373665633736303030326463393137303433 + 3363396638383438640a623261393130613464663031663935373332636239343465646163313037 + 63613634376133656632333239356332633036623736643162323863666634343262376435363662 + 38343665363462653961663138333139646231633336623961633965303166663362666532363663 + 33396663353762646265656138373637343464326666383138643431346165356436353766363337 + 31333431663364343064623762396563386230393763346131333535316637373939316437616237 + 3938 +OTA_ENGINE_GITHUB_TOKEN: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 31666336386433366234346638633536333835383736626638313263366332623766336537376664 + 3339393538613034363738363037623533626435333663380a393434613630613033653838303836 + 31663437343730313030653331653335343361663634306432633166636163313665633937393762 + 6263623932643439350a646535663638643635333930396438303635303531383431646539336533 + 65343835356534366464323566653734386466633431383464353233346438623363353930363237 + 63356536326362616337373965653032353661323535366539336632323764613836383264393935 + 30323531316235653466333563666232646262336165366166323934373865623066326235393236 + 66616235656261343139616235306231396632643838323436383830663638663039366564623939 + 3833 +OTA_FEDERATED_API_SMTP_PASSWORD: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 31336232636137366331316132356539623061306138323465613662633336393536363932373831 + 3633346231366163353166396364333135623462393939340a356636346433623963633837323239 + 63653665616230383965346663363732386665663463373665633736303030326463393137303433 + 3363396638383438640a623261393130613464663031663935373332636239343465646163313037 + 63613634376133656632333239356332633036623736643162323863666634343262376435363662 + 38343665363462653961663138333139646231633336623961633965303166663362666532363663 + 33396663353762646265656138373637343464326666383138643431346165356436353766363337 + 31333431663364343064623762396563386230393763346131333535316637373939316437616237 + 3938 From 1d0c9c5d6f010934052b5596bad952745f421d3c Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 6 May 2024 17:15:13 +0200 Subject: [PATCH 07/49] Expose VM remote port 80 to host 8080 to test APIs --- tests/Vagrantfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/Vagrantfile b/tests/Vagrantfile index a8e6433..f259f20 100644 --- a/tests/Vagrantfile +++ b/tests/Vagrantfile @@ -1,6 +1,9 @@ # -*- mode: ruby -*- # vi: set ft=ruby : +guest_port = 80 +host_port = ENV["PORT"] || 8080 + Vagrant.configure("2") do |config| config.vm.hostname = "vagrant" @@ -10,6 +13,8 @@ Vagrant.configure("2") do |config| # if necessary, create the key with `ssh-keygen -f ~/.ssh/ota-vagrant -q -N ""` # CAUTION: use of `~` in path causes problems with ssh config.vm.provision "file", source: File.join(ENV['HOME'], ".ssh", "ota-vagrant.pub"), destination: "/home/vagrant/.ssh/authorized_keys" + + config.vm.network "forwarded_port", guest: guest_port, host: host_port # based on https://github.com/rofrano/vagrant-docker-provider#example-vagrantfile config.vm.provider :docker do |docker, override| From 602986b66ac673f87376e823e42fb09bb51742c5 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 15 May 2024 10:40:57 +0200 Subject: [PATCH 08/49] Rename ota/applications in ota/apps --- playbooks/deploy.yml | 6 +-- roles/ota/applications/defaults/main.yml | 1 - roles/ota/applications/tasks/main.yml | 30 --------------- roles/ota/applications/templates/.env | 17 --------- roles/ota/{applications => apps}/README.md | 0 roles/ota/apps/defaults/main.yml | 3 ++ .../ota/{applications => apps}/meta/main.yml | 0 roles/ota/apps/tasks/main.yml | 37 +++++++++++++++++++ roles/ota/database/tasks/main.yml | 4 +- roles/pm2/tasks/main.yml | 4 +- 10 files changed, 47 insertions(+), 55 deletions(-) delete mode 100644 roles/ota/applications/defaults/main.yml delete mode 100644 roles/ota/applications/tasks/main.yml delete mode 100644 roles/ota/applications/templates/.env rename roles/ota/{applications => apps}/README.md (100%) create mode 100644 roles/ota/apps/defaults/main.yml rename roles/ota/{applications => apps}/meta/main.yml (100%) create mode 100644 roles/ota/apps/tasks/main.yml diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index fb07ab9..4162be3 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -5,9 +5,9 @@ tasks: - name: Load OTA applications configs ansible.builtin.include_role: - name: ota/applications + name: ota/apps vars: - ota_app_read_config_only: true + ota_apps_read_config_only: true - block: - name: Install Node @@ -54,7 +54,7 @@ - name: Setup OTA applications ansible.builtin.include_role: - name: ota/applications + name: ota/apps - name: Start OTA applications ansible.builtin.include_role: diff --git a/roles/ota/applications/defaults/main.yml b/roles/ota/applications/defaults/main.yml deleted file mode 100644 index fc329a3..0000000 --- a/roles/ota/applications/defaults/main.yml +++ /dev/null @@ -1 +0,0 @@ -ota_app_read_config_only: false diff --git a/roles/ota/applications/tasks/main.yml b/roles/ota/applications/tasks/main.yml deleted file mode 100644 index ad61a87..0000000 --- a/roles/ota/applications/tasks/main.yml +++ /dev/null @@ -1,30 +0,0 @@ -- name: Clone repository - ansible.builtin.git: - repo: '{{ ota_app_repository }}' - dest: '/home/{{ ansible_user }}/{{ ota_app_directory }}' - version: '{{ ota_app_repository_branch }}' - force: true - key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' - depth: 1 - -- name: Read the production config - shell: cat '/home/{{ ansible_user }}/{{ ota_app_directory }}/config/production.json' - register: read_config - -- name: Save production config data into a variable - set_fact: - ota_apps_config: "{{ read_config.stdout | from_json }}" - -- block: - - name: Install dependencies - ansible.builtin.command: - cmd: npm install ci - chdir: '/home/{{ ansible_user }}/{{ ota_app_directory }}' - - - name: Add .env file - ansible.builtin.template: - src: .env - dest: '/home/{{ ansible_user }}/{{ ota_app_directory }}/.env' - force: true - mode: "644" - when: not ota_app_read_config_only | bool diff --git a/roles/ota/applications/templates/.env b/roles/ota/applications/templates/.env deleted file mode 100644 index 824c8ad..0000000 --- a/roles/ota/applications/templates/.env +++ /dev/null @@ -1,17 +0,0 @@ -{{ ansible_managed | comment }} - -{% if OTA_ENGINE_SENDINBLUE_API_KEY is defined %} -OTA_ENGINE_SENDINBLUE_API_KEY={{ OTA_ENGINE_SENDINBLUE_API_KEY }} -{% endif %} - -{% if OTA_ENGINE_SMTP_PASSWORD is defined %} -OTA_ENGINE_SMTP_PASSWORD={{ OTA_ENGINE_SMTP_PASSWORD }} -{% endif %} - -{% if OTA_ENGINE_GITHUB_TOKEN is defined %} -OTA_ENGINE_GITHUB_TOKEN={{ OTA_ENGINE_GITHUB_TOKEN }} -{% endif %} - -{% if OTA_FEDERATED_API_SMTP_PASSWORD is defined %} -OTA_FEDERATED_API_SMTP_PASSWORD={{ OTA_FEDERATED_API_SMTP_PASSWORD }} -{% endif %} diff --git a/roles/ota/applications/README.md b/roles/ota/apps/README.md similarity index 100% rename from roles/ota/applications/README.md rename to roles/ota/apps/README.md diff --git a/roles/ota/apps/defaults/main.yml b/roles/ota/apps/defaults/main.yml new file mode 100644 index 0000000..19b0fe1 --- /dev/null +++ b/roles/ota/apps/defaults/main.yml @@ -0,0 +1,3 @@ +ota_apps_read_config_only: false +ota_apps_repository_branch: main +ota_apps_directory: "{{ ota_apps_repository | urlsplit('path') | regex_replace('^.*/(.+?)(\\.git)?$', '\\1') }}" # extract the repository name from the URL diff --git a/roles/ota/applications/meta/main.yml b/roles/ota/apps/meta/main.yml similarity index 100% rename from roles/ota/applications/meta/main.yml rename to roles/ota/apps/meta/main.yml diff --git a/roles/ota/apps/tasks/main.yml b/roles/ota/apps/tasks/main.yml new file mode 100644 index 0000000..3d85d15 --- /dev/null +++ b/roles/ota/apps/tasks/main.yml @@ -0,0 +1,37 @@ +- name: Clone repository + ansible.builtin.git: + repo: '{{ ota_apps_repository }}' + dest: '/home/{{ ansible_user }}/{{ ota_apps_directory }}' + version: '{{ ota_apps_repository_branch }}' + force: true + key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' + depth: 1 + +- name: Read the production config + shell: cat '/home/{{ ansible_user }}/{{ ota_apps_directory }}/config/production.json' + register: read_config + +- name: Save production config data into a variable + set_fact: + ota_apps_config: "{{ read_config.stdout | from_json }}" + +- block: + - name: Install dependencies + ansible.builtin.command: + cmd: npm ci + chdir: '/home/{{ ansible_user }}/{{ ota_apps_directory }}' + + - name: Check if .env file is provided + ansible.builtin.stat: + path: "{{ inventory_dir }}/.env" + register: env_file + + - name: Add .env file + ansible.builtin.copy: + src: "{{ inventory_dir }}/.env" + dest: '/home/{{ ansible_user }}/{{ ota_apps_directory }}/.env' + force: true + mode: "644" + when: env_file.stat.exists + + when: not ota_apps_read_config_only | bool diff --git a/roles/ota/database/tasks/main.yml b/roles/ota/database/tasks/main.yml index fdf51cd..3e29626 100644 --- a/roles/ota/database/tasks/main.yml +++ b/roles/ota/database/tasks/main.yml @@ -5,7 +5,7 @@ engine_database_name: snapshots engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository }}" engine_database_branch: "{{ ota_database_snapshots_branch }}" - engine_database_directory: "/home/{{ ansible_user }}/{{ ota_app_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }}" + engine_database_directory: "/home/{{ ansible_user }}/{{ ota_apps_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }}" - name: Setup versions git repository ansible.builtin.include_tasks: database.yml @@ -14,4 +14,4 @@ engine_database_name: versions engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository }}" engine_database_branch: "{{ ota_database_versions_branch }}" - engine_database_directory: "/home/{{ ansible_user }}/{{ ota_app_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }}" + engine_database_directory: "/home/{{ ansible_user }}/{{ ota_apps_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }}" diff --git a/roles/pm2/tasks/main.yml b/roles/pm2/tasks/main.yml index 9fb2e17..b0f9aae 100644 --- a/roles/pm2/tasks/main.yml +++ b/roles/pm2/tasks/main.yml @@ -5,7 +5,7 @@ - name: Stop Open Terms Archive applications ansible.builtin.command: cmd: pm2 stop pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_app_directory }}' + chdir: '/home/{{ ansible_user }}/{{ ota_apps_directory }}' tags: - stop when: not ota_pm2_install_only | bool @@ -13,7 +13,7 @@ - name: Start Open Terms Archive applications ansible.builtin.command: cmd: pm2 startOrRestart pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_app_directory }}' + chdir: '/home/{{ ansible_user }}/{{ ota_apps_directory }}' environment: NODE_ENV: production tags: From 211eccb0eda449e9b66bd114d26139a4f085212d Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 15 May 2024 10:42:11 +0200 Subject: [PATCH 09/49] Remove obsolete file --- playbooks/deploy.yml | 1 - tests/secrets.yml | 33 --------------------------------- 2 files changed, 34 deletions(-) delete mode 100644 tests/secrets.yml diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 4162be3..0ead3e5 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -1,7 +1,6 @@ --- - name: Deploy the Open Terms Archive applications hosts: all - vars_files: "{{ inventory_dir }}/secrets.yml" tasks: - name: Load OTA applications configs ansible.builtin.include_role: diff --git a/tests/secrets.yml b/tests/secrets.yml deleted file mode 100644 index e39bd48..0000000 --- a/tests/secrets.yml +++ /dev/null @@ -1,33 +0,0 @@ -OTA_ENGINE_SMTP_PASSWORD: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 31336232636137366331316132356539623061306138323465613662633336393536363932373831 - 3633346231366163353166396364333135623462393939340a356636346433623963633837323239 - 63653665616230383965346663363732386665663463373665633736303030326463393137303433 - 3363396638383438640a623261393130613464663031663935373332636239343465646163313037 - 63613634376133656632333239356332633036623736643162323863666634343262376435363662 - 38343665363462653961663138333139646231633336623961633965303166663362666532363663 - 33396663353762646265656138373637343464326666383138643431346165356436353766363337 - 31333431663364343064623762396563386230393763346131333535316637373939316437616237 - 3938 -OTA_ENGINE_GITHUB_TOKEN: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 31666336386433366234346638633536333835383736626638313263366332623766336537376664 - 3339393538613034363738363037623533626435333663380a393434613630613033653838303836 - 31663437343730313030653331653335343361663634306432633166636163313665633937393762 - 6263623932643439350a646535663638643635333930396438303635303531383431646539336533 - 65343835356534366464323566653734386466633431383464353233346438623363353930363237 - 63356536326362616337373965653032353661323535366539336632323764613836383264393935 - 30323531316235653466333563666232646262336165366166323934373865623066326235393236 - 66616235656261343139616235306231396632643838323436383830663638663039366564623939 - 3833 -OTA_FEDERATED_API_SMTP_PASSWORD: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 31336232636137366331316132356539623061306138323465613662633336393536363932373831 - 3633346231366163353166396364333135623462393939340a356636346433623963633837323239 - 63653665616230383965346663363732386665663463373665633736303030326463393137303433 - 3363396638383438640a623261393130613464663031663935373332636239343465646163313037 - 63613634376133656632333239356332633036623736643162323863666634343262376435363662 - 38343665363462653961663138333139646231633336623961633965303166663362666532363663 - 33396663353762646265656138373637343464326666383138643431346165356436353766363337 - 31333431663364343064623762396563386230393763346131333535316637373939316437616237 - 3938 From e6d1e06baa6dbd203a43c159fc86cfa2c0b0bdc8 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 15 May 2024 10:42:49 +0200 Subject: [PATCH 10/49] Improve variable name --- roles/github/tasks/main.yml | 2 +- tests/github-bot-private-key | 26 ++++++++++++++++++++++++++ tests/ota-github-bot-private-key | 26 -------------------------- 3 files changed, 27 insertions(+), 27 deletions(-) create mode 100644 tests/github-bot-private-key delete mode 100644 tests/ota-github-bot-private-key diff --git a/roles/github/tasks/main.yml b/roles/github/tasks/main.yml index 87c0728..3987b30 100644 --- a/roles/github/tasks/main.yml +++ b/roles/github/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Add GitHub Bot SSH key ansible.builtin.copy: - src: "{{ inventory_dir }}/ota-github-bot-private-key" + src: "{{ inventory_dir }}/github-bot-private-key" dest: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' owner: '{{ ansible_user }}' group: '{{ ansible_user }}' diff --git a/tests/github-bot-private-key b/tests/github-bot-private-key new file mode 100644 index 0000000..f1a3ccd --- /dev/null +++ b/tests/github-bot-private-key @@ -0,0 +1,26 @@ +$ANSIBLE_VAULT;1.1;AES256 +32383732393936323334373030666366633637306438373563613266366138333837346135666265 +3935326632373536326331393331356637376233343535620a616637343063346437393062636439 +66333464396232343462353637356236663337643838623866353166326535643934306638666164 +3061616138363064640a353335643837366564346136313636323331316539623961313963336336 +34343633336137636532356261646663383636666339336634653232653262353538373732346366 +65356534373838316234353330306536313265663638663262303437636537333139373365316433 +36343139376265646263386364336164343565356336373930353133636633316138366565643263 +65666366353766376134363230303964373163636538373364646164343962303166366337343137 +61613961396334653536333336303134663366623663633633656130306637376164346230373135 +64616361333462646334613132366330326634313638313134666237386638393333393662373861 +33343839363731633336653130396662326134613236353031396431666636646236643937323038 +33383038623365616130633234333331313531383436393465316164633335663163626634643738 +31663231613031313331323739313833353039373639396336336639623664346533386539666238 +64336635376564363766343965306535613838353135643734653535633264343166393939613932 +33326435343265343863636338653135333566366239623538373665646131613432653962303766 +66643433666161636631643130643935663761633337663839663338646461356237353562376638 +63366439653932383635313138353336666362626231663133393232613633386639323631613363 +30633762333235333637616565396562666436353435626534366661633265613530383538306338 +64366338396566323531366534353164326463646135363237353864376630633065363163356335 +34613964313566643366313562343664666666326361303364626234326162343362383764646366 +30323530316238656536643730363930303533653464383936373630323232386264373163376265 +64646665626534663133303937316365383935666336353666306435303634393664306539656265 +66333366393038343566326237353063326337356361643663326339663633333263326666363238 +31626231303533373065313662353132623035626638346331303238336331383035316465376437 +66643339653036363438323062326234623865643063613530366335373535646363 diff --git a/tests/ota-github-bot-private-key b/tests/ota-github-bot-private-key deleted file mode 100644 index 160a9e4..0000000 --- a/tests/ota-github-bot-private-key +++ /dev/null @@ -1,26 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -39326264316331366661316164613539613733613336363930356335636434383532336136363266 -6431383230663031663139333533376664393135353231620a633632326139353439636530653966 -35326135336534323532323237393736613239323239623963333937353263376437613635356334 -3133616165646236640a363331656333336464663563303932313937313764633036363133306630 -66646637326564323931653363383130383233656634636330303262386264643439613132626439 -63653935353834626332333665616462633135653863663431633434386530636536656162396532 -64393862366537336366353639623236616434323139376563353932623765386432613463313034 -36353634666436383635333865363835346136613762373461316666386138666331636338646638 -38316465663332333737313465306330373262363761373766366232613233313063666262643930 -39343535386265333832303465376263336436663362623930666264383836396633376535643538 -31346434613635383930643737323265346539663461316636393137636236336135303833643166 -39636132393235333961643262373130376537653036393731633736613864306430646639313366 -32363166653731343830363666653963666266613162396539353965323336373439336231346131 -64653338366166336130366661653163623636623732336431643064363731333236356162663832 -35656339623335303262386633636336393661626564336339376537666630336432663062636339 -61383563653235393761313633346337343830333837386265343037396532303961616166666530 -62653634613962313539633331376464306461376131363366663038663535373139613936626136 -63363439643163326564363233316430643837323262626639353933333833646531353130646563 -38323465323434383566616263386265316665306536353663633438383761613033316161623638 -34373763663134653564663462363663643334643137333737323362623036663032313364636634 -36326330306366386464653639616664383036326433663564353834393431363433396165643166 -32396263303839613362643836353061333164623635663266666566393239346532323137383737 -61613362656161346130333362373766626263393734303262346466653764306432336165323239 -35383837383234393966666361316631353436636665646231663531653033316637386134333030 -37386534366561303734316632333936346331626634306335366665636333656433 From 7bc626670a08467b2258d8f2f5022c91032e6860 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 15 May 2024 10:43:19 +0200 Subject: [PATCH 11/49] Rename federated-api in federation-api --- playbooks/templates/nginx-reverse-proxy-conf.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/templates/nginx-reverse-proxy-conf.j2 b/playbooks/templates/nginx-reverse-proxy-conf.j2 index 2633873..dd066c5 100644 --- a/playbooks/templates/nginx-reverse-proxy-conf.j2 +++ b/playbooks/templates/nginx-reverse-proxy-conf.j2 @@ -13,11 +13,11 @@ server { } {% endif %} -{% if ota_apps_config['@opentermsarchive/federated-api'].basePath is defined and ota_apps_config['@opentermsarchive/federated-api'].port is defined %} - location {{ ota_apps_config['@opentermsarchive/federated-api'].basePath }} { +{% if ota_apps_config['@opentermsarchive/federation-api'].basePath is defined and ota_apps_config['@opentermsarchive/federation-api'].port is defined %} + location {{ ota_apps_config['@opentermsarchive/federation-api'].basePath }} { # Allowing for a `burst` of up to 5 requests beyond the specified rate limit. The `nodelay` parameter ensures that excessive requests beyond the burst limit are immediately rejected with a 429 error response instead of being queued. See https://www.nginx.com/blog/rate-limiting-nginx/. limit_req zone=limited burst=5 nodelay; - proxy_pass http://localhost:{{ ota_apps_config['@opentermsarchive/federated-api'].port }}; + proxy_pass http://localhost:{{ ota_apps_config['@opentermsarchive/federation-api'].port }}; proxy_redirect off; } {% endif %} From a92374a4bea815c90943f76d247c26afe30805bd Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 15 May 2024 10:43:59 +0200 Subject: [PATCH 12/49] Ensure ota apps variables are available to the play --- playbooks/deploy.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 0ead3e5..f77c522 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -5,6 +5,7 @@ - name: Load OTA applications configs ansible.builtin.include_role: name: ota/apps + public: true # ensure that the role's variables and defaults are accessible to the play vars: ota_apps_read_config_only: true @@ -41,13 +42,13 @@ # See https://www.mongodb.com/docs/manual/installation/#supported-platforms - ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64') become: true - tags: + tags: - infrastructure - name: Setup OTA snapshots and versions databases ansible.builtin.include_role: name: ota/database - when: + when: - ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository is defined or ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository is defined From 0b3f291cfc9ab7544c9ad93b9e9b6614ebd49025 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 15 May 2024 10:45:17 +0200 Subject: [PATCH 13/49] Remove obsolete code --- tests/Vagrantfile | 6 ------ tests/inventory.yml | 1 - 2 files changed, 7 deletions(-) diff --git a/tests/Vagrantfile b/tests/Vagrantfile index f259f20..c9d353b 100644 --- a/tests/Vagrantfile +++ b/tests/Vagrantfile @@ -8,12 +8,6 @@ Vagrant.configure("2") do |config| config.vm.hostname = "vagrant" config.vm.box = "debian/bullseye64" # Unable to locate package mongodb-org - - # in order to have the same config for both Docker and VirtualBox providers, we load the key manually - # if necessary, create the key with `ssh-keygen -f ~/.ssh/ota-vagrant -q -N ""` - # CAUTION: use of `~` in path causes problems with ssh - config.vm.provision "file", source: File.join(ENV['HOME'], ".ssh", "ota-vagrant.pub"), destination: "/home/vagrant/.ssh/authorized_keys" - config.vm.network "forwarded_port", guest: guest_port, host: host_port # based on https://github.com/rofrano/vagrant-docker-provider#example-vagrantfile diff --git a/tests/inventory.yml b/tests/inventory.yml index e7c98ca..b717113 100644 --- a/tests/inventory.yml +++ b/tests/inventory.yml @@ -4,7 +4,6 @@ vagrant: ansible_user: vagrant ansible_port: 2222 ansible_python_interpreter: /usr/bin/python3 - ansible_ssh_private_key_file: ~/.ssh/ota-vagrant ota_app_repository: https://github.com/OpenTermsArchive/demo-declarations.git ota_app_directory: demo From 398d62025f522a2f2d54c4a05a68ca80430980be Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 15 May 2024 10:45:30 +0200 Subject: [PATCH 14/49] Name the VM --- tests/Vagrantfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Vagrantfile b/tests/Vagrantfile index c9d353b..c98e262 100644 --- a/tests/Vagrantfile +++ b/tests/Vagrantfile @@ -5,6 +5,7 @@ guest_port = 80 host_port = ENV["PORT"] || 8080 Vagrant.configure("2") do |config| + config.vm.define "opentermsarchive_deployment" config.vm.hostname = "vagrant" config.vm.box = "debian/bullseye64" # Unable to locate package mongodb-org From a0bd6474f1377c18d5c6b70c8be686a8063bdc98 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 15 May 2024 10:47:05 +0200 Subject: [PATCH 15/49] Avoid to specify to ansible the python interpreter --- tests/Vagrantfile | 21 +++++++-------------- tests/inventory.yml | 1 - 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/Vagrantfile b/tests/Vagrantfile index c98e262..58657c6 100644 --- a/tests/Vagrantfile +++ b/tests/Vagrantfile @@ -20,19 +20,12 @@ Vagrant.configure("2") do |config| docker.privileged = true docker.volumes = ["/sys/fs/cgroup:/sys/fs/cgroup:rw"] docker.create_args = ["--cgroupns=host"] - - # python is not installed by default in the vagrant-provider image - # and deploying results in /bin/sh: 1: /usr/bin/python: not found - # use a provision to fix that - # only with debian, no need with ubuntu - # Also need to name the provisioner, so that it runs only once https://github.com/hashicorp/vagrant/issues/7685#issuecomment-308281283 - config.vm.provision "install_python3", type: "shell", inline: $installPython3 end -end -$installPython3 = <<-SCRIPT -echo Updating apt... -sudo apt-get update --fix-missing # Needed to fix "No package matching 'chromium' is available" -echo Installing python... -sudo apt-get --assume-yes install python3 python3-pip -SCRIPT + # Provisioning step to install Python 3 and make it the default + config.vm.provision "shell", inline: <<-SHELL + apt-get update + apt-get install -y python3 python3-pip + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + SHELL +end diff --git a/tests/inventory.yml b/tests/inventory.yml index b717113..3392369 100644 --- a/tests/inventory.yml +++ b/tests/inventory.yml @@ -3,7 +3,6 @@ vagrant: 127.0.0.1: ansible_user: vagrant ansible_port: 2222 - ansible_python_interpreter: /usr/bin/python3 ota_app_repository: https://github.com/OpenTermsArchive/demo-declarations.git ota_app_directory: demo From a417c67e493350258be4ccd37d720d40788846c1 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 15 May 2024 10:47:31 +0200 Subject: [PATCH 16/49] Update test inventory --- tests/inventory.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/inventory.yml b/tests/inventory.yml index 3392369..fec2229 100644 --- a/tests/inventory.yml +++ b/tests/inventory.yml @@ -4,5 +4,5 @@ vagrant: ansible_user: vagrant ansible_port: 2222 - ota_app_repository: https://github.com/OpenTermsArchive/demo-declarations.git - ota_app_directory: demo + ota_apps_repository: https://github.com/OpenTermsArchive/demo-declarations.git + ota_apps_repository_branch: test-new-config From a69744bf97047ea0d8d6a9f0fef1d9ddc5ea5211 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 20 May 2024 10:54:22 +0200 Subject: [PATCH 17/49] Improve variables naming --- roles/ota/apps/defaults/main.yml | 4 ++-- roles/ota/apps/tasks/main.yml | 12 ++++++------ roles/ota/database/tasks/main.yml | 4 ++-- tests/inventory.yml | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/roles/ota/apps/defaults/main.yml b/roles/ota/apps/defaults/main.yml index 19b0fe1..bdaa79e 100644 --- a/roles/ota/apps/defaults/main.yml +++ b/roles/ota/apps/defaults/main.yml @@ -1,3 +1,3 @@ ota_apps_read_config_only: false -ota_apps_repository_branch: main -ota_apps_directory: "{{ ota_apps_repository | urlsplit('path') | regex_replace('^.*/(.+?)(\\.git)?$', '\\1') }}" # extract the repository name from the URL +ota_source_repository_branch: main +ota_directory: "{{ ota_source_repository | urlsplit('path') | regex_replace('^.*/(.+?)(\\.git)?$', '\\1') }}" # extract the repository name from the URL diff --git a/roles/ota/apps/tasks/main.yml b/roles/ota/apps/tasks/main.yml index 3d85d15..3ec3d2c 100644 --- a/roles/ota/apps/tasks/main.yml +++ b/roles/ota/apps/tasks/main.yml @@ -1,14 +1,14 @@ - name: Clone repository ansible.builtin.git: - repo: '{{ ota_apps_repository }}' - dest: '/home/{{ ansible_user }}/{{ ota_apps_directory }}' - version: '{{ ota_apps_repository_branch }}' + repo: '{{ ota_source_repository }}' + dest: '/home/{{ ansible_user }}/{{ ota_directory }}' + version: '{{ ota_source_repository_branch }}' force: true key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' depth: 1 - name: Read the production config - shell: cat '/home/{{ ansible_user }}/{{ ota_apps_directory }}/config/production.json' + shell: cat '/home/{{ ansible_user }}/{{ ota_directory }}/config/production.json' register: read_config - name: Save production config data into a variable @@ -19,7 +19,7 @@ - name: Install dependencies ansible.builtin.command: cmd: npm ci - chdir: '/home/{{ ansible_user }}/{{ ota_apps_directory }}' + chdir: '/home/{{ ansible_user }}/{{ ota_directory }}' - name: Check if .env file is provided ansible.builtin.stat: @@ -29,7 +29,7 @@ - name: Add .env file ansible.builtin.copy: src: "{{ inventory_dir }}/.env" - dest: '/home/{{ ansible_user }}/{{ ota_apps_directory }}/.env' + dest: '/home/{{ ansible_user }}/{{ ota_directory }}/.env' force: true mode: "644" when: env_file.stat.exists diff --git a/roles/ota/database/tasks/main.yml b/roles/ota/database/tasks/main.yml index 3e29626..b5088c2 100644 --- a/roles/ota/database/tasks/main.yml +++ b/roles/ota/database/tasks/main.yml @@ -5,7 +5,7 @@ engine_database_name: snapshots engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository }}" engine_database_branch: "{{ ota_database_snapshots_branch }}" - engine_database_directory: "/home/{{ ansible_user }}/{{ ota_apps_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }}" + engine_database_directory: "/home/{{ ansible_user }}/{{ ota_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }}" - name: Setup versions git repository ansible.builtin.include_tasks: database.yml @@ -14,4 +14,4 @@ engine_database_name: versions engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository }}" engine_database_branch: "{{ ota_database_versions_branch }}" - engine_database_directory: "/home/{{ ansible_user }}/{{ ota_apps_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }}" + engine_database_directory: "/home/{{ ansible_user }}/{{ ota_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }}" diff --git a/tests/inventory.yml b/tests/inventory.yml index fec2229..34ad263 100644 --- a/tests/inventory.yml +++ b/tests/inventory.yml @@ -4,5 +4,5 @@ vagrant: ansible_user: vagrant ansible_port: 2222 - ota_apps_repository: https://github.com/OpenTermsArchive/demo-declarations.git - ota_apps_repository_branch: test-new-config + ota_source_repository: https://github.com/OpenTermsArchive/demo-declarations.git + ota_source_repository_branch: test-new-config From a13677360b8d3025d18d7a0a711f138ea98b68a1 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 20 May 2024 10:55:44 +0200 Subject: [PATCH 18/49] Fix .env file existence check on local machine --- roles/ota/apps/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/ota/apps/tasks/main.yml b/roles/ota/apps/tasks/main.yml index 3ec3d2c..1b47db1 100644 --- a/roles/ota/apps/tasks/main.yml +++ b/roles/ota/apps/tasks/main.yml @@ -21,10 +21,11 @@ cmd: npm ci chdir: '/home/{{ ansible_user }}/{{ ota_directory }}' - - name: Check if .env file is provided + - name: Check if .env file is provided (on local machine) ansible.builtin.stat: path: "{{ inventory_dir }}/.env" register: env_file + delegate_to: localhost - name: Add .env file ansible.builtin.copy: From 60a1695df2b594ff69e13737290f733bc448fec3 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 20 May 2024 10:55:59 +0200 Subject: [PATCH 19/49] Add Github private file existence check on local machine --- roles/github/tasks/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/github/tasks/main.yml b/roles/github/tasks/main.yml index 3987b30..a41fe5a 100644 --- a/roles/github/tasks/main.yml +++ b/roles/github/tasks/main.yml @@ -1,4 +1,10 @@ --- +- name: Check if github-bot-private-key file is provided (on local machine) + ansible.builtin.stat: + path: "{{ inventory_dir }}/github-bot-private-key" + register: github_bot_private_key_file + delegate_to: localhost + - name: Add GitHub Bot SSH key ansible.builtin.copy: src: "{{ inventory_dir }}/github-bot-private-key" @@ -6,6 +12,7 @@ owner: '{{ ansible_user }}' group: '{{ ansible_user }}' mode: "600" + when: github_bot_private_key_file.stat.exists - name: Add GitHub SSH key to known_hosts ansible.builtin.known_hosts: From 13511cfc90ef62a5c2bc00d9e0ef7a589ec4045a Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 20 May 2024 10:57:05 +0200 Subject: [PATCH 20/49] Make pm2 config file required next to the inventory --- roles/pm2/tasks/main.yml | 40 ++++++++++++++++++++++++---------------- tests/pm2.config.cjs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 tests/pm2.config.cjs diff --git a/roles/pm2/tasks/main.yml b/roles/pm2/tasks/main.yml index b0f9aae..ab55264 100644 --- a/roles/pm2/tasks/main.yml +++ b/roles/pm2/tasks/main.yml @@ -2,21 +2,29 @@ ansible.builtin.command: npm install -g pm2@5 --production=true when: ota_pm2_install_only | bool -- name: Stop Open Terms Archive applications - ansible.builtin.command: - cmd: pm2 stop pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_apps_directory }}' - tags: - - stop - when: not ota_pm2_install_only | bool +- block: + - name: Add pm2.config.cjs file + ansible.builtin.copy: + src: "{{ inventory_dir }}/pm2.config.cjs" + dest: '/home/{{ ansible_user }}/{{ ota_directory }}/pm2.config.cjs' + force: true + mode: "644" + + - name: Stop Open Terms Archive applications + ansible.builtin.command: + cmd: pm2 stop pm2.config.cjs + chdir: '/home/{{ ansible_user }}/{{ ota_directory }}' + tags: + - stop + + - name: Start Open Terms Archive applications + ansible.builtin.command: + cmd: pm2 startOrRestart pm2.config.cjs + chdir: '/home/{{ ansible_user }}/{{ ota_directory }}' + environment: + NODE_ENV: production + tags: + - restart + - start -- name: Start Open Terms Archive applications - ansible.builtin.command: - cmd: pm2 startOrRestart pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_apps_directory }}' - environment: - NODE_ENV: production - tags: - - restart - - start when: not ota_pm2_install_only | bool diff --git a/tests/pm2.config.cjs b/tests/pm2.config.cjs new file mode 100644 index 0000000..6d99bc1 --- /dev/null +++ b/tests/pm2.config.cjs @@ -0,0 +1,33 @@ +module.exports = { + apps: [ + { + name: 'ota', + script: 'npm', + args: 'run start:schedule', + max_restarts: 2, + min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. + restart_delay: 10800000, + }, + { + name: 'ota-api', + script: 'npm', + args: 'run start:api', + }, + { + name: 'ota-release', + script: 'npm', + args: 'run dataset:schedule', + max_restarts: 2, + min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. + restart_delay: 15 * 60 * 1000, + }, + { + name: 'ota-federation-api', + script: 'npm', + args: 'run start:federation-api', + max_restarts: 2, + min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. + restart_delay: 15 * 60 * 1000, + } + ], +}; From 0601316bcf031608125ac1affcb78c40b56fbdf6 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 20 May 2024 10:57:26 +0200 Subject: [PATCH 21/49] Ensure `delegate_to` works with Vagrant --- tests/inventory.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/inventory.yml b/tests/inventory.yml index 34ad263..aa776ab 100644 --- a/tests/inventory.yml +++ b/tests/inventory.yml @@ -1,8 +1,9 @@ vagrant: hosts: - 127.0.0.1: + vagrant: # Name the host instead of using 127.0.0.1 to ensure `delegate_to` distinguishes between the local machine and the locally running Vagrant VM ansible_user: vagrant ansible_port: 2222 + ansible_ssh_host: 127.0.0.1 ota_source_repository: https://github.com/OpenTermsArchive/demo-declarations.git ota_source_repository_branch: test-new-config From d6701abf42d90d1131a75a0e20fd43705fc237c8 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 20 May 2024 10:57:32 +0200 Subject: [PATCH 22/49] Update README --- README.md | 252 ++++++++++++++++++++++-------------------------------- 1 file changed, 100 insertions(+), 152 deletions(-) diff --git a/README.md b/README.md index 1f684d1..4388d01 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,13 @@ This repository contains the `opentermsarchive.deployment` Ansible collection. This Ansible collection provides playbooks to set up the infrastructure of and deploy Open Terms Archive applications. +To prevent confusion between the notion of Ansible [Collection](https://docs.ansible.com/ansible/latest/collections_guide/index.html) and an Open Terms Archive [Collection](https://docs.opentermsarchive.org/#collection), we will refer to Ansible Collection only as “Playbook”, as this is the main entry point to interact with it. + ## Installation -[Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) is required to use this collection. +[Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) is required to use this playbook. -This collection can be installed from Ansible Galaxy manually with the `ansible-galaxy` command-line tool: +This playbook can be installed from Ansible Galaxy manually with the `ansible-galaxy` command-line tool: ```sh ansible-galaxy collection install opentermsarchive.deployment @@ -28,200 +30,102 @@ ansible-galaxy collection install -r requirements.yml ## Usage -Once installed, some playbooks are available to deploy the two main Open Terms Archive applications: [Engine](https://github.com/OpenTermsArchive/engine) and [Federated API](https://github.com/OpenTermsArchive/federated-api). +Once installed, the playbook `deploy` allows to set up the two main Open Terms Archive applications: [Engine](https://github.com/OpenTermsArchive/engine) and [Federated API](https://github.com/OpenTermsArchive/federated-api). -Each playbook can be executed using the `ansible-playbook` command-line tool: +The playbook can be executed using the `ansible-playbook` command-line tool: ```sh -ansible-playbook opentermsarchive.deployment. +ansible-playbook opentermsarchive.deployment.deploy ``` -Refer to the application related sections below for a list of available playbooks. - _It is possible to check a playbook execution without actually applying changes with `check` and `diff` options:_ ```sh -ansible-playbook opentermsarchive.deployment. --check --diff +ansible-playbook opentermsarchive.deployment.deploy --check --diff ``` > See “[Using collections](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html)” in Ansible’s user guide for more information about Ansible collections. - - - -### Engine application - -Available playbooks for the engine application: - -| Playbook name | Description | Command example | -| --- | --- | --- | -| `engine.infrastructure` | Set up and configure the infrastructure required by the Open Terms Archive engine | `ansible-playbook opentermsarchive.deployment.engine.infrastructure` | -| `engine.application` | Deploy the Open Terms Archive engine | `ansible-playbook opentermsarchive.deployment.engine.application` | -| `engine.all` | Set up infrastructure and deploy the Open Terms Archive engine | `ansible-playbook opentermsarchive.deployment.engine.all` | +### Configuration -#### Configuration +Available [variables](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html) are listed below, along with their default values: -Available [variables](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html) are listed below, along with default values: +| Variable | Description | Default Value | Required | +|--------------------------------|----------------------------------------------------------------------------------------------------------------------------|------------------------|----------| +| `ota_source_repository` | URL of the source repository | No default value | ✔︎ | +| `ota_source_repository_branch` | [Git branch or tag](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftree-ishatree-ishalsotreeish) of the source repository | `main` | | +| `ota_directory` | Directory path where the code will be deployed on the server | Name of the repository | | -| Variable | Description | Default value | Required | -| --- | --- | --- | --- | -| `ota_engine_github_bot_private_key` | SSH private key contents for GitHub user with privileges on snapshots and versions repositories | No default value | ✔︎ | -| `ota_engine_github_token` | GitHub token to enable issue creation on the declarations repository and publish releases on versions repository | No default value | ✔︎ | -| `ota_engine_smtp_password` | Password for the SMTP server used for sending error notifications by email | No default value | - | -| `ota_engine_sendinblue_api_key` | SendInBlue API key used to send email notifications | No default value | - | -| `ota_engine_config_path` | Path to the engine config file, relative to the inventory file | `../config/production.json` | - | -| `ota_engine_declarations_branch` | [Git branch or tag](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftree-ishatree-ishalsotreeish) of the declarations repository to use | `main` | - | -| `ota_engine_snapshots_branch` | [Git branch or tag](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftree-ishatree-ishalsotreeish) of the snapshots repository to use | `main` | - | -| `ota_engine_versions_branch` | [Git branch or tag](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftree-ishatree-ishalsotreeish) of the versions repository to use | `main` | - | -| `ota_engine_declarations_directory` | Path of the directory where the code will be deployed on the server | Value declared in the `name` key in the engine config file | - | -| `ota_engine_restart_delay` | Delay, in milliseconds, before restarting the engine after a crash | `10800000` _(3 hours)_ | - | - -For encryption of sensitive configuration entries, please refer to the [dedicated section](#encrypt-sensitive-configuration-entries). - -These variables can be overriden in the inventory file, for example: +These variables can be defined in the inventory file, for example: ```yml all: hosts: 127.0.0.1: ansible_user: debian - ota_engine_config_path: ./engine_config.json - ota_engine_declarations_branch: new-feature + ota_source_repository: https://github.com/OpenTermsArchive/demo-declarations.git + ota_directory: demo ``` -#### Tags - -Available [tags](https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html) to refine what will happen, use them with `--tags`: - -| Tag | Description | Command example | -| --- | --- | --- | -| `start` | Start the engine | `ansible-playbook opentermsarchive.deployment.engine.application --tags start` | -| `stop` | Stop the engine | `ansible-playbook opentermsarchive.deployment.engine.application --tags stop` | -| `restart` | Restart the engine | `ansible-playbook opentermsarchive.deployment.engine.application --tags restart` | -| `update-declarations` | Update service declarations (pull declarations, install dependencies, and restart engine) | `ansible-playbook opentermsarchive.deployment.engine.application --tags update-declarations` | - -- - - - -### Federated API application - -Available playbooks for the Federated API application: - -| Playbook name | Description | Command example | -| --- | --- | --- | -| `federated_api.infrastructure` | Set up and configure the infrastructure required by the Open Terms Archive federated API | `ansible-playbook opentermsarchive.deployment.federated_api.infrastructure` | -| `federated_api.application` | Deploy the Open Terms Archive federated API | `ansible-playbook opentermsarchive.deployment.federated_api.application` | -| `federated_api.all` | Set up infrastructure and deploy the Open Terms Archive federated API | `ansible-playbook opentermsarchive.deployment.federated_api.all` | - -#### Configuration - -Available variables are listed below, along with default values: +#### Additional files -| Variable | Description | Default value | Required | -| --- | --- | --- | --- | -| `ota_federated_api_repo` | Repository URL of the federated API code | `https://github.com/OpenTermsArchive/federated-api.git` | - | -| `ota_federated_api_directory` | Path of the directory where the code will be deployed on the server | `federated-api` | - | -| `ota_federated_api_branch` | [Git branch or tag](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftree-ishatree-ishalsotreeish) of the federated API repository to use | `main` | - | -| `ota_federated_api_smtp_password` | Password for the SMTP server used for sending errors notifications by email. | - | - | +The `deploy` playbook requires additional files to be placed alongside the `inventory.yml` file. These files are necessary for deploying the Open Terms Archive applications properly. Below are the required and optional files and their purposes: -For encryption of sensitive configuration entries, please refer to the [dedicated section](#encrypt-sensitive-configuration-entries). +| File | Description | Required | Encryption Required | +|--------------------------|---------------------------------------------------------------------------------------------------------|----------|---------------------| +| `pm2.config.cjs` | Configuration file describing the processes to be started and managed by PM2 | ✔︎ | | +| `github-bot-private-key` | Private SSH key for accessing SSH Git URLs | Required if `ota_source_repository` is an SSH Git URL or if the URLs for versions and/or snapshots repositories in the `config/production.json` file of the source repository are SSH Git URLs | ✔︎ | +| `.env` | File defining environment variables required by the deployed application | | ✔︎ | -These variables can be overridden in the inventory file, for example: +Here is an example of the directory structure: -```yml -all: - hosts: - 127.0.0.1: - ansible_user: debian - ota_federated_api_repo: https://github.com/OpenTermsArchive/federated-api.git - ota_federated_api_branch: new-feature +```plaintext +ops/ + ├── inventory.yml + ├── pm2.config.cjs + ├── github-bot-private-key + └── .env ``` -#### Tags - -Available [tags](https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html) to refine what will happen, use them with `--tags`: - -| Tag | Description | Command example | -| --- | --- | --- | -| `start` | To only start the Federated API | `ansible-playbook opentermsarchive.deployment.federated_api.application --tags start` | -| `stop` | To only stop the Federated API | `ansible-playbook opentermsarchive.deployment.federated_api.application --tags stop` | -| `restart` | To only restart the Federated API | `ansible-playbook opentermsarchive.deployment.federated_api.application --tags restart` | - -- - - - -### Engine and Federated API applications - -Available playbooks to deploy both the Open Terms Archive Engine and Federated API applications on a single server. - -| Playbook name | Description | Command example | -| --- | --- | --- | -| `engine_and_federated_api.infrastructure` | Set up and configure the infrastructure required by the Open Terms Archive engine and federated API applications | `ansible-playbook opentermsarchive.deployment.engine_and_federated_api.infrastructure` | -| `engine_and_federated_api.application` | Deploy the Open Terms Archive engine and federated API applications | `ansible-playbook opentermsarchive.deployment.engine_and_federated_api.application` | -| `engine_and_federated_api.all` | Set up infrastructure and deploy the Open Terms Archive engine and federated API applications | `ansible-playbook opentermsarchive.deployment.engine_and_federated_api.all` | - -#### Configuration - -Available variables are listed below, along with default values: - -| Variable | Description | Default value | Required | -| --- | --- | --- | --- | -| `ota_reverse_proxy_engine_path` | Path where the collection API embed with the engine will be available | `/collection-api` | - | -| `ota_reverse_proxy_federated_api_path` | Path where the federated API will be available | `/federation-api` | - | +#### Encrypting sensitive configuration files -- - - - -## Encrypt sensitive configuration entries +Sensitive configuration files should be encrypted using [Ansible Vault](https://docs.ansible.com/ansible/latest/vault_guide/index.html). -Certain configuration entries contain sensitive information that should be encrypted to ensure security. Ansible provides a convenient way to encrypt such strings using its built-in [vault feature](https://docs.ansible.com/ansible/2.9/user_guide/vault.html): +Examples: -```sh -ansible-vault encrypt_string --name -``` +- Encrypt the `github-bot-private-key` file: `ansible-vault encrypt github-bot-private-key` +- Decrypt the `github-bot-private-key` file: `ansible-vault decrypt github-bot-private-key` +- Encrypting with a password stored in a file: + - `echo 'your_password' > vault.key` + - `ansible-vault encrypt --vault-password-file vault.key github-bot-private-key` -For example, to encrypt the GitHub bot private key used by the engine to push updates: +To run the playbook with encrypted files: ```sh -ansible-vault encrypt_string --name 'ota_engine_github_bot_private_key' '-----BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn -… -UlcCkBZ5IkI0eNAAAAE25kcG50QE1CUC1OZHBudC5sYW4BAgMEBQYH ------END OPENSSH PRIVATE KEY----- -' +ansible-playbook playbook.yml --ask-vault-pass ``` -The encrypted result will look like this: +Or with a password file: ```sh -ota_engine_github_bot_private_key: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 62313438616266383732353634343736623532666365643364396464633732613966636235616261 - 3136656665316437613434323561613732373361306161640a306132316531356537373862363838 - 66363763613833373530633831653163303961376331393761366261633561656463626563383931 - 3361643836623239660a333134626139626465303234313366313433653261376437316231363834 - 32643261303534366333383131633430396366343631656363663965633964663331346231663166 - 3331316462356461373134303666613035393335333139613639 +ansible-playbook playbook.yml --vault-password-file vault.key ``` -Then it can be used directly in the inventory file: +Please note that the data will be stored unencrypted on the deployment server. -```yml -all: - hosts: - 127.0.0.1: - ansible_user: debian - ota_engine_config_path: ./engine_config.json - ota_engine_declarations_branch: new-feature - ota_engine_github_bot_private_key: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 62313438616266383732353634343736623532666365643364396464633732613966636235616261 - 3136656665316437613434323561613732373361306161640a306132316531356537373862363838 - 66363763613833373530633831653163303961376331393761366261633561656463626563383931 - 3361643836623239660a333134626139626465303234313366313433653261376437316231363834 - 32643261303534366333383131633430396366343631656363663965633964663331346231663166 - 3331316462356461373134303666613035393335333139613639 -``` +### Refining playbook execution -Repeat the process for each sensitive configuration entry that needs encryption. +Use [tags](https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html) to refine playbook execution. Example commands: -Please note that the data will be stored unencrypted on the deployment server. +| Tag | Description | Command Example | +|-----------------|-----------------------------------------|----------------------------------------------------------------------------------| +| `start` | Start Open Terms Archive applications | `ansible-playbook opentermsarchive.deployment.deploy --tags start` | +| `stop` | Stop Open Terms Archive applications | `ansible-playbook opentermsarchive.deployment.deploy --tags stop` | +| `restart` | Restart Open Terms Archive applications | `ansible-playbook opentermsarchive.deployment.deploy --tags restart` | +| `infrastructure`| Set up the infrastructure only | `ansible-playbook opentermsarchive.deployment.deploy --tags infrastructure` | +| `infrastructure`| Skip the infrastructure | `ansible-playbook opentermsarchive.deployment.deploy --skip-tags infrastructure` | - - - @@ -258,7 +162,7 @@ Then the code can be deployed to the running machine with all the options descri Testing the Ansible collection locally is crucial to ensure that changes function properly before submitting them as a pull request. -The testing environment is preconfigured for Open Terms Archive maintainers. For other contributors, the configuration file `tests/engine_config.json` needs to be updated to specify repositories where they have authorizations. Additionally, the `ota_engine_github_bot_private_key` value in the inventory file `tests/inventory.yml` should be updated. +The testing environment is preconfigured for Open Terms Archive maintainers. For other contributors, the inventory file `tests/inventory.yml` needs to be updated to specify repositories where they have authorizations. Additionally, the `github-bot-private-key` file should be updated. Follow these instructions to test the collection in a local environment: @@ -270,7 +174,7 @@ vagrant up - Apply the changes to the virtual machine: ```sh -ansible-playbook ../playbooks/engine/all.yml +ansible-playbook ../playbooks/deploy.yml ``` - Connect to the virtual machine to verify that changes were applied successfully: @@ -283,6 +187,50 @@ vagrant ssh # use "vagrant" as password pm2 logs ``` +#### Troubleshooting + +If you encounter an error while running the playbook, such as: + +```sh +PLAY [Deploy the Open Terms Archive applications] ************************************************ + +TASK [Gathering Facts] *************************************************************************** +fatal: [127.0.0.1]: UNREACHABLE! => changed=false + msg: 'Failed to connect to the host via ssh: vagrant@127.0.0.1: Permission denied (publickey,password).' + unreachable: true +``` + +Do the following: + +1. Run `vagrant ssh-config` and note the `IdentityFile` output: + +```sh +vagrant ssh-config +``` + +Example output: + +```plaintext +Host opentermsarchive_deployment + HostName 127.0.0.1 + User vagrant + Port 2222 + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + PasswordAuthentication no + IdentityFile /path/to/your/vagrant/private_key + IdentitiesOnly yes + LogLevel FATAL + PubkeyAcceptedKeyTypes +ssh-rsa + HostKeyAlgorithms +ssh-rsa +``` + +2. Add the private key to your SSH agent: + +```sh +ssh-add /path/to/your/vagrant/private_key +``` + --- ## License From ceb81bed7cc53976bcada3e118b431f93ae0e0e6 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 20 May 2024 14:52:48 +0200 Subject: [PATCH 23/49] Add changelog entry --- CHANGELOG.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d972b8..148bdbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,33 @@ All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## Unreleased [major] + +_Full changeset and discussions: [#41](https://github.com/OpenTermsArchive/deployment/pull/41)._ + +> Development of this release was supported by the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs. + +### Changed + +- **Breaking:** Replace all playbooks with a single `deploy` playbook; update your scripts by using `ansible-playbook opentermsarchive.deployment.deploy` and use [tags to refine execution](./README.md#refining-playbook-execution) +- **Breaking:** Change the `federation-api` deployment process to be a dependency of the source repository +- **Breaking:** Require `engine>=2` and `federation-api>=v2` +- **Breaking:** Make `ota_source_repository` variable mandatory +- **Breaking:** Define environment variables in a `.env` file instead of in inventory variables; move `ota_engine_github_token`, `ota_engine_smtp_password`, `ota_engine_sendinblue_api_key`, and `ota_federated_api_smtp_password` to a `.env` file placed next to the inventory file and rename them according to the new naming conventions in `engine` v2 and `federation-api` v2 +- **Breaking:** Rename `ota_engine_declarations_branch` to `ota_source_repository_branch` +- **Breaking:** Rename `ota_engine_declarations_directory` to `ota_directory` +- **Breaking:** Extract the PM2 config file `pm2.config.cjs`; provide this file next to the inventory file +- **Breaking:** Remove the `ota_engine_restart_delay` variable; define this setting directly in the `pm2.config.cjs` +- **Breaking:** Remove the `ota_engine_github_bot_private_key` variable in favor of the `github-bot-private-key` file; define the GitHub SSH private key in the `github-bot-private-key` + +### Removed + +- **Breaking:** Remove `ota_reverse_proxy_federated_api_path` config; define this path in your `config/production.json` under the key `@opentermsarchive/federation-api: { basePath: "" }` +- **Breaking:** Remove `ota_reverse_proxy_engine_path` config; define this path in your `config/production.json` under the key `@opentermsarchive/engine: { collection-api: { basePath: "" } }` +- **Breaking:** Remove obsolete `ota_federated_api_branch`, `ota_federated_api_directory`, and `ota_federated_api_repo` configs +- **Breaking:** Remove obsolete `ota_engine_config_path` config +- **Breaking:** Remove obsolete `update-declarations` tag + ## 1.2.1 - 2024-05-22 From c360ddd3937d78b8738c78801f4b2edc71cee7f5 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 22 May 2024 10:34:49 +0200 Subject: [PATCH 24/49] Improve copywriting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Biron --- README.md | 4 ++-- playbooks/deploy.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4388d01..95a8bce 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ all: #### Additional files -The `deploy` playbook requires additional files to be placed alongside the `inventory.yml` file. These files are necessary for deploying the Open Terms Archive applications properly. Below are the required and optional files and their purposes: +The `deploy` playbook requires additional files to be placed alongside the `inventory.yml` file. These files are necessary for deploying Open Terms Archive applications properly. Below are the required and optional files and their purposes: | File | Description | Required | Encryption Required | |--------------------------|---------------------------------------------------------------------------------------------------------|----------|---------------------| @@ -192,7 +192,7 @@ pm2 logs If you encounter an error while running the playbook, such as: ```sh -PLAY [Deploy the Open Terms Archive applications] ************************************************ +PLAY [Deploy Open Terms Archive applications] ************************************************ TASK [Gathering Facts] *************************************************************************** fatal: [127.0.0.1]: UNREACHABLE! => changed=false diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index f77c522..0cebeab 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -1,5 +1,5 @@ --- -- name: Deploy the Open Terms Archive applications +- name: Deploy Open Terms Archive applications hosts: all tasks: - name: Load OTA applications configs From 2d215fe32d629cb9be6cb0929a90b8de480123ae Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 22 May 2024 10:35:19 +0200 Subject: [PATCH 25/49] Fix application name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Biron --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 95a8bce..2eb9e6f 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ ansible-galaxy collection install -r requirements.yml ## Usage -Once installed, the playbook `deploy` allows to set up the two main Open Terms Archive applications: [Engine](https://github.com/OpenTermsArchive/engine) and [Federated API](https://github.com/OpenTermsArchive/federated-api). +Once installed, the playbook `deploy` allows to set up the two main Open Terms Archive applications: [Engine](https://github.com/OpenTermsArchive/engine) and [Federation API](https://github.com/OpenTermsArchive/federation-api). The playbook can be executed using the `ansible-playbook` command-line tool: From 79ecd5992e0df14593618bd5a4ada1c061d523c8 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 22 May 2024 10:35:46 +0200 Subject: [PATCH 26/49] Display all variables in example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Biron --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2eb9e6f..79725f8 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ all: 127.0.0.1: ansible_user: debian ota_source_repository: https://github.com/OpenTermsArchive/demo-declarations.git + ota_source_repository_branch: main ota_directory: demo ``` From 71cba8ab2cf3e0f560cfbb574c257b40edd59322 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 22 May 2024 11:02:02 +0200 Subject: [PATCH 27/49] Remove useless READMEs --- roles/chromium/README.md | 3 --- roles/common/README.md | 1 - roles/github/README.md | 1 - roles/mongo/README.md | 1 - roles/node/README.md | 1 - roles/ota/apps/README.md | 2 -- roles/ota/database/README.md | 2 -- 7 files changed, 11 deletions(-) delete mode 100644 roles/chromium/README.md delete mode 100644 roles/common/README.md delete mode 100644 roles/github/README.md delete mode 100644 roles/mongo/README.md delete mode 100644 roles/node/README.md delete mode 100644 roles/ota/apps/README.md delete mode 100644 roles/ota/database/README.md diff --git a/roles/chromium/README.md b/roles/chromium/README.md deleted file mode 100644 index 7e1f097..0000000 --- a/roles/chromium/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Chromium - -Used by the engine as a [puppeteer](https://pptr.dev) dependency. diff --git a/roles/common/README.md b/roles/common/README.md deleted file mode 100644 index 019b25f..0000000 --- a/roles/common/README.md +++ /dev/null @@ -1 +0,0 @@ -# Common diff --git a/roles/github/README.md b/roles/github/README.md deleted file mode 100644 index c27f953..0000000 --- a/roles/github/README.md +++ /dev/null @@ -1 +0,0 @@ -# GitHub diff --git a/roles/mongo/README.md b/roles/mongo/README.md deleted file mode 100644 index e679fb7..0000000 --- a/roles/mongo/README.md +++ /dev/null @@ -1 +0,0 @@ -# Mongo diff --git a/roles/node/README.md b/roles/node/README.md deleted file mode 100644 index 6d9836e..0000000 --- a/roles/node/README.md +++ /dev/null @@ -1 +0,0 @@ -# Node diff --git a/roles/ota/apps/README.md b/roles/ota/apps/README.md deleted file mode 100644 index efebbdb..0000000 --- a/roles/ota/apps/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Engine - diff --git a/roles/ota/database/README.md b/roles/ota/database/README.md deleted file mode 100644 index efebbdb..0000000 --- a/roles/ota/database/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Engine - From 55b23b06071b39dbb0f3eb9c2a318471778c95a2 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 22 May 2024 11:00:50 +0200 Subject: [PATCH 28/49] Lint --- roles/chromium/meta/main.yml | 1 + roles/chromium/tasks/main.yml | 2 +- roles/github/tasks/main.yml | 14 +++---- roles/mongo/meta/main.yml | 1 + roles/mongo/tasks/main.yml | 2 +- roles/nginx/defaults/main.yml | 1 + roles/nginx/meta/main.yml | 1 + roles/nginx/tasks/main.yml | 54 ++++++++++++++------------- roles/node/meta/main.yml | 1 + roles/node/tasks/main.yml | 3 +- roles/ota/apps/defaults/main.yml | 1 + roles/ota/apps/meta/main.yml | 1 + roles/ota/apps/tasks/main.yml | 53 +++++++++++++------------- roles/ota/database/defaults/main.yml | 1 + roles/ota/database/meta/main.yml | 1 + roles/ota/database/tasks/database.yml | 33 ++++++++-------- roles/ota/database/tasks/main.yml | 5 ++- roles/pm2/README.md | 1 - roles/pm2/defaults/main.yml | 1 + roles/pm2/meta/main.yml | 1 + roles/pm2/tasks/main.yml | 49 ++++++++++++------------ 21 files changed, 121 insertions(+), 106 deletions(-) delete mode 100644 roles/pm2/README.md diff --git a/roles/chromium/meta/main.yml b/roles/chromium/meta/main.yml index b1656ff..6b4fff8 100644 --- a/roles/chromium/meta/main.yml +++ b/roles/chromium/meta/main.yml @@ -1,2 +1,3 @@ +--- dependencies: - { role: common } diff --git a/roles/chromium/tasks/main.yml b/roles/chromium/tasks/main.yml index 527444d..d40fef3 100644 --- a/roles/chromium/tasks/main.yml +++ b/roles/chromium/tasks/main.yml @@ -15,7 +15,7 @@ state: latest when: ansible_distribution == 'Ubuntu' - # See https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#recommended-enable-user-namespace-cloning +# See https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#recommended-enable-user-namespace-cloning - name: Enable user namespace cloning to allow running Chromium in a sandbox ansible.builtin.command: sysctl -w kernel.unprivileged_userns_clone=1 when: ansible_facts['architecture'] != 'aarch64' diff --git a/roles/github/tasks/main.yml b/roles/github/tasks/main.yml index a41fe5a..949086d 100644 --- a/roles/github/tasks/main.yml +++ b/roles/github/tasks/main.yml @@ -8,9 +8,9 @@ - name: Add GitHub Bot SSH key ansible.builtin.copy: src: "{{ inventory_dir }}/github-bot-private-key" - dest: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' - owner: '{{ ansible_user }}' - group: '{{ ansible_user }}' + dest: /home/{{ ansible_user }}/.ssh/ota-github-bot-key + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" mode: "600" when: github_bot_private_key_file.stat.exists @@ -20,7 +20,7 @@ key: "{{ item }}" path: ~/.ssh/known_hosts with_items: # GitHub's SSH key fingerprints can be found here: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints - - "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" - - "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" - - "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" - + - github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + - github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= + - github.com ssh-rsa + AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= diff --git a/roles/mongo/meta/main.yml b/roles/mongo/meta/main.yml index b1656ff..6b4fff8 100644 --- a/roles/mongo/meta/main.yml +++ b/roles/mongo/meta/main.yml @@ -1,2 +1,3 @@ +--- dependencies: - { role: common } diff --git a/roles/mongo/tasks/main.yml b/roles/mongo/tasks/main.yml index fe2a47c..92a8d80 100644 --- a/roles/mongo/tasks/main.yml +++ b/roles/mongo/tasks/main.yml @@ -30,7 +30,7 @@ - name: Add mongod.conf ansible.builtin.copy: src: mongod.conf - dest: '/etc/mongod.conf' + dest: /etc/mongod.conf mode: "644" - name: Create data directory diff --git a/roles/nginx/defaults/main.yml b/roles/nginx/defaults/main.yml index b4d3bd0..fe113b0 100644 --- a/roles/nginx/defaults/main.yml +++ b/roles/nginx/defaults/main.yml @@ -1,3 +1,4 @@ +--- ota_nginx_install: true ota_nginx_configure: false ota_nginx_configure_reverse_proxy: false diff --git a/roles/nginx/meta/main.yml b/roles/nginx/meta/main.yml index b1656ff..6b4fff8 100644 --- a/roles/nginx/meta/main.yml +++ b/roles/nginx/meta/main.yml @@ -1,2 +1,3 @@ +--- dependencies: - { role: common } diff --git a/roles/nginx/tasks/main.yml b/roles/nginx/tasks/main.yml index a3b71c2..f226e80 100644 --- a/roles/nginx/tasks/main.yml +++ b/roles/nginx/tasks/main.yml @@ -6,36 +6,38 @@ state: latest when: ota_nginx_install | bool -- block: - - name: Clean default NGINX site - ansible.builtin.file: - state: absent - path: /etc/nginx/sites-enabled/default - - - name: Setup NGINX conf - ansible.builtin.template: - src: "{{ ota_nginx_config_template }}" - dest: /etc/nginx/conf.d/ota.conf - owner: root - group: root - mode: "644" +- name: Configure NGINX notify: Restart NGINX when: ota_nginx_configure | bool + block: + - name: Clean default NGINX site + ansible.builtin.file: + state: absent + path: /etc/nginx/sites-enabled/default -- block: - - name: Add conf in NGINX sites-available - ansible.builtin.template: - src: "{{ ota_nginx_reverse_proxy_config_template }}" - dest: '/etc/nginx/sites-available/ota' - force: true - mode: "644" + - name: Setup NGINX conf + ansible.builtin.template: + src: "{{ ota_nginx_config_template }}" + dest: /etc/nginx/conf.d/ota.conf + owner: root + group: root + mode: "644" - - name: Link conf from sites-available to sites-enabled - ansible.builtin.file: - src: '/etc/nginx/sites-available/ota' - dest: '/etc/nginx/sites-enabled/ota' - state: link - force: true +- name: Configure NGINX Reverse Proxy become: true notify: Restart NGINX when: ota_nginx_configure_reverse_proxy | bool + block: + - name: Add conf in NGINX sites-available + ansible.builtin.template: + src: "{{ ota_nginx_reverse_proxy_config_template }}" + dest: /etc/nginx/sites-available/ota + force: true + mode: "644" + + - name: Link conf from sites-available to sites-enabled + ansible.builtin.file: + src: /etc/nginx/sites-available/ota + dest: /etc/nginx/sites-enabled/ota + state: link + force: true diff --git a/roles/node/meta/main.yml b/roles/node/meta/main.yml index b1656ff..6b4fff8 100644 --- a/roles/node/meta/main.yml +++ b/roles/node/meta/main.yml @@ -1,2 +1,3 @@ +--- dependencies: - { role: common } diff --git a/roles/node/tasks/main.yml b/roles/node/tasks/main.yml index f0df1bb..f275b09 100644 --- a/roles/node/tasks/main.yml +++ b/roles/node/tasks/main.yml @@ -11,8 +11,7 @@ executable: /bin/bash - name: Create deb repository - ansible.builtin.shell: - set -o pipefail && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list # Remember to update the major version of NPM when updating the major version of Node + ansible.builtin.shell: set -o pipefail && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list # Remember to update the major version of NPM when updating the major version of Node args: executable: /bin/bash diff --git a/roles/ota/apps/defaults/main.yml b/roles/ota/apps/defaults/main.yml index bdaa79e..73a9514 100644 --- a/roles/ota/apps/defaults/main.yml +++ b/roles/ota/apps/defaults/main.yml @@ -1,3 +1,4 @@ +--- ota_apps_read_config_only: false ota_source_repository_branch: main ota_directory: "{{ ota_source_repository | urlsplit('path') | regex_replace('^.*/(.+?)(\\.git)?$', '\\1') }}" # extract the repository name from the URL diff --git a/roles/ota/apps/meta/main.yml b/roles/ota/apps/meta/main.yml index d0c083a..2f3426c 100644 --- a/roles/ota/apps/meta/main.yml +++ b/roles/ota/apps/meta/main.yml @@ -1,3 +1,4 @@ +--- dependencies: - { role: common } - { role: github } diff --git a/roles/ota/apps/tasks/main.yml b/roles/ota/apps/tasks/main.yml index 1b47db1..c719695 100644 --- a/roles/ota/apps/tasks/main.yml +++ b/roles/ota/apps/tasks/main.yml @@ -1,38 +1,39 @@ +--- - name: Clone repository ansible.builtin.git: - repo: '{{ ota_source_repository }}' - dest: '/home/{{ ansible_user }}/{{ ota_directory }}' - version: '{{ ota_source_repository_branch }}' + repo: "{{ ota_source_repository }}" + dest: /home/{{ ansible_user }}/{{ ota_directory }} + version: "{{ ota_source_repository_branch }}" force: true - key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' + key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key depth: 1 - name: Read the production config - shell: cat '/home/{{ ansible_user }}/{{ ota_directory }}/config/production.json' + ansible.builtin.command: cat '/home/{{ ansible_user }}/{{ ota_directory }}/config/production.json' register: read_config - + - name: Save production config data into a variable - set_fact: + ansible.builtin.set_fact: ota_apps_config: "{{ read_config.stdout | from_json }}" -- block: - - name: Install dependencies - ansible.builtin.command: - cmd: npm ci - chdir: '/home/{{ ansible_user }}/{{ ota_directory }}' +- name: Set up environment variables and install dependencies + when: not ota_apps_read_config_only | bool + block: + - name: Install dependencies + ansible.builtin.command: + cmd: npm ci + chdir: /home/{{ ansible_user }}/{{ ota_directory }} - - name: Check if .env file is provided (on local machine) - ansible.builtin.stat: - path: "{{ inventory_dir }}/.env" - register: env_file - delegate_to: localhost + - name: Check if .env file is provided (on local machine) + ansible.builtin.stat: + path: "{{ inventory_dir }}/.env" + register: env_file + delegate_to: localhost - - name: Add .env file - ansible.builtin.copy: - src: "{{ inventory_dir }}/.env" - dest: '/home/{{ ansible_user }}/{{ ota_directory }}/.env' - force: true - mode: "644" - when: env_file.stat.exists - - when: not ota_apps_read_config_only | bool + - name: Add .env file + ansible.builtin.copy: + src: "{{ inventory_dir }}/.env" + dest: /home/{{ ansible_user }}/{{ ota_directory }}/.env + force: true + mode: "644" + when: env_file.stat.exists diff --git a/roles/ota/database/defaults/main.yml b/roles/ota/database/defaults/main.yml index 9b0f865..4b234ad 100644 --- a/roles/ota/database/defaults/main.yml +++ b/roles/ota/database/defaults/main.yml @@ -1,2 +1,3 @@ +--- ota_database_snapshots_branch: main ota_database_versions_branch: main diff --git a/roles/ota/database/meta/main.yml b/roles/ota/database/meta/main.yml index d0c083a..2f3426c 100644 --- a/roles/ota/database/meta/main.yml +++ b/roles/ota/database/meta/main.yml @@ -1,3 +1,4 @@ +--- dependencies: - { role: common } - { role: github } diff --git a/roles/ota/database/tasks/database.yml b/roles/ota/database/tasks/database.yml index a83a01b..a05aab6 100644 --- a/roles/ota/database/tasks/database.yml +++ b/roles/ota/database/tasks/database.yml @@ -1,40 +1,41 @@ -- name: "Check if {{ engine_database_name }} base data has already been obtained from {{ engine_database_repository }}" +--- +- name: Check if {{ engine_database_name }} base data has already been obtained from {{ engine_database_repository }} ansible.builtin.git: - repo: '{{ engine_database_repository }}' - version: '{{ engine_database_branch }}' - dest: '{{ engine_database_directory }}' + repo: "{{ engine_database_repository }}" + version: "{{ engine_database_branch }}" + dest: "{{ engine_database_directory }}" clone: false update: false - key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' + key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key # the `before` property of the return value can tell us if the repository has been cloned already or not, # see register: existing_repository - name: Obtain {{ engine_database_name }} initial data from branch {{ engine_database_branch }} of {{ engine_database_repository }} ansible.builtin.git: - repo: '{{ engine_database_repository }}' - version: '{{ engine_database_branch }}' - dest: '{{ engine_database_directory }}' - key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' + repo: "{{ engine_database_repository }}" + version: "{{ engine_database_branch }}" + dest: "{{ engine_database_directory }}" + key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key when: existing_repository.before is defined and not existing_repository.before # if existing_repository.before is null, then the repository is new - name: Configure {{ engine_database_name }} repository to use the GitHub Bot SSH Key - ansible.builtin.git_config: + community.general.git_config: name: core.sshCommand value: ssh -i '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' scope: local - repo: '{{ engine_database_directory }}' + repo: "{{ engine_database_directory }}" - name: Remove existing locks in {{ engine_database_name }} ansible.builtin.file: - path: '{{ engine_database_directory }}/.git/index.lock' + path: "{{ engine_database_directory }}/.git/index.lock" state: absent - name: Get latest data from {{ engine_database_repository }} ansible.builtin.git: - repo: '{{ engine_database_repository }}' - version: '{{ engine_database_branch }}' - dest: '{{ engine_database_directory }}' + repo: "{{ engine_database_repository }}" + version: "{{ engine_database_branch }}" + dest: "{{ engine_database_directory }}" update: true force: true # discard any modified files - key_file: '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' + key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key diff --git a/roles/ota/database/tasks/main.yml b/roles/ota/database/tasks/main.yml index b5088c2..a1c861a 100644 --- a/roles/ota/database/tasks/main.yml +++ b/roles/ota/database/tasks/main.yml @@ -1,3 +1,4 @@ +--- - name: Setup snapshots git repository ansible.builtin.include_tasks: database.yml when: ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository is defined @@ -5,7 +6,7 @@ engine_database_name: snapshots engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository }}" engine_database_branch: "{{ ota_database_snapshots_branch }}" - engine_database_directory: "/home/{{ ansible_user }}/{{ ota_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }}" + engine_database_directory: /home/{{ ansible_user }}/{{ ota_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }} - name: Setup versions git repository ansible.builtin.include_tasks: database.yml @@ -14,4 +15,4 @@ engine_database_name: versions engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository }}" engine_database_branch: "{{ ota_database_versions_branch }}" - engine_database_directory: "/home/{{ ansible_user }}/{{ ota_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }}" + engine_database_directory: /home/{{ ansible_user }}/{{ ota_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }} diff --git a/roles/pm2/README.md b/roles/pm2/README.md deleted file mode 100644 index 3ccafa4..0000000 --- a/roles/pm2/README.md +++ /dev/null @@ -1 +0,0 @@ -# PM2 diff --git a/roles/pm2/defaults/main.yml b/roles/pm2/defaults/main.yml index 5b1b8ab..94dcbcc 100644 --- a/roles/pm2/defaults/main.yml +++ b/roles/pm2/defaults/main.yml @@ -1 +1,2 @@ +--- ota_pm2_install_only: false diff --git a/roles/pm2/meta/main.yml b/roles/pm2/meta/main.yml index d6477f8..3b80e22 100644 --- a/roles/pm2/meta/main.yml +++ b/roles/pm2/meta/main.yml @@ -1,3 +1,4 @@ +--- dependencies: - { role: common } - { role: node } diff --git a/roles/pm2/tasks/main.yml b/roles/pm2/tasks/main.yml index ab55264..a542cb5 100644 --- a/roles/pm2/tasks/main.yml +++ b/roles/pm2/tasks/main.yml @@ -1,30 +1,31 @@ +--- - name: Install PM2 to latest version 5 ansible.builtin.command: npm install -g pm2@5 --production=true when: ota_pm2_install_only | bool -- block: - - name: Add pm2.config.cjs file - ansible.builtin.copy: - src: "{{ inventory_dir }}/pm2.config.cjs" - dest: '/home/{{ ansible_user }}/{{ ota_directory }}/pm2.config.cjs' - force: true - mode: "644" - - - name: Stop Open Terms Archive applications - ansible.builtin.command: - cmd: pm2 stop pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_directory }}' - tags: - - stop +- name: Manage PM2 configuration and application lifecycle + when: not ota_pm2_install_only | bool + block: + - name: Add pm2.config.cjs file + ansible.builtin.copy: + src: "{{ inventory_dir }}/pm2.config.cjs" + dest: /home/{{ ansible_user }}/{{ ota_directory }}/pm2.config.cjs + force: true + mode: "644" - - name: Start Open Terms Archive applications - ansible.builtin.command: - cmd: pm2 startOrRestart pm2.config.cjs - chdir: '/home/{{ ansible_user }}/{{ ota_directory }}' - environment: - NODE_ENV: production - tags: - - restart - - start + - name: Stop Open Terms Archive applications + ansible.builtin.command: + cmd: pm2 stop pm2.config.cjs + chdir: /home/{{ ansible_user }}/{{ ota_directory }} + tags: + - stop - when: not ota_pm2_install_only | bool + - name: Start Open Terms Archive applications + ansible.builtin.command: + cmd: pm2 startOrRestart pm2.config.cjs + chdir: /home/{{ ansible_user }}/{{ ota_directory }} + environment: + NODE_ENV: production + tags: + - restart + - start From b6f623c81173c54c2082e5fd1720982b8cb69d20 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 11:15:49 +0200 Subject: [PATCH 29/49] Improve wording Co-authored-by: Matti Schneider --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 79725f8..44d3abe 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repository contains the `opentermsarchive.deployment` Ansible collection. This Ansible collection provides playbooks to set up the infrastructure of and deploy Open Terms Archive applications. -To prevent confusion between the notion of Ansible [Collection](https://docs.ansible.com/ansible/latest/collections_guide/index.html) and an Open Terms Archive [Collection](https://docs.opentermsarchive.org/#collection), we will refer to Ansible Collection only as “Playbook”, as this is the main entry point to interact with it. +To prevent confusion between the notion of Ansible [Collection](https://docs.ansible.com/ansible/latest/collections_guide/index.html) and an Open Terms Archive [Collection](https://docs.opentermsarchive.org/#collection), this documentation will refer to Ansible Collection only as “Playbook”, as this is the main entry point to interact with it. ## Installation @@ -38,7 +38,7 @@ The playbook can be executed using the `ansible-playbook` command-line tool: ansible-playbook opentermsarchive.deployment.deploy ``` -_It is possible to check a playbook execution without actually applying changes with `check` and `diff` options:_ +_It is possible to check the playbook execution without actually applying changes with the `check` and `diff` options:_ ```sh ansible-playbook opentermsarchive.deployment.deploy --check --diff @@ -58,7 +58,7 @@ Available [variables](https://docs.ansible.com/ansible/latest/playbook_guide/pla | `ota_source_repository_branch` | [Git branch or tag](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftree-ishatree-ishalsotreeish) of the source repository | `main` | | | `ota_directory` | Directory path where the code will be deployed on the server | Name of the repository | | -These variables can be defined in the inventory file, for example: +These variables are defined in the inventory file, for example: ```yml all: @@ -72,18 +72,18 @@ all: #### Additional files -The `deploy` playbook requires additional files to be placed alongside the `inventory.yml` file. These files are necessary for deploying Open Terms Archive applications properly. Below are the required and optional files and their purposes: +Beyond inventory variables, the playbook uses additional files alongside the `inventory.yml` file: | File | Description | Required | Encryption Required | |--------------------------|---------------------------------------------------------------------------------------------------------|----------|---------------------| -| `pm2.config.cjs` | Configuration file describing the processes to be started and managed by PM2 | ✔︎ | | +| `pm2.config.cjs` | Configuration for the [PM2](https://pm2.keymetrics.io) process manager used to start the applications | ✔︎ | | | `github-bot-private-key` | Private SSH key for accessing SSH Git URLs | Required if `ota_source_repository` is an SSH Git URL or if the URLs for versions and/or snapshots repositories in the `config/production.json` file of the source repository are SSH Git URLs | ✔︎ | -| `.env` | File defining environment variables required by the deployed application | | ✔︎ | +| `.env` | Environment variables of the deployed applications | | ✔︎ | Here is an example of the directory structure: ```plaintext -ops/ +deployment/ ├── inventory.yml ├── pm2.config.cjs ├── github-bot-private-key @@ -114,7 +114,7 @@ Or with a password file: ansible-playbook playbook.yml --vault-password-file vault.key ``` -Please note that the data will be stored unencrypted on the deployment server. +Please note that encrypted files will be decrypted and stored in plaintext on the deployment server. Always protect access to your production server. ### Refining playbook execution From ad5a05eb3de3e2b9faa388228911fd72872173c3 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 11:17:02 +0200 Subject: [PATCH 30/49] Improve documentation Co-authored-by: Matti Schneider --- README.md | 34 +++------------------------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 44d3abe..17914d6 100644 --- a/README.md +++ b/README.md @@ -190,40 +190,12 @@ pm2 logs #### Troubleshooting -If you encounter an error while running the playbook, such as: +If you encounter a `Permission denied` error at the “Gathering Facts” step of the playbook, this most likely means the Vagrant-managed SSH key is not loaded in your SSH agent. -```sh -PLAY [Deploy Open Terms Archive applications] ************************************************ - -TASK [Gathering Facts] *************************************************************************** -fatal: [127.0.0.1]: UNREACHABLE! => changed=false - msg: 'Failed to connect to the host via ssh: vagrant@127.0.0.1: Permission denied (publickey,password).' - unreachable: true -``` - -Do the following: - -1. Run `vagrant ssh-config` and note the `IdentityFile` output: +1. Identify which SSH key is used by Vagrant: ```sh -vagrant ssh-config -``` - -Example output: - -```plaintext -Host opentermsarchive_deployment - HostName 127.0.0.1 - User vagrant - Port 2222 - UserKnownHostsFile /dev/null - StrictHostKeyChecking no - PasswordAuthentication no - IdentityFile /path/to/your/vagrant/private_key - IdentitiesOnly yes - LogLevel FATAL - PubkeyAcceptedKeyTypes +ssh-rsa - HostKeyAlgorithms +ssh-rsa +vagrant ssh-config | grep IdentityFile ``` 2. Add the private key to your SSH agent: From f857f5c5271909d424241d954f7b31563d4363a0 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 10:05:37 +0200 Subject: [PATCH 31/49] Improve roles structure --- playbooks/deploy.yml | 21 +++++-------- roles/nginx/README.md | 1 - roles/nginx/{ => configure}/handlers/main.yml | 0 roles/nginx/{ => configure}/meta/main.yml | 0 roles/nginx/{ => configure}/tasks/main.yml | 10 ------ roles/nginx/defaults/main.yml | 4 --- roles/nginx/install/meta/main.yml | 3 ++ roles/nginx/install/tasks/main.yml | 6 ++++ roles/pm2/defaults/main.yml | 2 -- roles/pm2/{ => install}/meta/main.yml | 0 roles/pm2/install/tasks/main.yml | 3 ++ roles/pm2/manage/meta/main.yml | 4 +++ roles/pm2/manage/tasks/main.yml | 29 +++++++++++++++++ roles/pm2/tasks/main.yml | 31 ------------------- 14 files changed, 52 insertions(+), 62 deletions(-) delete mode 100644 roles/nginx/README.md rename roles/nginx/{ => configure}/handlers/main.yml (100%) rename roles/nginx/{ => configure}/meta/main.yml (100%) rename roles/nginx/{ => configure}/tasks/main.yml (79%) delete mode 100644 roles/nginx/defaults/main.yml create mode 100644 roles/nginx/install/meta/main.yml create mode 100644 roles/nginx/install/tasks/main.yml delete mode 100644 roles/pm2/defaults/main.yml rename roles/pm2/{ => install}/meta/main.yml (100%) create mode 100644 roles/pm2/install/tasks/main.yml create mode 100644 roles/pm2/manage/meta/main.yml create mode 100644 roles/pm2/manage/tasks/main.yml delete mode 100644 roles/pm2/tasks/main.yml diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 0cebeab..3adec27 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -16,21 +16,16 @@ - name: Install PM2 ansible.builtin.include_role: - name: pm2 - vars: - ota_pm2_install_only: true + name: pm2/install - name: Install Chromium ansible.builtin.include_role: name: chromium when: ota_apps_config['@opentermsarchive/engine'] is defined - - name: Install and setup Nginx + - name: Install Nginx ansible.builtin.include_role: - name: nginx - vars: - ota_nginx_configure: true - ota_nginx_config_template: ./templates/nginx.conf.j2 + name: nginx/install - name: Install and setup Mongo ansible.builtin.include_role: @@ -58,15 +53,13 @@ - name: Start OTA applications ansible.builtin.include_role: - name: pm2 + name: pm2/manage - - name: Setup reverse proxy + - name: Configure NGINX ansible.builtin.include_role: - name: nginx + name: nginx/configure apply: become: true vars: - ota_nginx_install: false - ota_nginx_configure: false - ota_nginx_configure_reverse_proxy: true + ota_nginx_config_template: ./templates/nginx.conf.j2 ota_nginx_reverse_proxy_config_template: ./templates/nginx-reverse-proxy-conf.j2 diff --git a/roles/nginx/README.md b/roles/nginx/README.md deleted file mode 100644 index b30386d..0000000 --- a/roles/nginx/README.md +++ /dev/null @@ -1 +0,0 @@ -# Nginx diff --git a/roles/nginx/handlers/main.yml b/roles/nginx/configure/handlers/main.yml similarity index 100% rename from roles/nginx/handlers/main.yml rename to roles/nginx/configure/handlers/main.yml diff --git a/roles/nginx/meta/main.yml b/roles/nginx/configure/meta/main.yml similarity index 100% rename from roles/nginx/meta/main.yml rename to roles/nginx/configure/meta/main.yml diff --git a/roles/nginx/tasks/main.yml b/roles/nginx/configure/tasks/main.yml similarity index 79% rename from roles/nginx/tasks/main.yml rename to roles/nginx/configure/tasks/main.yml index f226e80..c7a0363 100644 --- a/roles/nginx/tasks/main.yml +++ b/roles/nginx/configure/tasks/main.yml @@ -1,14 +1,6 @@ --- -- name: Install NGINX package - ansible.builtin.apt: - name: nginx - update_cache: true - state: latest - when: ota_nginx_install | bool - - name: Configure NGINX notify: Restart NGINX - when: ota_nginx_configure | bool block: - name: Clean default NGINX site ansible.builtin.file: @@ -24,9 +16,7 @@ mode: "644" - name: Configure NGINX Reverse Proxy - become: true notify: Restart NGINX - when: ota_nginx_configure_reverse_proxy | bool block: - name: Add conf in NGINX sites-available ansible.builtin.template: diff --git a/roles/nginx/defaults/main.yml b/roles/nginx/defaults/main.yml deleted file mode 100644 index fe113b0..0000000 --- a/roles/nginx/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -ota_nginx_install: true -ota_nginx_configure: false -ota_nginx_configure_reverse_proxy: false diff --git a/roles/nginx/install/meta/main.yml b/roles/nginx/install/meta/main.yml new file mode 100644 index 0000000..6b4fff8 --- /dev/null +++ b/roles/nginx/install/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: common } diff --git a/roles/nginx/install/tasks/main.yml b/roles/nginx/install/tasks/main.yml new file mode 100644 index 0000000..c48b5af --- /dev/null +++ b/roles/nginx/install/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Install NGINX package + ansible.builtin.apt: + name: nginx + update_cache: true + state: latest diff --git a/roles/pm2/defaults/main.yml b/roles/pm2/defaults/main.yml deleted file mode 100644 index 94dcbcc..0000000 --- a/roles/pm2/defaults/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -ota_pm2_install_only: false diff --git a/roles/pm2/meta/main.yml b/roles/pm2/install/meta/main.yml similarity index 100% rename from roles/pm2/meta/main.yml rename to roles/pm2/install/meta/main.yml diff --git a/roles/pm2/install/tasks/main.yml b/roles/pm2/install/tasks/main.yml new file mode 100644 index 0000000..5c4637b --- /dev/null +++ b/roles/pm2/install/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Install PM2 to latest version 5 + ansible.builtin.command: npm install --global pm2@5 --production=true diff --git a/roles/pm2/manage/meta/main.yml b/roles/pm2/manage/meta/main.yml new file mode 100644 index 0000000..3b80e22 --- /dev/null +++ b/roles/pm2/manage/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - { role: common } + - { role: node } diff --git a/roles/pm2/manage/tasks/main.yml b/roles/pm2/manage/tasks/main.yml new file mode 100644 index 0000000..a978952 --- /dev/null +++ b/roles/pm2/manage/tasks/main.yml @@ -0,0 +1,29 @@ +- name: Check if pm2.config.cjs file is provided + ansible.builtin.stat: + path: /home/{{ ansible_user }}/{{ ota_directory }}/pm2.config.cjs + register: pm2_config + +- name: Stop Open Terms Archive applications + ansible.builtin.command: + cmd: pm2 stop pm2.config.cjs + chdir: /home/{{ ansible_user }}/{{ ota_directory }} + tags: + - stop + when: pm2_config.stat.exists + +- name: Copy pm2.config.cjs file + ansible.builtin.copy: + src: "{{ inventory_dir }}/pm2.config.cjs" + dest: /home/{{ ansible_user }}/{{ ota_directory }}/pm2.config.cjs + force: true + mode: "644" + +- name: Start Open Terms Archive applications + ansible.builtin.command: + cmd: pm2 startOrRestart pm2.config.cjs + chdir: /home/{{ ansible_user }}/{{ ota_directory }} + environment: + NODE_ENV: production + tags: + - restart + - start diff --git a/roles/pm2/tasks/main.yml b/roles/pm2/tasks/main.yml deleted file mode 100644 index a542cb5..0000000 --- a/roles/pm2/tasks/main.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: Install PM2 to latest version 5 - ansible.builtin.command: npm install -g pm2@5 --production=true - when: ota_pm2_install_only | bool - -- name: Manage PM2 configuration and application lifecycle - when: not ota_pm2_install_only | bool - block: - - name: Add pm2.config.cjs file - ansible.builtin.copy: - src: "{{ inventory_dir }}/pm2.config.cjs" - dest: /home/{{ ansible_user }}/{{ ota_directory }}/pm2.config.cjs - force: true - mode: "644" - - - name: Stop Open Terms Archive applications - ansible.builtin.command: - cmd: pm2 stop pm2.config.cjs - chdir: /home/{{ ansible_user }}/{{ ota_directory }} - tags: - - stop - - - name: Start Open Terms Archive applications - ansible.builtin.command: - cmd: pm2 startOrRestart pm2.config.cjs - chdir: /home/{{ ansible_user }}/{{ ota_directory }} - environment: - NODE_ENV: production - tags: - - restart - - start From e8eb61d9d553c4ab5058a5738ed91f7f1ed14732 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 10:06:01 +0200 Subject: [PATCH 32/49] Use long option name --- tests/Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Vagrantfile b/tests/Vagrantfile index 58657c6..f1fd94a 100644 --- a/tests/Vagrantfile +++ b/tests/Vagrantfile @@ -25,7 +25,7 @@ Vagrant.configure("2") do |config| # Provisioning step to install Python 3 and make it the default config.vm.provision "shell", inline: <<-SHELL apt-get update - apt-get install -y python3 python3-pip + apt-get install --assume-yes python3 python3-pip update-alternatives --install /usr/bin/python python /usr/bin/python3 1 SHELL end From 975d50a6f1e6989286ad21fd8343612dc826be33 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 10:50:12 +0200 Subject: [PATCH 33/49] Improve mongo role structure --- playbooks/deploy.yml | 31 ++++++++++++------- roles/mongo/{ => configure}/files/mongod.conf | 0 roles/mongo/configure/handlers/main.yml | 5 +++ roles/mongo/{ => configure}/meta/main.yml | 0 roles/mongo/configure/tasks/main.yml | 22 +++++++++++++ roles/mongo/install/meta/main.yml | 3 ++ roles/mongo/{ => install}/tasks/main.yml | 24 -------------- 7 files changed, 50 insertions(+), 35 deletions(-) rename roles/mongo/{ => configure}/files/mongod.conf (100%) create mode 100644 roles/mongo/configure/handlers/main.yml rename roles/mongo/{ => configure}/meta/main.yml (100%) create mode 100644 roles/mongo/configure/tasks/main.yml create mode 100644 roles/mongo/install/meta/main.yml rename roles/mongo/{ => install}/tasks/main.yml (69%) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 3adec27..d8bbac0 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -1,6 +1,12 @@ --- - name: Deploy Open Terms Archive applications hosts: all + vars: + mongo_required: > + (ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type == 'mongo') or + (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') + skip_debian_arm: ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64') # Skip Debian 11 with ARM architecture as it is not currently supported by MongoDB; see https://www.mongodb.com/docs/manual/installation/#supported-platforms + tasks: - name: Load OTA applications configs ansible.builtin.include_role: @@ -27,19 +33,22 @@ ansible.builtin.include_role: name: nginx/install - - name: Install and setup Mongo + - name: Install Mongo ansible.builtin.include_role: - name: mongo + name: mongo/install when: - - (ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type == 'mongo') or - (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') - # Skip Debian 11 with ARM architecture as it is not currently supported by MongoDB. - # See https://www.mongodb.com/docs/manual/installation/#supported-platforms - - ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64') - become: true - tags: - - infrastructure - + - mongo_condition + - skip_debian_arm + + - name: Configure Mongo + ansible.builtin.include_role: + name: mongo/configure + apply: + become: true + when: + - mongo_condition + - skip_debian_arm + - name: Setup OTA snapshots and versions databases ansible.builtin.include_role: name: ota/database diff --git a/roles/mongo/files/mongod.conf b/roles/mongo/configure/files/mongod.conf similarity index 100% rename from roles/mongo/files/mongod.conf rename to roles/mongo/configure/files/mongod.conf diff --git a/roles/mongo/configure/handlers/main.yml b/roles/mongo/configure/handlers/main.yml new file mode 100644 index 0000000..47b3720 --- /dev/null +++ b/roles/mongo/configure/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart MongoDB + ansible.builtin.service: + name: mongod + state: restarted diff --git a/roles/mongo/meta/main.yml b/roles/mongo/configure/meta/main.yml similarity index 100% rename from roles/mongo/meta/main.yml rename to roles/mongo/configure/meta/main.yml diff --git a/roles/mongo/configure/tasks/main.yml b/roles/mongo/configure/tasks/main.yml new file mode 100644 index 0000000..35991cd --- /dev/null +++ b/roles/mongo/configure/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: Configure MongoDB + notify: Restart MongoDB + block: + - name: Add mongod.conf + ansible.builtin.copy: + src: mongod.conf + dest: /etc/mongod.conf + mode: "644" + + - name: Create data directory + ansible.builtin.file: + path: /mnt/disk/mongodb + state: directory + mode: "744" + + - name: Set database files permissions + ansible.builtin.file: + path: /mnt/disk/mongodb + owner: mongodb + group: mongodb + recurse: true diff --git a/roles/mongo/install/meta/main.yml b/roles/mongo/install/meta/main.yml new file mode 100644 index 0000000..6b4fff8 --- /dev/null +++ b/roles/mongo/install/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: common } diff --git a/roles/mongo/tasks/main.yml b/roles/mongo/install/tasks/main.yml similarity index 69% rename from roles/mongo/tasks/main.yml rename to roles/mongo/install/tasks/main.yml index 92a8d80..88598f6 100644 --- a/roles/mongo/tasks/main.yml +++ b/roles/mongo/install/tasks/main.yml @@ -26,27 +26,3 @@ name: mongodb-org update_cache: true state: latest - -- name: Add mongod.conf - ansible.builtin.copy: - src: mongod.conf - dest: /etc/mongod.conf - mode: "644" - -- name: Create data directory - ansible.builtin.file: - path: /mnt/disk/mongodb - state: directory - mode: "744" - -- name: Set database files permissions - ansible.builtin.file: - path: /mnt/disk/mongodb - owner: mongodb - group: mongodb - recurse: true - -- name: Start MongoDB service - ansible.builtin.service: - name: mongod - state: restarted From 54b60b42e74996bbc70751d654b234cc6660634d Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 10:50:38 +0200 Subject: [PATCH 34/49] Make block more readable --- playbooks/deploy.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index d8bbac0..9247aec 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -15,7 +15,10 @@ vars: ota_apps_read_config_only: true - - block: + - name: Install infrastructure + block: + become: true + tags: [infrastructure] - name: Install Node ansible.builtin.include_role: name: node From 5b1fafefa677744c001d10e8adf8db78df670f4a Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 10:51:05 +0200 Subject: [PATCH 35/49] Improve deploy playbook --- playbooks/deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 9247aec..e87da31 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -2,6 +2,7 @@ - name: Deploy Open Terms Archive applications hosts: all vars: + chromium_required: ota_apps_config['@opentermsarchive/engine'] is defined mongo_required: > (ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type == 'mongo') or (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') @@ -30,7 +31,7 @@ - name: Install Chromium ansible.builtin.include_role: name: chromium - when: ota_apps_config['@opentermsarchive/engine'] is defined + when: chromium_required - name: Install Nginx ansible.builtin.include_role: From a23df238f9ebe718f4c416e5ba639b930a8da80b Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 10:51:41 +0200 Subject: [PATCH 36/49] Rename role --- playbooks/deploy.yml | 4 ++-- roles/ota/{database => git-database}/defaults/main.yml | 0 roles/ota/{database => git-database}/meta/main.yml | 0 roles/ota/{database => git-database}/tasks/database.yml | 0 roles/ota/{database => git-database}/tasks/main.yml | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename roles/ota/{database => git-database}/defaults/main.yml (100%) rename roles/ota/{database => git-database}/meta/main.yml (100%) rename roles/ota/{database => git-database}/tasks/database.yml (100%) rename roles/ota/{database => git-database}/tasks/main.yml (100%) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index e87da31..118b0ff 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -53,12 +53,12 @@ - mongo_condition - skip_debian_arm - - name: Setup OTA snapshots and versions databases + - name: Setup Git-based databases ansible.builtin.include_role: - name: ota/database when: - ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository is defined or ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository is defined + name: ota/git-database - name: Setup OTA applications ansible.builtin.include_role: diff --git a/roles/ota/database/defaults/main.yml b/roles/ota/git-database/defaults/main.yml similarity index 100% rename from roles/ota/database/defaults/main.yml rename to roles/ota/git-database/defaults/main.yml diff --git a/roles/ota/database/meta/main.yml b/roles/ota/git-database/meta/main.yml similarity index 100% rename from roles/ota/database/meta/main.yml rename to roles/ota/git-database/meta/main.yml diff --git a/roles/ota/database/tasks/database.yml b/roles/ota/git-database/tasks/database.yml similarity index 100% rename from roles/ota/database/tasks/database.yml rename to roles/ota/git-database/tasks/database.yml diff --git a/roles/ota/database/tasks/main.yml b/roles/ota/git-database/tasks/main.yml similarity index 100% rename from roles/ota/database/tasks/main.yml rename to roles/ota/git-database/tasks/main.yml From 37568e7324e63d1de6ba56bea13b9126c59dcb73 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 10:51:47 +0200 Subject: [PATCH 37/49] Remove obsolete condition --- playbooks/deploy.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 118b0ff..fa0f190 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -55,9 +55,6 @@ - name: Setup Git-based databases ansible.builtin.include_role: - when: - - ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository is defined or - ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository is defined name: ota/git-database - name: Setup OTA applications From 36604f6cff35eb8828e6a2e41efa8b665066fb8a Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 10:52:09 +0200 Subject: [PATCH 38/49] Update test pm2 config --- tests/pm2.config.cjs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/pm2.config.cjs b/tests/pm2.config.cjs index 6d99bc1..09521d9 100644 --- a/tests/pm2.config.cjs +++ b/tests/pm2.config.cjs @@ -6,28 +6,34 @@ module.exports = { args: 'run start:schedule', max_restarts: 2, min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. - restart_delay: 10800000, + restart_delay: 180 * 60 * 1000, }, { name: 'ota-api', script: 'npm', args: 'run start:api', + min_uptime: '10s', + max_restarts: 10, + restart_delay: 1000, + exponential_backoff_restart_delay: true }, { name: 'ota-release', script: 'npm', args: 'run dataset:schedule', - max_restarts: 2, - min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. - restart_delay: 15 * 60 * 1000, + min_uptime: '10s', + max_restarts: 10, + restart_delay: 1000, + exponential_backoff_restart_delay: true }, { name: 'ota-federation-api', script: 'npm', args: 'run start:federation-api', - max_restarts: 2, - min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. - restart_delay: 15 * 60 * 1000, + min_uptime: '10s', + max_restarts: 10, + restart_delay: 1000, + exponential_backoff_restart_delay: true } ], }; From ee875e06c9e5d5e18be6047db558dafacd73ea9f Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 11:13:24 +0200 Subject: [PATCH 39/49] Fix deploy playbook --- playbooks/deploy.yml | 46 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index fa0f190..62dc3c2 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -17,32 +17,32 @@ ota_apps_read_config_only: true - name: Install infrastructure - block: become: true tags: [infrastructure] - - name: Install Node - ansible.builtin.include_role: - name: node - - - name: Install PM2 - ansible.builtin.include_role: - name: pm2/install + block: + - name: Install Node + ansible.builtin.include_role: + name: node + + - name: Install PM2 + ansible.builtin.include_role: + name: pm2/install - - name: Install Chromium - ansible.builtin.include_role: - name: chromium - when: chromium_required + - name: Install Chromium + ansible.builtin.include_role: + name: chromium + when: chromium_required - - name: Install Nginx - ansible.builtin.include_role: - name: nginx/install - - - name: Install Mongo - ansible.builtin.include_role: - name: mongo/install - when: - - mongo_condition - - skip_debian_arm + - name: Install Nginx + ansible.builtin.include_role: + name: nginx/install + + - name: Install Mongo + ansible.builtin.include_role: + name: mongo/install + when: + - mongo_required + - skip_debian_arm - name: Configure Mongo ansible.builtin.include_role: @@ -50,8 +50,8 @@ apply: become: true when: - - mongo_condition - skip_debian_arm + - mongo_required - name: Setup Git-based databases ansible.builtin.include_role: From faab2b7694a163c615daff4f91f4a9dc6dab7348 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 11:13:38 +0200 Subject: [PATCH 40/49] Factorize condition --- playbooks/deploy.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 62dc3c2..5fa21f2 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -3,10 +3,13 @@ hosts: all vars: chromium_required: ota_apps_config['@opentermsarchive/engine'] is defined + # Skip Debian 11 with ARM architecture as it is not currently supported by MongoDB; see https://www.mongodb.com/docs/manual/installation/#supported-platforms mongo_required: > - (ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type == 'mongo') or - (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') - skip_debian_arm: ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64') # Skip Debian 11 with ARM architecture as it is not currently supported by MongoDB; see https://www.mongodb.com/docs/manual/installation/#supported-platforms + ( ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64')) + and ( + (ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type == 'mongo') or + (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') + ) tasks: - name: Load OTA applications configs @@ -42,7 +45,6 @@ name: mongo/install when: - mongo_required - - skip_debian_arm - name: Configure Mongo ansible.builtin.include_role: @@ -50,7 +52,6 @@ apply: become: true when: - - skip_debian_arm - mongo_required - name: Setup Git-based databases From 118622aeb3e7d6250ee513d6716572cbc74587a3 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 11:17:50 +0200 Subject: [PATCH 41/49] Group entries in changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 148bdbf..a17ce5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,7 @@ _Full changeset and discussions: [#41](https://github.com/OpenTermsArchive/deplo - **Breaking:** Remove `ota_reverse_proxy_federated_api_path` config; define this path in your `config/production.json` under the key `@opentermsarchive/federation-api: { basePath: "" }` - **Breaking:** Remove `ota_reverse_proxy_engine_path` config; define this path in your `config/production.json` under the key `@opentermsarchive/engine: { collection-api: { basePath: "" } }` -- **Breaking:** Remove obsolete `ota_federated_api_branch`, `ota_federated_api_directory`, and `ota_federated_api_repo` configs -- **Breaking:** Remove obsolete `ota_engine_config_path` config +- **Breaking:** Remove obsolete `ota_engine_config_path`, `ota_federated_api_branch`, `ota_federated_api_directory`, and `ota_federated_api_repo` configs - **Breaking:** Remove obsolete `update-declarations` tag From 08592ec65b8bca526933b71c1684f7b81d422e09 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 14:51:49 +0200 Subject: [PATCH 42/49] Update Readme --- README.md | 66 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 17914d6..2feb06c 100644 --- a/README.md +++ b/README.md @@ -46,17 +46,29 @@ ansible-playbook opentermsarchive.deployment.deploy --check --diff > See “[Using collections](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html)” in Ansible’s user guide for more information about Ansible collections. -- - - +## Configuration + +Configuration is done through various files located in the `deployment` folder. Below is an example of the directory structure: + +```plaintext +deployment/ + ├── inventory.yml + ├── pm2.config.cjs + ├── github-bot-private-key + └── .env +``` + +- ### Inventory File — `inventory.yml` -### Configuration +**This file is mandatory** -Available [variables](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html) are listed below, along with their default values: +The `inventory.yml` file defines the hosts and the variables required for the deployment. This file should contain all the necessary variables as described below. -| Variable | Description | Default Value | Required | -|--------------------------------|----------------------------------------------------------------------------------------------------------------------------|------------------------|----------| -| `ota_source_repository` | URL of the source repository | No default value | ✔︎ | -| `ota_source_repository_branch` | [Git branch or tag](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftree-ishatree-ishalsotreeish) of the source repository | `main` | | -| `ota_directory` | Directory path where the code will be deployed on the server | Name of the repository | | +| Variable | Description | Required or default Value | +|--------------------------------|----------------------------------------------------------------------------------------------------------------------------|------------------------| +| `ota_source_repository` | URL of the declarations repository to deploy | **required** | +| `ota_source_repository_branch` | [Git branch or tag](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftree-ishatree-ishalsotreeish) of the source repository | `main` | +| `ota_directory` | Directory path where the code will be deployed on the server | Name of the repository | These variables are defined in the inventory file, for example: @@ -66,35 +78,33 @@ all: 127.0.0.1: ansible_user: debian ota_source_repository: https://github.com/OpenTermsArchive/demo-declarations.git - ota_source_repository_branch: main - ota_directory: demo + ota_source_repository_branch: master + ota_directory: opentermsarchive-demo ``` -#### Additional files +- ### PM2 Configuration File — `pm2.config.cjs` -Beyond inventory variables, the playbook uses additional files alongside the `inventory.yml` file: +**This file is mandatory** -| File | Description | Required | Encryption Required | -|--------------------------|---------------------------------------------------------------------------------------------------------|----------|---------------------| -| `pm2.config.cjs` | Configuration for the [PM2](https://pm2.keymetrics.io) process manager used to start the applications | ✔︎ | | -| `github-bot-private-key` | Private SSH key for accessing SSH Git URLs | Required if `ota_source_repository` is an SSH Git URL or if the URLs for versions and/or snapshots repositories in the `config/production.json` file of the source repository are SSH Git URLs | ✔︎ | -| `.env` | Environment variables of the deployed applications | | ✔︎ | +The `pm2.config.cjs` file is used to configure the [PM2](https://pm2.keymetrics.io) process manager, which is used to start the applications. -Here is an example of the directory structure: +- ### GitHub Bot Private Key — `github-bot-private-key` -```plaintext -deployment/ - ├── inventory.yml - ├── pm2.config.cjs - ├── github-bot-private-key - └── .env -``` +The `github-bot-private-key` file contains a private SSH key for accessing and pushing to SSH Git URLs. This file is required if `ota_source_repository` is an SSH Git URL or if the URLs for versions and/or snapshots repositories in the `config/production.json` file of the source repository are SSH Git URLs. + +It is strongly recommended to [encrypt this file](#file-encryption) if is is checked in to a public repository. + +- ### Environment Variables File — `.env` + +The `.env` file contains the environment variables for the deployed applications. + +It is strongly recommended to [encrypt this file](#file-encryption) if is is checked in to a public repository. -#### Encrypting sensitive configuration files +## File encryption Sensitive configuration files should be encrypted using [Ansible Vault](https://docs.ansible.com/ansible/latest/vault_guide/index.html). -Examples: +Examples of encrypting and decrypting a file: - Encrypt the `github-bot-private-key` file: `ansible-vault encrypt github-bot-private-key` - Decrypt the `github-bot-private-key` file: `ansible-vault decrypt github-bot-private-key` @@ -116,7 +126,7 @@ ansible-playbook playbook.yml --vault-password-file vault.key Please note that encrypted files will be decrypted and stored in plaintext on the deployment server. Always protect access to your production server. -### Refining playbook execution +## Playbook execution refinement Use [tags](https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html) to refine playbook execution. Example commands: From afccdf0caeb809c39e29eb3f14294a4fff0cead4 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 14:59:00 +0200 Subject: [PATCH 43/49] Avoid regex --- roles/ota/apps/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ota/apps/defaults/main.yml b/roles/ota/apps/defaults/main.yml index 73a9514..979421f 100644 --- a/roles/ota/apps/defaults/main.yml +++ b/roles/ota/apps/defaults/main.yml @@ -1,4 +1,4 @@ --- ota_apps_read_config_only: false ota_source_repository_branch: main -ota_directory: "{{ ota_source_repository | urlsplit('path') | regex_replace('^.*/(.+?)(\\.git)?$', '\\1') }}" # extract the repository name from the URL +ota_directory: "{{ ota_source_repository | ansible.builtin.urlsplit('path') | ansible.builtin.basename | ansible.builtin.splitext | first }}" # extract the repository name from the URL From cf20df2e74de5d05e9b6601eb238865ef30834f7 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 16:10:36 +0200 Subject: [PATCH 44/49] Fix variable definition --- playbooks/deploy.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 5fa21f2..ca1fd14 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -1,16 +1,6 @@ --- - name: Deploy Open Terms Archive applications hosts: all - vars: - chromium_required: ota_apps_config['@opentermsarchive/engine'] is defined - # Skip Debian 11 with ARM architecture as it is not currently supported by MongoDB; see https://www.mongodb.com/docs/manual/installation/#supported-platforms - mongo_required: > - ( ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64')) - and ( - (ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type == 'mongo') or - (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') - ) - tasks: - name: Load OTA applications configs ansible.builtin.include_role: @@ -19,6 +9,19 @@ vars: ota_apps_read_config_only: true + - name: Set required variables + set_fact: + chromium_required: "{{ ota_apps_config['@opentermsarchive/engine'] is defined }}" + # Skip Debian 11 with ARM architecture as it is not currently supported by MongoDB; see https://www.mongodb.com/docs/manual/installation/#supported-platforms + mongo_required: + "{{ + (ansible_distribution != 'Debian' or (ansible_distribution == 'Debian' and ansible_facts['architecture'] != 'aarch64')) + and ( + (ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.type == 'mongo') + or + (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') + ) | bool + }}" - name: Install infrastructure become: true tags: [infrastructure] From 6d802066d6134544b98a51e7bf109a6d46f6853f Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 16:11:33 +0200 Subject: [PATCH 45/49] Improve git based database role --- playbooks/deploy.yml | 23 +++++++++- roles/ota/git-database/defaults/main.yml | 3 -- roles/ota/git-database/tasks/database.yml | 41 ----------------- roles/ota/git-database/tasks/main.yml | 55 ++++++++++++++++------- 4 files changed, 61 insertions(+), 61 deletions(-) delete mode 100644 roles/ota/git-database/defaults/main.yml delete mode 100644 roles/ota/git-database/tasks/database.yml diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index ca1fd14..a6748f9 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -22,6 +22,11 @@ (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') ) | bool }}" + snapshots_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository }}" + snapshots_path: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }}" + versions_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository }}" + versions_path: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }}" + - name: Install infrastructure become: true tags: [infrastructure] @@ -57,9 +62,25 @@ when: - mongo_required - - name: Setup Git-based databases + - name: Setup Git-based versions database ansible.builtin.include_role: name: ota/git-database + vars: + ota_git_database_repository: "{{ versions_repository }}" + ota_git_database_directory: "{{ versions_path }}" + ota_git_database_branch: main + when: + - versions_repository and versions_path + + - name: Setup Git-based snapshots database + ansible.builtin.include_role: + name: ota/git-database + vars: + ota_git_database_repository: "{{ snapshots_repository }}" + ota_git_database_directory: "{{ snapshots_path }}" + ota_git_database_branch: main + when: + - snapshots_repository and snapshots_path - name: Setup OTA applications ansible.builtin.include_role: diff --git a/roles/ota/git-database/defaults/main.yml b/roles/ota/git-database/defaults/main.yml deleted file mode 100644 index 4b234ad..0000000 --- a/roles/ota/git-database/defaults/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -ota_database_snapshots_branch: main -ota_database_versions_branch: main diff --git a/roles/ota/git-database/tasks/database.yml b/roles/ota/git-database/tasks/database.yml deleted file mode 100644 index a05aab6..0000000 --- a/roles/ota/git-database/tasks/database.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- name: Check if {{ engine_database_name }} base data has already been obtained from {{ engine_database_repository }} - ansible.builtin.git: - repo: "{{ engine_database_repository }}" - version: "{{ engine_database_branch }}" - dest: "{{ engine_database_directory }}" - clone: false - update: false - key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key - # the `before` property of the return value can tell us if the repository has been cloned already or not, - # see - register: existing_repository - -- name: Obtain {{ engine_database_name }} initial data from branch {{ engine_database_branch }} of {{ engine_database_repository }} - ansible.builtin.git: - repo: "{{ engine_database_repository }}" - version: "{{ engine_database_branch }}" - dest: "{{ engine_database_directory }}" - key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key - when: existing_repository.before is defined and not existing_repository.before # if existing_repository.before is null, then the repository is new - -- name: Configure {{ engine_database_name }} repository to use the GitHub Bot SSH Key - community.general.git_config: - name: core.sshCommand - value: ssh -i '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' - scope: local - repo: "{{ engine_database_directory }}" - -- name: Remove existing locks in {{ engine_database_name }} - ansible.builtin.file: - path: "{{ engine_database_directory }}/.git/index.lock" - state: absent - -- name: Get latest data from {{ engine_database_repository }} - ansible.builtin.git: - repo: "{{ engine_database_repository }}" - version: "{{ engine_database_branch }}" - dest: "{{ engine_database_directory }}" - update: true - force: true # discard any modified files - key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key diff --git a/roles/ota/git-database/tasks/main.yml b/roles/ota/git-database/tasks/main.yml index a1c861a..c0d4754 100644 --- a/roles/ota/git-database/tasks/main.yml +++ b/roles/ota/git-database/tasks/main.yml @@ -1,18 +1,41 @@ --- -- name: Setup snapshots git repository - ansible.builtin.include_tasks: database.yml - when: ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository is defined - vars: - engine_database_name: snapshots - engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository }}" - engine_database_branch: "{{ ota_database_snapshots_branch }}" - engine_database_directory: /home/{{ ansible_user }}/{{ ota_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }} +- name: Check if base data has already been obtained from {{ ota_git_database_repository }} + ansible.builtin.git: + repo: "{{ ota_git_database_repository }}" + version: "{{ ota_git_database_branch }}" + dest: "{{ ota_git_database_directory }}" + clone: false + update: false + key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key + # the `before` property of the return value can tell us if the repository has been cloned already or not, + # see + register: existing_repository -- name: Setup versions git repository - ansible.builtin.include_tasks: database.yml - when: ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository is defined - vars: - engine_database_name: versions - engine_database_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository }}" - engine_database_branch: "{{ ota_database_versions_branch }}" - engine_database_directory: /home/{{ ansible_user }}/{{ ota_directory }}/{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }} +- name: Obtain initial data from branch {{ ota_git_database_branch }} of {{ ota_git_database_repository }} + ansible.builtin.git: + repo: "{{ ota_git_database_repository }}" + version: "{{ ota_git_database_branch }}" + dest: "{{ ota_git_database_directory }}" + key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key + when: existing_repository.before is defined and not existing_repository.before # if existing_repository.before is null, then the repository is new + +- name: Configure {{ ota_git_database_directory }} repository to use the GitHub Bot SSH Key + community.general.git_config: + name: core.sshCommand + value: ssh -i '/home/{{ ansible_user }}/.ssh/ota-github-bot-key' + scope: local + repo: "{{ ota_git_database_directory }}" + +- name: Remove existing locks in {{ ota_git_database_directory }} + ansible.builtin.file: + path: "{{ ota_git_database_directory }}/.git/index.lock" + state: absent + +- name: Get latest data from {{ ota_git_database_repository }} + ansible.builtin.git: + repo: "{{ ota_git_database_repository }}" + version: "{{ ota_git_database_branch }}" + dest: "{{ ota_git_database_directory }}" + update: true + force: true # discard any modified files + key_file: /home/{{ ansible_user }}/.ssh/ota-github-bot-key From 407b4701057caf8d796c0281c588090fd0380ac8 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 29 May 2024 16:22:46 +0200 Subject: [PATCH 46/49] Use ota_apps_config variables only in the playbook --- playbooks/deploy.yml | 8 ++++++++ playbooks/templates/nginx-reverse-proxy-conf.j2 | 12 ++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index a6748f9..3a79e11 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -22,10 +22,18 @@ (ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.type == 'mongo') ) | bool }}" + snapshots_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.repository }}" snapshots_path: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path is defined and ota_apps_config['@opentermsarchive/engine'].recorder.snapshots.storage.git.path }}" + versions_repository: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.repository }}" versions_path: "{{ ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path is defined and ota_apps_config['@opentermsarchive/engine'].recorder.versions.storage.git.path }}" + + collection_api_basePath: "{{ ota_apps_config['@opentermsarchive/engine']['collection-api'].basePath is defined and ota_apps_config['@opentermsarchive/engine']['collection-api'].basePath }}" + collection_api_port: "{{ ota_apps_config['@opentermsarchive/engine']['collection-api'].port is defined and ota_apps_config['@opentermsarchive/engine']['collection-api'].port }}" + + federation_api_basePath: "{{ ota_apps_config['@opentermsarchive/federation-api'].basePath is defined and ota_apps_config['@opentermsarchive/federation-api'].basePath }}" + federation_api_port: "{{ ota_apps_config['@opentermsarchive/federation-api'].port is defined and ota_apps_config['@opentermsarchive/federation-api'].port }}" - name: Install infrastructure become: true diff --git a/playbooks/templates/nginx-reverse-proxy-conf.j2 b/playbooks/templates/nginx-reverse-proxy-conf.j2 index dd066c5..5a452fb 100644 --- a/playbooks/templates/nginx-reverse-proxy-conf.j2 +++ b/playbooks/templates/nginx-reverse-proxy-conf.j2 @@ -4,20 +4,20 @@ server { listen 80; server_name {{ inventory_hostname }}; -{% if ota_apps_config['@opentermsarchive/engine']['collection-api'].basePath is defined and ota_apps_config['@opentermsarchive/engine']['collection-api'].port is defined %} - location {{ ota_apps_config['@opentermsarchive/engine']['collection-api'].basePath }} { +{% if collection_api_basePath and collection_api_port %} + location {{ collection_api_basePath }} { # Allowing for a `burst` of up to 5 requests beyond the specified rate limit. The `nodelay` parameter ensures that excessive requests beyond the burst limit are immediately rejected with a 429 error response instead of being queued. See https://www.nginx.com/blog/rate-limiting-nginx/. limit_req zone=limited burst=5 nodelay; - proxy_pass http://localhost:{{ ota_apps_config['@opentermsarchive/engine']['collection-api'].port }}; + proxy_pass http://localhost:{{ collection_api_port }}; proxy_redirect off; } {% endif %} -{% if ota_apps_config['@opentermsarchive/federation-api'].basePath is defined and ota_apps_config['@opentermsarchive/federation-api'].port is defined %} - location {{ ota_apps_config['@opentermsarchive/federation-api'].basePath }} { +{% if federation_api_basePath and federation_api_port %} + location {{ federation_api_basePath }} { # Allowing for a `burst` of up to 5 requests beyond the specified rate limit. The `nodelay` parameter ensures that excessive requests beyond the burst limit are immediately rejected with a 429 error response instead of being queued. See https://www.nginx.com/blog/rate-limiting-nginx/. limit_req zone=limited burst=5 nodelay; - proxy_pass http://localhost:{{ ota_apps_config['@opentermsarchive/federation-api'].port }}; + proxy_pass http://localhost:{{ federation_api_port }}; proxy_redirect off; } {% endif %} From 634498f723bae72cab7982319cac442cfd3dd784 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Fri, 31 May 2024 14:33:14 +0200 Subject: [PATCH 47/49] Improve restart delays Co-authored-by: Matti Schneider --- tests/pm2.config.cjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pm2.config.cjs b/tests/pm2.config.cjs index 09521d9..05ebc5c 100644 --- a/tests/pm2.config.cjs +++ b/tests/pm2.config.cjs @@ -6,7 +6,7 @@ module.exports = { args: 'run start:schedule', max_restarts: 2, min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable. - restart_delay: 180 * 60 * 1000, + restart_delay: 3 * 60 * 60 * 1000, // likely related to a connectivity problem that will take some time to be fixed }, { name: 'ota-api', @@ -23,7 +23,7 @@ module.exports = { args: 'run dataset:schedule', min_uptime: '10s', max_restarts: 10, - restart_delay: 1000, + restart_delay: 60 * 60 * 1000, // likely related to a GitHub availability problem that will take some time to be fixed exponential_backoff_restart_delay: true }, { From a13f2d5dd613e1e59099abce2e51c2d3327d4f97 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Fri, 31 May 2024 14:34:32 +0200 Subject: [PATCH 48/49] Minor README improvements --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2feb06c..f7cb3fb 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ deployment/ The `inventory.yml` file defines the hosts and the variables required for the deployment. This file should contain all the necessary variables as described below. -| Variable | Description | Required or default Value | +| Variable | Description | Required or default value | |--------------------------------|----------------------------------------------------------------------------------------------------------------------------|------------------------| | `ota_source_repository` | URL of the declarations repository to deploy | **required** | | `ota_source_repository_branch` | [Git branch or tag](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddeftree-ishatree-ishalsotreeish) of the source repository | `main` | @@ -79,7 +79,7 @@ all: ansible_user: debian ota_source_repository: https://github.com/OpenTermsArchive/demo-declarations.git ota_source_repository_branch: master - ota_directory: opentermsarchive-demo + ota_directory: demo ``` - ### PM2 Configuration File — `pm2.config.cjs` From c3200a0a2a1f4fe1324c72f43166fc69e2689010 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Fri, 31 May 2024 14:36:25 +0200 Subject: [PATCH 49/49] Remove obsolete test config --- tests/inventory.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/inventory.yml b/tests/inventory.yml index aa776ab..4940463 100644 --- a/tests/inventory.yml +++ b/tests/inventory.yml @@ -4,6 +4,4 @@ vagrant: ansible_user: vagrant ansible_port: 2222 ansible_ssh_host: 127.0.0.1 - ota_source_repository: https://github.com/OpenTermsArchive/demo-declarations.git - ota_source_repository_branch: test-new-config