diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..5626e6c4 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,93 @@ +# Workflow to deploy NCBO Cron to stage/prod systems +# +# Required github secrets: +# +# CONFIG_REPO - github repo containing config and customizations for NCBO_CRON. Format 'author/private_config_repo' +# it is used for getting capistrano deployment configuration for stages on the github actions runner and +# PRIVATE_CONFIG_REPO env var is constructed from it which is used by capistrano on the API hosts for pulling configs. +# +# GH_PAT - github Personal Access Token for accessing private config repo +# +# SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if API nodes live on private network. +# SSH_JUMPHOST_USER - username to use to connect to the ssh jump/proxy. +# +# DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/ +# this SSH key is used for accessing jump host, API nodes, and private github repo. + +name: Capistrano Deployment +# Controls when the action will run. +on: + push: + branches: + - stage + - development + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + inputs: + BRANCH: + description: "Branch/tag to deploy" + type: choice + options: + - stage + - development + - master + default: stage + required: true + environment: + description: "target environment to deploy to" + type: choice + options: + - staging + - test + - agroportal + default: stage +jobs: + deploy: + runs-on: ubuntu-latest + env: + BUNDLE_WITHOUT: default #install gems required primarely for deployment in order to speed up workflow + PRIVATE_CONFIG_REPO: ${{ format('git@github.com:{0}.git', secrets.CONFIG_REPO) }} + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: set branch/tag and environment to deploy from inputs + run: | + # workflow_dispatch default input doesn't get set on push so we need to set defaults + # via shell parameter expansion + + USER_INPUT_BRANCH="${{ inputs.branch || github.head_ref || 'master' }}" + echo "BRANCH=${USER_INPUT_BRANCH}" >> $GITHUB_ENV + + USER_INPUT_ENVIRONMENT=${{ inputs.environment }} + echo "TARGET=${USER_INPUT_ENVIRONMENT:-test}" >> $GITHUB_ENV + + CONFIG_REPO=${{ secrets.CONFIG_REPO }} + GH_PAT=${{ secrets.GH_PAT }} + echo "PRIVATE_CONFIG_REPO=https://${GH_PAT}@github.com/${CONFIG_REPO}" >> $GITHUB_ENV + + echo "SSH_JUMPHOST=${{ secrets.SSH_JUMPHOST }}" >> $GITHUB_ENV + echo "SSH_JUMPHOST_USER=${{ secrets.SSH_JUMPHOST_USER }}" >> $GITHUB_ENV + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7.8 # Not needed with a .ruby-version file + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: get-deployment-config + uses: actions/checkout@v3 + with: + repository: ${{ secrets.CONFIG_REPO }} # repository containing deployment settings + token: ${{ secrets.GH_PAT }} # `GH_PAT` is a secret that contains your PAT + path: deploy_config + - name: copy-deployment-config + run: cp -r deploy_config/ncbo_cron/${{ inputs.environment }}/* . + # add ssh hostkey so that capistrano doesn't complain + - name: Add jumphost's hostkey to Known Hosts + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SSH_JUMPHOST }}" + ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts + shell: bash + - uses: miloserdow/capistrano-deploy@master + with: + target: ${{ env.TARGET }} # which environment to deploy + deploy_key: ${{ secrets.DEPLOY_ENC_KEY }} # Name of the variable configured in Settings/Secrets of your github project \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3fdd0c6d..4c1af01d 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ repo* .DS_Store tmp +vendor/bundle/* + # Code coverage reports coverage* diff --git a/Capfile b/Capfile new file mode 100644 index 00000000..ea67e952 --- /dev/null +++ b/Capfile @@ -0,0 +1,40 @@ +# Load DSL and set up stages +require "capistrano/setup" + +# Include default deployment tasks +require "capistrano/deploy" + +# Load the SCM plugin appropriate to your project: +# +# require "capistrano/scm/hg" +# install_plugin Capistrano::SCM::Hg +# or +# require "capistrano/scm/svn" +# install_plugin Capistrano::SCM::Svn +# or +require "capistrano/scm/git" +install_plugin Capistrano::SCM::Git + +# Include tasks from other gems included in your Gemfile +# +# For documentation on these, see for example: +# +# https://github.com/capistrano/rvm +# https://github.com/capistrano/rbenv +# https://github.com/capistrano/chruby +# https://github.com/capistrano/bundler +# https://github.com/capistrano/rails +# https://github.com/capistrano/passenger +# +# require "capistrano/rvm" +require "capistrano/rbenv" +# require "capistrano/chruby" +require "capistrano/bundler" +# require "capistrano/rails/assets" +# require "capistrano/rails/migrations" +# require "capistrano/passenger" +require 'capistrano/locally' +#require 'new_relic/recipes' # announce deployments in NewRelic + +# Load custom tasks from `lib/capistrano/tasks` if you have any defined +Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r } diff --git a/Gemfile b/Gemfile index dcaf2c45..4410dcf8 100644 --- a/Gemfile +++ b/Gemfile @@ -40,4 +40,14 @@ group :test do gem 'test-unit-minitest' end +group :deployment do + # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 + gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false + gem 'capistrano', '~> 3', require: false + gem 'capistrano-bundler', require: false + gem 'capistrano-locally', require: false + gem 'capistrano-rbenv', require: false + gem 'ed25519', '>= 1.2', '< 2.0', require: false +end + gem "binding_of_caller", "~> 1.0" diff --git a/Gemfile.lock b/Gemfile.lock index 85a031c1..9ddc1f0e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -29,7 +29,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 6fd6960a1f134ec8c8aff07a5897bb0da12bcc8d + revision: e3cf9888c2113152aaa250ee0cfaae37c534a53a branch: development specs: ontologies_linked_data (0.0.1) @@ -79,12 +79,27 @@ GEM multi_json (~> 1.0) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) + airbrussh (1.5.3) + sshkit (>= 1.6.1, != 1.7.0) base64 (0.2.0) bcrypt (3.1.20) + bcrypt_pbkdf (1.1.1) bigdecimal (3.1.8) binding_of_caller (1.0.1) debug_inspector (>= 1.2.0) builder (3.3.0) + capistrano (3.19.2) + airbrussh (>= 1.0.0) + i18n + rake (>= 10.0.0) + sshkit (>= 1.9.0) + capistrano-bundler (2.1.1) + capistrano (~> 3.1) + capistrano-locally (0.3.0) + capistrano (~> 3.0) + capistrano-rbenv (2.2.0) + capistrano (~> 3.1) + sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.3.4) connection_pool (2.4.1) @@ -94,6 +109,7 @@ GEM declarative (0.0.20) docile (1.4.1) domain_name (0.6.20240107) + ed25519 (1.3.0) email_spec (2.1.1) htmlentities (~> 4.3.3) launchy (~> 2.1) @@ -193,6 +209,11 @@ GEM mutex_m (0.3.0) net-http-persistent (4.0.5) connection_pool (~> 2.2) + net-scp (4.0.0) + net-ssh (>= 2.6.5, < 8.0.0) + net-sftp (4.0.0) + net-ssh (>= 5.0.0, < 8.0.0) + net-ssh (7.3.0) netrc (0.11.0) oj (3.16.7) bigdecimal (>= 3.0) @@ -267,6 +288,12 @@ GEM simplecov (~> 0.19) simplecov-html (0.13.1) simplecov_json_formatter (0.1.4) + sshkit (1.23.2) + base64 + net-scp (>= 1.1.2) + net-sftp (>= 2.1.2) + net-ssh (>= 2.8.0) + ostruct sys-proctable (1.3.0) ffi (~> 1.1) systemu (2.6.5) @@ -283,8 +310,14 @@ PLATFORMS x86_64-linux DEPENDENCIES + bcrypt_pbkdf (>= 1.0, < 2.0) binding_of_caller (~> 1.0) + capistrano (~> 3) + capistrano-bundler + capistrano-locally + capistrano-rbenv cube-ruby + ed25519 (>= 1.2, < 2.0) email_spec ffi (~> 1.16.3) goo! diff --git a/config/deploy.rb b/config/deploy.rb new file mode 100644 index 00000000..59c11707 --- /dev/null +++ b/config/deploy.rb @@ -0,0 +1,105 @@ +set :author, "ontoportal-lirmm" +set :application, "ncbo_cron" +set :repo_url, "https://github.com/#{fetch(:author)}/#{fetch(:application)}.git" + +set :deploy_via, :remote_cache + +# Default branch is :master +# ask :branch, `git rev-parse --abbrev-ref HEAD`.chomp + +# Default deploy_to directory is /var/www/my_app_name +set :deploy_to, "/srv/ontoportal/#{fetch(:application)}" + +# Default value for :log_level is :debug +set :log_level, :debug + +# Default value for :linked_files is [] +# append :linked_files, "config/database.yml", 'config/master.key' + +# Default value for linked_dirs is [] +# set :linked_dirs, %w{log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system} +set :linked_dirs, %w{log vendor/bundle tmp/pids tmp/sockets public/system} + + +# Default value for keep_releases is 5 +set :keep_releases, 5 +set :config_folder_path, "#{fetch(:application)}/#{fetch(:stage)}" + + +# If you want to restart using `touch tmp/restart.txt`, add this to your config/deploy.rb: + +SSH_JUMPHOST = ENV.include?('SSH_JUMPHOST') ? ENV['SSH_JUMPHOST'] : 'jumpbox.hostname.com' +SSH_JUMPHOST_USER = ENV.include?('SSH_JUMPHOST_USER') ? ENV['SSH_JUMPHOST_USER'] : 'username' + +JUMPBOX_PROXY = "#{SSH_JUMPHOST_USER}@#{SSH_JUMPHOST}" +set :ssh_options, { + user: 'ontoportal', + forward_agent: 'true', + keys: %w(config/deploy_id_rsa), + auth_methods: %w(publickey), + # use ssh proxy if API servers are on a private network + proxy: Net::SSH::Proxy::Command.new("ssh #{JUMPBOX_PROXY} -W %h:%p") +} + +# private git repo for configuraiton +PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : 'https://your_github_pat_token@github.com/your_organization/ontoportal-configs.git' +desc "Check if agent forwarding is working" +task :forwarding do + on roles(:all) do |h| + if test("env | grep SSH_AUTH_SOCK") + info "Agent forwarding is up to #{h}" + else + error "Agent forwarding is NOT up to #{h}" + end + end +end + +# Smoke test for checking if the service is up +desc 'Smoke test: Check if ncbo_cron service is running' +task :smoke_test do + on roles(:app), in: :sequence, wait: 5 do + # Check if the service is running using systemctl + result = `systemctl is-active ncbo_cron` + if result.strip == 'active' + info "ncbo_cron service is up and running!" + else + error "ncbo_cron service failed to start." + end + end +end + +namespace :deploy do + + desc 'Incorporate the private repository content' + # Get cofiguration from repo if PRIVATE_CONFIG_REPO env var is set + # or get config from local directory if LOCAL_CONFIG_PATH env var is set + task :get_config do + if defined?(PRIVATE_CONFIG_REPO) + TMP_CONFIG_PATH = "/tmp/#{SecureRandom.hex(15)}".freeze + on roles(:app) do + execute "git clone -q #{PRIVATE_CONFIG_REPO} #{TMP_CONFIG_PATH}" + execute "rsync -av #{TMP_CONFIG_PATH}/#{fetch(:config_folder_path)}/ #{release_path}/" + execute "rm -rf #{TMP_CONFIG_PATH}" + end + elsif defined?(LOCAL_CONFIG_PATH) + on roles(:app) do + execute "rsync -av #{LOCAL_CONFIG_PATH}/#{fetch(:application)}/ #{release_path}/" + end + end + end + + desc 'Restart application' + task :restart do + on roles(:app), in: :sequence, wait: 5 do + # Your restart mechanism here, for example: + # execute :touch, release_path.join('tmp/restart.txt') + execute 'sudo systemctl restart ncbo_cron' + execute 'sleep 5' + end + end + + after :updating, :get_config + after :publishing, :restart + after :restart, :smoke_test + +end diff --git a/config/deploy/agroportal.rb b/config/deploy/agroportal.rb new file mode 100644 index 00000000..0ce7bb55 --- /dev/null +++ b/config/deploy/agroportal.rb @@ -0,0 +1,16 @@ +# Simple Role Syntax +# ================== +# Supports bulk-adding hosts to roles, the primary +# server in each group is considered to be the first +# unless any hosts have the primary property set. +# Don't declare `role :all`, it's a meta role +role :app, %w[agroportal.lirmm.fr] +set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'master' +# Extended Server Syntax +# ====================== +# This can be used to drop a more detailed server +# definition into the server list. The second argument +# something that quacks like a hash can be used to set +# extended properties on the server. +# server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value +set :log_level, :error diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb new file mode 100644 index 00000000..8e2db320 --- /dev/null +++ b/config/deploy/staging.rb @@ -0,0 +1,16 @@ +# Simple Role Syntax +# ================== +# Supports bulk-adding hosts to roles, the primary +# server in each group is considered to be the first +# unless any hosts have the primary property set. +# Don't declare `role :all`, it's a meta role +role :app, %w{stageportal.lirmm.fr} +set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'stage' +# Extended Server Syntax +# ====================== +# This can be used to drop a more detailed server +# definition into the server list. The second argument +# something that quacks like a hash can be used to set +# extended properties on the server. +#server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value +set :log_level, :error \ No newline at end of file diff --git a/config/deploy/test.rb b/config/deploy/test.rb new file mode 100644 index 00000000..8fc790cb --- /dev/null +++ b/config/deploy/test.rb @@ -0,0 +1,16 @@ +# Simple Role Syntax +# ================== +# Supports bulk-adding hosts to roles, the primary +# server in each group is considered to be the first +# unless any hosts have the primary property set. +# Don't declare `role :all`, it's a meta role +role :app, %w{testportal.lirmm.fr} +set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'development' +# Extended Server Syntax +# ====================== +# This can be used to drop a more detailed server +# definition into the server list. The second argument +# something that quacks like a hash can be used to set +# extended properties on the server. +#server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value +set :log_level, :error \ No newline at end of file