Merge pull request #31 from ontoportal-lirmm/development

Merge to master: Release 3.0.0 - Triples stores migration tools and Ruby 3.2 support

.github/workflows/deploy.yml

@@ -0,0 +1,97 @@
+# Workflow to deploy NCBO Cron to stage/prod systems
+#
+# Required github secrets:
+#
+# CONFIG_REPO - github repo containing config and customizations for NCBO_CRON. Format 'author/private_config_repo'
+# it is used for getting capistrano deployment configuration for stages on the github actions runner and
+# PRIVATE_CONFIG_REPO env var is constructed from it which is used by capistrano on the API hosts for pulling configs.
+#
+# GH_PAT - github Personal Access Token for accessing private config repo
+#
+# SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if API nodes live on private network.
+# SSH_JUMPHOST_USER - username  to use to connect to the ssh jump/proxy.
+#
+# DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/
+# this SSH key is used for accessing jump host, API nodes, and private github repo.
+
+name: Capistrano Deployment
+# Controls when the action will run.
+on:
+  push:
+    branches:
+      - stage
+      - development
+  # Allows running this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      BRANCH:
+        description: "Branch/tag to deploy"
+        type: choice
+        options:
+          - stage
+          - development
+          - master
+        default: stage
+        required: true
+      environment:
+        description: "target environment to deploy to"
+        type: choice
+        options:
+          - staging
+          - test
+          - agroportal
+        default: stage
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_WITHOUT: default #install gems required primarely for deployment in order to speed up workflow
+      PRIVATE_CONFIG_REPO: ${{ format('[email protected]:{0}.git', secrets.CONFIG_REPO) }}
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      - name: set branch/tag and environment to deploy from inputs
+        run: |
+          # workflow_dispatch default input doesn't get set on push so we need to set defaults
+          # via shell parameter expansion       
+        
+          USER_INPUT_BRANCH="${{ inputs.branch || github.head_ref || 'master' }}"
+          echo "BRANCH=${USER_INPUT_BRANCH}" >> $GITHUB_ENV
+          
+          USER_INPUT_ENVIRONMENT=${{ inputs.environment }}
+          echo "TARGET=${USER_INPUT_ENVIRONMENT:-test}" >> $GITHUB_ENV
+
+          CONFIG_REPO=${{ secrets.CONFIG_REPO }}
+          GH_PAT=${{ secrets.GH_PAT }}
+          echo "PRIVATE_CONFIG_REPO=https://${GH_PAT}@github.com/${CONFIG_REPO}" >> $GITHUB_ENV
+
+          echo "SSH_JUMPHOST=${{ secrets.SSH_JUMPHOST }}" >> $GITHUB_ENV
+          echo "SSH_JUMPHOST_USER=${{ secrets.SSH_JUMPHOST_USER }}" >> $GITHUB_ENV
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.1.0 # Not needed with a .ruby-version file
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Install dependencies (explicit)
+        run: bundle install
+      - name: Check installed gems
+        run: bundle list
+      - name: get-deployment-config
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ secrets.CONFIG_REPO }} # repository containing deployment settings
+          token: ${{ secrets.GH_PAT }} # `GH_PAT` is a secret that contains your PAT
+          path: deploy_config
+      - name: copy-deployment-config
+        run: cp -r deploy_config/ncbo_cron/${{ inputs.environment }}/* .
+        # add ssh hostkey so that capistrano doesn't complain
+      - name: Add jumphost's hostkey to Known Hosts
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.SSH_JUMPHOST }}"
+          ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts
+        shell: bash
+      - uses: miloserdow/capistrano-deploy@master
+        with:
+          target: ${{ env.TARGET }} # which environment to deploy
+          deploy_key: ${{ secrets.DEPLOY_ENC_KEY }} # Name of the variable configured in Settings/Secrets of your github project

.github/workflows/ruby-unit-tests.yml

@@ -9,8 +9,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        goo-slice: [ '20', '100', '500' ]
-        ruby-version: [ '2.7' ]
+        goo-slice: [ '100' ]
+        ruby-version: [ '3.2.0' ]
         triplestore: [ 'fs', 'ag', 'vo', 'gb' ]
     runs-on: ubuntu-latest
     steps:
@@ -34,7 +34,7 @@ jobs:
         # http://docs.codecov.io/docs/testing-with-docker
         run: |
           ci_env=`bash <(curl -s https://codecov.io/env)`
-          GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v"
+          GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }}
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
         with:

.gitignore

@@ -15,6 +15,8 @@ repo*
 .DS_Store
 tmp
 
+vendor/bundle/*
+
 # Code coverage reports
 coverage*
 
@@ -29,3 +31,9 @@ logs/
 
 # Ignore jEnv files
 .java-version
+
+processed_files/
+
+queries.txt
+
+graph_comparison.csv

Capfile

@@ -0,0 +1,28 @@
+# Load DSL and Setup Up Stages
+require 'capistrano/setup'
+
+# Includes default deployment tasks
+require 'capistrano/deploy'
+require "capistrano/scm/git"
+install_plugin Capistrano::SCM::Git
+# Includes tasks from other gems included in your Gemfile
+#
+# For documentation on these, see for example:
+#
+#   https://github.com/capistrano/rvm
+#   https://github.com/capistrano/rbenv
+#   https://github.com/capistrano/chruby
+#   https://github.com/capistrano/bundler
+#   https://github.com/capistrano/rails
+#
+# require 'capistrano/rvm'
+# require 'capistrano/rbenv'
+# require 'capistrano/chruby'
+require 'capistrano/bundler'
+# require 'capistrano/rails/assets'
+# require 'capistrano/rails/migrations'
+require 'capistrano/locally'
+# require 'new_relic/recipes' # announce deployments in NewRelic
+# Loads custom tasks from `lib/capistrano/tasks' if you have any defined.
+Dir.glob('lib/capistrano/tasks/*.cap').each { |r| import r }
+

Gemfile

@@ -22,22 +22,41 @@ gem 'sys-proctable'
 gem 'request_store'
 gem 'parallel'
 gem 'json-ld'
-gem 'ffi', '~> 1.16.3'
+gem 'ffi'
+gem 'activesupport', '~> 5.0'
+gem 'rackup'
 
-# Monitoring
-gem 'cube-ruby', require: 'cube'
 
-gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development'
+gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'feature/migrate-ruby-3.2'
+gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'feature/migrate-ruby-3.2'
 gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development'
-gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'development'
 gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'development'
+
 # Testing
 group :test do
   gem 'email_spec'
-  gem 'minitest', '< 5.0'
+  gem 'minitest'
   gem 'simplecov'
   gem 'simplecov-cobertura' # for codecov.io
-  gem 'test-unit-minitest'
+  # gem 'test-unit-minitest'
+  gem 'crack', '0.4.5'
+  gem 'webmock'
+  gem "minitest-hooks", "~> 1.5"
+  gem 'webrick'
+end
+
+group :development do
+  # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42
+  gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false
+  gem 'capistrano', '~> 3', require: false
+  gem 'capistrano-bundler', require: false
+  gem 'capistrano-locally', require: false
+  gem 'capistrano-rbenv', require: false
+  gem 'ed25519', '>= 1.2', '< 2.0', require: false
 end
 
+gem 'cube-ruby'
 gem "binding_of_caller", "~> 1.0"
+gem 'concurrent-ruby', '1.3.4'
+gem 'net-smtp'
+gem 'net-ftp'