CI/CD - move to github actions (#31)

- Moved to github actions for CI/CD
- Added end-to-end testing

.github/workflows/build.yml

@@ -0,0 +1,100 @@
+name: build
+
+on:
+  push:
+    # Sequence of patterns matched against refs/tags
+    branches:
+      - '*'
+
+jobs:
+  build:
+    name: Build gem
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        logstash-version: ['6.7', '6.8', '7.0', '7.x', 'default' ]
+    env:
+      LOGSTASH_SOURCE: 1
+      LOGSTASH_PATH: ..//logstash
+    steps:
+      - if: matrix.logstash-version != 'default'
+        name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: jruby
+          bundler-cache: true
+      - if: matrix.logstash-version != 'default'
+        name: Checkout logstash ${{ matrix.logstash-version }}
+        uses: actions/checkout@v2
+        with:
+          repository: "elastic/logstash"
+          ref: ${{ matrix.logstash-version }}
+          path: logstash
+      - if: matrix.logstash-version != 'default'
+        name: Build logstash
+        run: ./gradlew assemble
+        working-directory: logstash
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          path: 'kusto'
+      - run: bundle install
+        working-directory: 'kusto'
+      - name: Test
+        run: bundle exec rake spec_junit
+        working-directory: 'kusto'
+      - run: gem build *.gemspec
+        working-directory: 'kusto'
+      - run: mv *.gem logstash-kusto.gem
+        working-directory: 'kusto'
+      - if: matrix.logstash-version == 'default'
+        name: Upload gem
+        uses: actions/upload-artifact@v2
+        with:
+          name: logstash-kusto.gem
+          path: 'kusto/logstash-kusto.gem'
+      - if: matrix.logstash-version == 'default'
+        name: Publish Unit Test Results
+        uses: EnricoMi/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          files: kusto/rspec.xml
+  e2e:
+    name: End-To-End Testing
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: jruby
+          bundler-cache: true
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Download gem
+        uses: actions/download-artifact@v2
+        with:
+          name: logstash-kusto.gem
+      - name: Install logstash # taken from logstash's website https://www.elastic.co/guide/en/logstash/7.10/installing-logstash.html#_apt
+        run: |
+          wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+          sudo apt-get install apt-transport-https
+          echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
+          sudo apt-get update && sudo apt-get install logstash
+      - name: Install plugin
+        run: sudo /usr/share/logstash/bin/logstash-plugin install logstash-kusto.gem
+      - run: sudo env "PATH=$PATH" bundle install
+      - run: sudo chmod -R 777 /usr/share/logstash
+      - run: sudo chmod -R 777 .
+      - name: Run e2e
+        run: ruby e2e.rb
+        working-directory: 'e2e'
+        env:
+          ENGINE_URL: ${{ secrets.ENGINE_URL }}
+          INGEST_URL: ${{ secrets.INGEST_URL }}
+          APP_ID: ${{ secrets.APP_ID }}
+          APP_KEY: ${{ secrets.APP_KEY }}
+          TENANT_ID: ${{ secrets.TENANT_ID }}
+          TEST_DATABASE: ${{ secrets.TEST_DATABASE }}

.github/workflows/deploy.yml

@@ -0,0 +1,43 @@
+on:
+  release:
+    types: [created]
+
+name: Deploy and Create Release
+
+jobs:
+  build:
+    name: Upload Release Asset
+    runs-on: ubuntu-latest
+    steps:
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: jruby
+          bundler-cache: true
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - run: bundle install
+      - name: Get release
+        id: get_release
+        uses: bruceadams/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - name: Publish gem
+        uses: dawidd6/action-publish-gem@v1
+        with:
+          api_key: ${{secrets.RUBYGEMS_KEY}}
+          github_token: ${{secrets.GITHUB_TOKEN}}
+      - name: Set artifact name
+        id: set_artifact_name
+        run: |
+          ARTIFACT_PATH=$(find . -maxdepth 1 -iname '*.gem')
+          echo "::set-output name=artifact_name::$ARTIFACT_PATH"
+      - name: Upload Release Asset
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.get_release.outputs.upload_url }}
+          asset_path: ./${{ steps.set_artifact_name.outputs.artifact_name }}
+          asset_name: ${{ steps.set_artifact_name.outputs.artifact_name }}
+          asset_content_type: application/zip

.gitignore

@@ -30,4 +30,12 @@ Gemfile.lock
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
-!.vscode/extensions.json
+!.vscode/extensions.json
+
+#IDEA
+.idea
+.idea/*
+
+#Jar files and generated files
+**/*.jar
+**/*_jars.rb

Gemfile

@@ -1,5 +1,3 @@
-gem 'ruby-maven', '~> 3.3.11'
-
 source 'https://rubygems.org'
 gemspec
 
@@ -8,7 +6,15 @@ gemspec
 logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
 use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
 
+puts "cwd: #{Dir.getwd} ,use_logstash_source: #{use_logstash_source}, logstash_path: #{logstash_path}, exists: #{Dir.exist?(logstash_path)}"
+
 if Dir.exist?(logstash_path) && use_logstash_source
+  puts "Using local logstash"
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+else
+  puts "using default logstash"
 end
+
+gem "rspec"
+gem "rspec_junit_formatter"

README.md

@@ -1,7 +1,7 @@
 # Logstash Output Plugin for Azure Data Explorer (Kusto)
 
-[![Build Status](https://img.shields.io/travis/azure/logstash-output-kusto/master.svg?label=master)](https://travis-ci.org/Azure/logstash-output-kusto) 
-[![Build Status](https://img.shields.io/travis/azure/logstash-output-kusto/dev.svg?label=dev)](https://travis-ci.org/Azure/logstash-output-kusto)
+![build](https://github.com/Azure/logstash-output-kusto/workflows/build/badge.svg)
+![build](https://github.com/Azure/logstash-output-kusto/workflows/build/badge.svg?branch=master)
 [![Gem](https://img.shields.io/gem/v/logstash-output-kusto.svg)](https://rubygems.org/gems/logstash-output-kusto)
 [![Gem](https://img.shields.io/gem/dt/logstash-output-kusto.svg)](https://rubygems.org/gems/logstash-output-kusto)
 

Rakefile

@@ -4,4 +4,14 @@ task :default do
   system("rake -T")
 end
 
-require "logstash/devutils/rake"
+require "logstash/devutils/rake"
+
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError
+end
+
+RSpec::Core::RakeTask.new(:spec_junit) do |t|
+  t.rspec_opts = '--format RspecJunitFormatter --out rspec.xml'
+end

e2e/dataset.csv

@@ -0,0 +1,10 @@
+0,00000000-0000-0000-0001-020304050607,0.0,0.0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null
+1,00000001-0000-0000-0001-020304050607,1.0001,1.01,1,1,1,1,1,1,1,1,2015-01-01T01:01:01.0000000Z,One,"One",1,00:00:01.0010001,,"{""arr"":[0,1],""rowId"":1}"
+2,00000002-0000-0000-0001-020304050607,2.0002,2.02,0,2,2,2,2,2,2,2,2016-01-01T01:01:01.0000000Z,Two,"Two",2,-00:00:02.0020002,,"{""arr"":[0,2],""rowId"":2}"
+3,00000003-0000-0000-0001-020304050607,3.0003,3.03,1,3,3,3,3,3,3,3,2017-01-01T01:01:01.0000000Z,Three,"Three",3,00:00:03.0030003,,"{""arr"":[0,3],""rowId"":3}"
+4,00000004-0000-0000-0001-020304050607,4.0004,4.04,0,4,4,4,4,4,4,4,2018-01-01T01:01:01.0000000Z,Four,"Four",4,-00:00:04.0040004,,"{""arr"":[0,4],""rowId"":4}"
+5,00000005-0000-0000-0001-020304050607,5.0005,5.05,1,5,5,5,5,5,5,5,2019-01-01T01:01:01.0000000Z,Five,"Five",5,00:00:05.0050005,,"{""arr"":[0,5],""rowId"":5}"
+6,00000006-0000-0000-0001-020304050607,6.0006,6.06,0,6,6,6,6,6,6,6,2020-01-01T01:01:01.0000000Z,Six,"Six",6,-00:00:06.0060006,,"{""arr"":[0,6],""rowId"":6}"
+7,00000007-0000-0000-0001-020304050607,7.0007,7.07,1,7,7,7,7,7,7,7,2021-01-01T01:01:01.0000000Z,Seven,"Seven",7,00:00:07.0070007,,"{""arr"":[0,7],""rowId"":7}"
+8,00000008-0000-0000-0001-020304050607,8.0008,8.08,0,8,8,8,8,8,8,8,2022-01-01T01:01:01.0000000Z,Eight,"Eight",8,-00:00:08.0080008,,"{""arr"":[0,8],""rowId"":8}"
+9,00000009-0000-0000-0001-020304050607,9.0009,9.09,1,9,9,9,9,9,9,9,2023-01-01T01:01:01.0000000Z,Nine,"Nine",9,00:00:09.0090009,,"{""arr"":[0,9],""rowId"":9}"

e2e/dataset_mapping.json

@@ -0,0 +1 @@
+[{"Properties":{"Path":"$.rownumber"},"column":"rownumber","datatype":"int"},{"Properties":{"Path":"$.rowguid"},"column":"rowguid","datatype":"string"},{"Properties":{"Path":"$.xdouble"},"column":"xdouble","datatype":"real"},{"Properties":{"Path":"$.xfloat"},"column":"xfloat","datatype":"real"},{"Properties":{"Path":"$.xbool"},"column":"xbool","datatype":"bool"},{"Properties":{"Path":"$.xint16"},"column":"xint16","datatype":"int"},{"Properties":{"Path":"$.xint32"},"column":"xint32","datatype":"int"},{"Properties":{"Path":"$.xint64"},"column":"xint64","datatype":"long"},{"Properties":{"Path":"$.xuint8"},"column":"xuint8","datatype":"long"},{"Properties":{"Path":"$.xuint16"},"column":"xuint16","datatype":"long"},{"Properties":{"Path":"$.xuint32"},"column":"xuint32","datatype":"long"},{"Properties":{"Path":"$.xuint64"},"column":"xuint64","datatype":"long"},{"Properties":{"Path":"$.xdate"},"column":"xdate","datatype":"datetime"},{"Properties":{"Path":"$.xsmalltext"},"column":"xsmalltext","datatype":"string"},{"Properties":{"Path":"$.xtext"},"column":"xtext","datatype":"string"},{"Properties":{"Path":"$.rowguid"},"column":"xnumberAsText","datatype":"string"},{"Properties":{"Path":"$.xtime"},"column":"xtime","datatype":"timespan"},{"Properties":{"Path":"$.xtextWithNulls"},"column":"xtextWithNulls","datatype":"string"},{"Properties":{"Path":"$.xdynamicWithNulls"},"column":"xdynamicWithNulls","datatype":"dynamic"}]