package yeti-web with ruby using rbenv

build process now complies neccessary ruby version using ruby-build and rbenv. Ruby version is set in .ruby-version file. Built ruby is used in tests and is included in debian package, which frees project from locking onto interpreter version from distribution repos. changes: * remove git-changelog for changelog generation * remove debbuild and lintian for package build * disable deprecated swagger task in Makefile * optimize dockerfile for build image * unlock bundler version in Makefile * fix bundler config commands in Makefile * package ruby using rbenv * update ruby to 2.7.4 * setup docker image build using github actions and gcr
yeti-switch · Oct 25, 2021 · ea579f7 · ea579f7
1 parent 30d5344
commit ea579f7
Show file tree

Hide file tree

Showing 18 changed files with 206 additions and 131 deletions.
diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml
@@ -0,0 +1,38 @@
+name: Build docker images
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '6 9 * * SUN'
+
+jobs:
+  ci-image:
+    name: Build images for CI
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push app image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ci/buster-build.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/yeti-web:buster-build
+
+      - name: Push db image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ci/pg13.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/yeti-web/pgsql:13
diff --git a/.github/workflows/nightly-packages.yml b/.github/workflows/nightly-packages.yml
@@ -8,10 +8,10 @@ jobs:
   deb10:
     name: Debian 10 nightly package building
     runs-on: ubuntu-latest
-    container: switchyeti/yeti-web:buster-build
+    container: ghcr.io/yeti-switch/yeti-web:buster-build
     services:
       db:
-        image: switchyeti/yeti-web:pg13
+        image: ghcr.io/yeti-switch/yeti-web/pgsql:13
 
     steps:
       - uses: actions/checkout@v2

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -8,10 +8,10 @@ jobs:
   deb10:
     name: Debian 10 release package building
     runs-on: ubuntu-latest
-    container: switchyeti/yeti-web:buster-build
+    container: ghcr.io/yeti-switch/yeti-web:buster-build
     services:
       db:
-        image: switchyeti/yeti-web:pg13
+        image: ghcr.io/yeti-switch/yeti-web/pgsql:13
 
     steps:
       - uses: actions/checkout@v2

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -8,7 +8,7 @@ jobs:
   gems-caching:
     name: GEMs caching
     runs-on: ubuntu-latest
-    container: switchyeti/yeti-web:buster-build
+    container: ghcr.io/yeti-switch/yeti-web:buster-build
     steps:
       - uses: actions/checkout@v2
       - name: Fetching gems cache
@@ -19,7 +19,8 @@ jobs:
             pgq-processors/vendor
             .bundle
             pgq-processors/.bundle
-          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}
+            /opt/yeti-web/vendor/rbenv
+          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}-${{hashFiles('.ruby-version')}}
       - name: install gems
         run: make gems-test
       - name: install gems for pgq
@@ -32,12 +33,13 @@ jobs:
             pgq-processors/vendor
             .bundle
             pgq-processors/.bundle
-          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}
+            /opt/yeti-web/vendor/rbenv
+          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}-${{hashFiles('.ruby-version')}}
 
   lint:
     name: Rubocop and bundle audit
     runs-on: ubuntu-latest
-    container: switchyeti/yeti-web:buster-build
+    container: ghcr.io/yeti-switch/yeti-web:buster-build
     needs: gems-caching
     steps:
       - uses: actions/checkout@v2
@@ -49,17 +51,18 @@ jobs:
             pgq-processors/vendor
             .bundle
             pgq-processors/.bundle
-          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}
+            /opt/yeti-web/vendor/rbenv
+          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}-${{hashFiles('.ruby-version')}}
       - name: Rubocop and bundle audit
         run: make lint
 
   pgq-processors:
     name: PqQ processors tests
     runs-on: ubuntu-latest
-    container: switchyeti/yeti-web:buster-build
+    container: ghcr.io/yeti-switch/yeti-web:buster-build
     services:
       db:
-        image: switchyeti/yeti-web:pg13
+        image: ghcr.io/yeti-switch/yeti-web/pgsql:13
 
     needs: gems-caching
     steps:
@@ -72,7 +75,8 @@ jobs:
             pgq-processors/vendor
             .bundle
             pgq-processors/.bundle
-          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}
+            /opt/yeti-web/vendor/rbenv
+          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}-${{hashFiles('.ruby-version')}}
       - name: Run test-pgq-processors
         run: make test-pgq-processors
         env:
@@ -84,10 +88,10 @@ jobs:
   rspec:
     name: Rspec tests
     runs-on: ubuntu-latest
-    container: switchyeti/yeti-web:buster-build
+    container: ghcr.io/yeti-switch/yeti-web:buster-build
     services:
       db:
-        image: switchyeti/yeti-web:pg13
+        image: ghcr.io/yeti-switch/yeti-web/pgsql:13
 
     needs: gems-caching
     strategy:
@@ -111,7 +115,8 @@ jobs:
             pgq-processors/vendor
             .bundle
             pgq-processors/.bundle
-          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}
+            /opt/yeti-web/vendor/rbenv
+          key: gems-${{runner.os}}-${{hashFiles('Gemfile.lock')}}-${{hashFiles('pgq-processors/Gemfile.lock')}}-${{hashFiles('.ruby-version')}}
 
       - name: Run rspec
         run: make rspec
@@ -143,7 +148,7 @@ jobs:
   finalize-coveralls:
     needs: rspec
     runs-on: ubuntu-latest
-    container: switchyeti/yeti-web:buster-build
+    container: ghcr.io/yeti-switch/yeti-web:buster-build
     steps:
       - name: Coveralls Finished
         uses: coverallsapp/github-action@master

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -209,7 +209,7 @@ GEM
     ast (2.4.0)
     awesome_print (1.8.0)
     bcrypt (3.1.13)
-    bootsnap (1.3.2)
+    bootsnap (1.9.1)
       msgpack (~> 1.0)
     builder (3.2.4)
     bundler-audit (0.8.0)
@@ -363,7 +363,7 @@ GEM
     mini_mime (1.1.1)
     mini_portile2 (2.6.1)
     minitest (5.14.4)
-    msgpack (1.2.6)
+    msgpack (1.4.2)
     multipart-post (2.1.1)
     mustache (1.0.5)
     net-ldap (0.16.1)

diff --git a/Makefile b/Makefile
@@ -18,17 +18,18 @@ app_files :=	bin \
 		Rakefile \
 		vendor \
 		pgq-processors \
-		$(version_file)
+		$(version_file) \
+		vendor/rbenv \
+		.ruby-version
 
 exclude_files :=	config/database.yml \
 			config/yeti_web.yml \
 			*.o \
 			*.a
 
-version = $(shell dpkg-parsechangelog --help | grep -q '\--show-field' \
-	&& dpkg-parsechangelog --show-field version \
-	|| dpkg-parsechangelog | grep Version | awk '{ print $$2; }')
-commit = $(shell git rev-parse HEAD)
+version != git describe --tags --abbrev=0
+debian_version := $(shell echo $(version) | sed 's/_/~/' | sed 's/-master/~master/' | sed 's/-rc/~rc/')-1
+commit != git rev-parse HEAD
 
 debian_host_release != lsb_release -sc
 export DEBFULLNAME ?= YETI team
@@ -38,19 +39,12 @@ bundle_bin := $(gems)/bin/bundle
 bundler_gems := $(CURDIR)/vendor/bundle
 export GEM_PATH := $(gems):$(bundler_gems)
 
-debuild_env :=	http_proxy \
-		https_proxy \
-		SSH_AUTH_SOCK \
-		TRAVIS_* \
-		CI_* \
-		GITLAB_* \
-		YETI_DB_HOST \
-		YETI_DB_PORT \
-		CDR_DB_HOST \
-		CDR_DB_PORT
+# must match final destination in debian package
+export RBENV_ROOT := $(app_dir)/vendor/rbenv
+export PATH := $(RBENV_ROOT)/shims:$(PATH)
+rbenv_version = $(file < .ruby-version)
 
-debuild_flags := $(foreach e,$(debuild_env),-e '$e') $(if $(findstring yes,$(lintian)),--lintian,--no-lintian)
-export no_proxy := 127.0.0.1,localhost
+export no_proxy ?= 127.0.0.1,localhost
 
 pgq_drop_roles :=	DROP ROLE IF EXISTS pgq_reader; \
 			DROP ROLE IF EXISTS pgq_writer; \
@@ -60,18 +54,26 @@ pgq_create_roles :=	CREATE ROLE pgq_reader; \
 			CREATE ROLE pgq_writer; \
 			CREATE ROLE pgq_admin in role pgq_reader,pgq_writer;
 
+export YETI_DB_HOST ?= db
+
 define info
        @printf '\n\e[33m> msg \e[0m\n\n'
 endef
 
+
 ###### Rules ######
 .PHONY: all
-all: docs assets pgq-processors-gems swagger
+all: docs assets pgq-processors-gems
 
 
 debian/changelog:
 	$(info:msg=Generating changelog)
-	changelog-gen -p "$(pkg_name)" -d "$(debian_host_release)" -A "s/_/~/g" "s/-master/~master/" "s/-rc/~rc/"
+	dch \
+		--create \
+		--package "$(pkg_name)" \
+		--newversion "$(debian_version)" \
+		--distribution "$(debian_host_release)" \
+		"Release $(version), commit: $(commit)"
 
 
 version.yml: debian/changelog
@@ -97,29 +99,44 @@ config/policy_roles.yml:
 	cp config/policy_roles.yml.distr config/policy_roles.yml
 
 
+$(RBENV_ROOT)/versions/$(rbenv_version):
+	$(info:msg=Installing ruby $(rbenv_version) into $(RBENV_ROOT))
+	mkdir -pv "$(RBENV_ROOT)"
+	curl -sSL "https://raw.githubusercontent.com/rbenv/ruby-build/master/share/ruby-build/$(rbenv_version)" \
+		> "debian/$(rbenv_version)"
+	rbenv install "debian/$(rbenv_version)"
+	rm "debian/$(rbenv_version)"
+
+
+.PHONY: ruby
+ruby: $(RBENV_ROOT)/versions/$(rbenv_version)
+
+
 .PHONY: bundler
-bundler:
+bundler: ruby
 	$(info:msg=Install bundler)
-	gem install --no-document --install-dir $(gems) bundler -v 2.1.4
+	gem install --no-document --install-dir $(gems) bundler
+	$(bundle_bin) config --local clean 'true'
+	$(bundle_bin) config --local jobs 4
+	$(bundle_bin) config --local deployment 'true'
 
 
 .PHONY: gems
 gems: bundler
 	$(info:msg=Install/Update gems)
-	$(bundle_bin) config --delete with
-	$(bundle_bin) config set without 'development test'
-	$(bundle_bin) config set deployment 'true'
-	$(bundle_bin) install --jobs=4
-	$(bundle_bin) clean
+	$(bundle_bin) config --local --delete with
+	$(bundle_bin) config --local --delete without
+	$(bundle_bin) config --local without 'development test'
+	$(bundle_bin) install
 
 
 .PHONY: gems-test
 gems-test: bundler
 	$(info:msg=Install/Update gems for tests)
-	$(bundle_bin) config set with 'development test'
-	$(bundle_bin) config set deployment 'true'
-	$(bundle_bin) install --jobs=4
-	$(bundle_bin) clean
+	$(bundle_bin) config --local --delete with
+	$(bundle_bin) config --local --delete without
+	$(bundle_bin) config --local with 'development test'
+	$(bundle_bin) install
 	$(bundle_bin) binstubs rspec-core
 
 
@@ -162,8 +179,8 @@ prepare-test-db: gems-test config/database.yml config/yeti_web.yml config/policy
 	@# avoid race condition when createing pgq roles in parallel with
 	@# parallel:spec
 	@# https://github.com/pgq/pgq/blob/master/functions/pgq.upgrade_schema.sql
-	psql -h db -U postgres -c '$(pgq_drop_roles)'
-	psql -h db -U postgres -c '$(pgq_create_roles)'
+	psql -h "$(YETI_DB_HOST)" -U postgres -c '$(pgq_drop_roles)'
+	psql -h "$(YETI_DB_HOST)" -U postgres -c '$(pgq_create_roles)'
 	RAILS_ENV=test $(bundle_bin) exec rake parallel:create
 	RAILS_ENV=test $(bundle_bin) exec rake parallel:load_schema
 	RAILS_ENV=test $(bundle_bin) exec rake parallel:rake[db:seed]
@@ -212,15 +229,16 @@ test-pgq-processors: config/database.yml config/yeti_web.yml config/policy_roles
 	RAILS_ENV=test PGQ_PROCESSORS_TEST=true $(bundle_bin) exec rake db:drop
 
 
+vendor/rbenv:
+	cp -r "$(RBENV_ROOT)" vendor/
+
+
 .PHONY: install
 install: $(app_files)
 	$(info:msg=install app files)
 	@mkdir -p $(DESTDIR)$(app_dir)
 	tar -c --no-auto-compress $(addprefix --exclude , $(exclude_files)) $^ | tar -x -C $(DESTDIR)$(app_dir)
 	@mkdir -v -p $(addprefix $(DESTDIR)$(app_dir)/, log tmp )
-
-	$(info:msg=install swagger specs)
-	@$(MAKE) -C swagger install DESTDIR=$(DESTDIR)$(app_dir)/public version=$(version)
 	@install -v -m0644 -D debian/$(pkg_name).rsyslog $(DESTDIR)/etc/rsyslog.d/$(pkg_name).conf
 	@install -v -m0644 -d $(DESTDIR)/var/log/yeti
 
@@ -252,4 +270,4 @@ clean-all:
 .PHONY: package
 package: debian/changelog
 	$(info:msg=Building package)
-	debuild $(debuild_flags) -uc -us -b
+	dpkg-buildpackage -uc -us -b
diff --git a/bin/bundle.sh b/bin/bundle.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
 export GEM_HOME=vendor/bundler
+export RBENV_ROOT="$(realpath ./vendor/rbenv)"
+export PATH="$RBENV_ROOT/shims:$GEM_HOME/bin:$PATH"
 ./vendor/bundler/bin/bundler $@