From 97888ef4f4139223d8eba0d8f3efbc6572621aee Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Sat, 27 Jan 2024 22:56:29 +0300 Subject: [PATCH 01/22] Run tests on different lisp implementations. --- .github/workflows/ci.yml | 85 +++++++++++++++++++++++++++++++++++ .github/workflows/docs.yml | 20 +++++++++ .github/workflows/release.yml | 16 +++++++ src/jobs/job.lisp | 9 +++- src/steps/step.lisp | 17 +++++++ src/vars.lisp | 5 +-- src/workflow.lisp | 53 +++++++++++++--------- 7 files changed, 180 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4019a3..129e8ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,38 @@ "cache": "true" } }, + { + "name": "Checkout Code", + "uses": "actions/checkout@v4" + }, + { + "name": "Setup Common Lisp Environment", + "uses": "40ants/setup-lisp@v4", + "with": { + "asdf-system": "40ants-ci", + "cache": "false" + } + }, + { + "name": "Change dist to Ultralisp if qlfile does not exist", + "run": "if [[ ! -e qlfile ]]; then echo 'dist ultralisp http://dist.ultralisp.org' > qlfile; fi", + "shell": "bash" + }, + { + "name": "Update Qlot", + "run": "qlot update --no-deps", + "shell": "bash" + }, + { + "name": "Install SBLint wrapper", + "run": "qlot exec ros install 40ants-asdf-system 40ants-linter", + "shell": "bash" + }, + { + "name": "Run Linter", + "run": "qlot exec 40ants-linter --system \"40ants-ci, 40ants-ci-tests\" --imports", + "shell": "bash" + }, { "name": "Change dist to Ultralisp if qlfile does not exist", "run": "if [[ ! -e qlfile ]]; then echo 'dist ultralisp http://dist.ultralisp.org' > qlfile; fi", @@ -76,6 +108,38 @@ "cache": "true" } }, + { + "name": "Checkout Code", + "uses": "actions/checkout@v4" + }, + { + "name": "Setup Common Lisp Environment", + "uses": "40ants/setup-lisp@v4", + "with": { + "asdf-system": "40ants-ci", + "cache": "false" + } + }, + { + "name": "Change dist to Ultralisp", + "run": "echo 'dist ultralisp http://dist.ultralisp.org' > qlfile", + "shell": "bash" + }, + { + "name": "Update Qlot", + "run": "qlot update || qlot update", + "shell": "bash" + }, + { + "name": "Install LISP-CRITIC wrapper", + "run": "qlot exec ros install 40ants-critic", + "shell": "bash" + }, + { + "name": "Run Critic for \"40ants-ci\" system", + "run": "qlot exec lisp-critic --ignore function-too-long,check-prefix 40ants-ci", + "shell": "bash" + }, { "name": "Change dist to Ultralisp", "run": "echo 'dist ultralisp http://dist.ultralisp.org' > qlfile", @@ -136,6 +200,27 @@ "cache": "true" } }, + { + "name": "Checkout Code", + "uses": "actions/checkout@v4" + }, + { + "name": "Setup Common Lisp Environment", + "uses": "40ants/setup-lisp@v4", + "with": { + "asdf-system": "40ants-ci", + "qlfile-template": "{% ifequal quicklisp_dist \"ultralisp\" %}\ndist ultralisp http://dist.ultralisp.org\n{% endifequal %}", + "cache": "false" + } + }, + { + "name": "Run Tests", + "uses": "40ants/run-tests@v2", + "with": { + "asdf-system": "40ants-ci", + "coveralls-token": "\n${{ matrix.lisp == 'sbcl-bin' &&\n matrix.os == 'ubuntu-latest' &&\n matrix.quicklisp == 'ultralisp' &&\n secrets.github_token }}" + } + }, { "name": "Run Tests", "uses": "40ants/run-tests@v2", diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0cd1739..40b80f2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -34,6 +34,26 @@ "cache": "true" } }, + { + "name": "Checkout Code", + "uses": "actions/checkout@v4" + }, + { + "name": "Setup Common Lisp Environment", + "uses": "40ants/setup-lisp@v4", + "with": { + "asdf-system": "40ants-ci", + "cache": "false" + } + }, + { + "name": "Build Docs", + "uses": "40ants/build-docs@v1", + "with": { + "asdf-system": "40ants-ci", + "error-on-warnings": true + } + }, { "name": "Build Docs", "uses": "40ants/build-docs@v1", diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4362396..73b9a8a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,22 @@ "OS": "ubuntu-latest" }, "steps": [ + { + "name": "Checkout Code", + "uses": "actions/checkout@v4" + }, + { + "name": "Create release tag", + "uses": "butlerlogic/action-autotag@8bc1ad456dcdee34e8c6ffbce991cc31793578c2", + "with": { + "root": "ChangeLog.md", + "regex_pattern": "^## (?\\d+\\.\\d+\\.\\d+.*?)( |\\n).*$", + "tag_prefix": "v" + }, + "env": { + "GITHUB_TOKEN": "${{ secrets.GITHUB_TOKEN }}" + } + }, { "name": "Checkout Code", "uses": "actions/checkout@v4" diff --git a/src/jobs/job.lisp b/src/jobs/job.lisp index 972c339..f422e45 100644 --- a/src/jobs/job.lisp +++ b/src/jobs/job.lisp @@ -7,6 +7,8 @@ #:length<) (:import-from #:alexandria #:length=) + (:import-from #:40ants-ci/steps/step + #:ensure-step) (:export #:job #:use-matrix-p #:steps @@ -16,7 +18,7 @@ #:make-env #:permissions #:make-permissions)) -(in-package 40ants-ci/jobs/job) +(in-package #:40ants-ci/jobs/job) (defclass job () @@ -51,7 +53,10 @@ (unless (slot-boundp job 'name) (setf (slot-value job 'name) (string-downcase - (class-name (class-of job)))))) + (class-name (class-of job))) + (slot-value job 'steps) + (mapcar #'ensure-step + (steps job))))) (defmethod os :around ((job job)) (uiop:ensure-list diff --git a/src/steps/step.lisp b/src/steps/step.lisp index 2967f49..d761de1 100644 --- a/src/steps/step.lisp +++ b/src/steps/step.lisp @@ -59,3 +59,20 @@ (when (step-if step) `(("if" . ,(step-if step)))))) + + +(defun ensure-step (step-or-step-definition) + (check-type step-or-step-definition (or step list)) + (etypecase step-or-step-definition + (step step-or-step-definition) + (list + (let ((head (car step-or-step-definition)) + (args (cdr step-or-step-definition))) + + (unless (symbolp head) + (error "~A is not a correct step definition." + step-or-step-definition)) + + (if (fboundp head) + (apply head args) + (apply #'make-instance head args)))))) diff --git a/src/vars.lisp b/src/vars.lisp index 6192066..2548f72 100644 --- a/src/vars.lisp +++ b/src/vars.lisp @@ -11,6 +11,5 @@ "When workflow is generated for ASDF system, this variable will contain a primary ASDF system.") -(defvar *use-cache*) -(setf (documentation '*use-cache* 'variable) - "Workflow will set this variable when preparing the data or YAML generation.") +(defvar *use-cache* nil + "Workflow will set this variable when preparing the data or YAML generation.") diff --git a/src/workflow.lisp b/src/workflow.lisp index 7cc75e5..a614f18 100644 --- a/src/workflow.lisp +++ b/src/workflow.lisp @@ -2,6 +2,7 @@ (:use #:cl) (:import-from #:40ants-ci/github #:*current-system*) + (:import-from #:40ants-ci/jobs/job) (:import-from #:40ants-ci/utils #:ensure-primary-system) (:import-from #:40ants-ci/vars) @@ -105,20 +106,24 @@ on-pull-request cache jobs) - `(progn - (defclass ,name (workflow) - ()) - (let* ((jobs (mapcar #'make-job ',jobs)) - (workflow (make-instance ',name - :name ',name - :jobs jobs - :on-push-to ',(uiop:ensure-list on-push-to) - :by-cron ',(uiop:ensure-list by-cron) - :on-pull-request ,on-pull-request - :cache ,cache))) - (register-workflow workflow) - (on-workflow-redefinition workflow) - workflow))) + (let ((make-func-name (intern (format nil "MAKE-~A-WORKFLOW" + name)))) + `(with-current-system () + (defclass ,name (workflow) + ()) + (defun ,make-func-name () + (let ((jobs (mapcar #'make-job ',jobs))) + (make-instance ',name + :name ',name + :jobs jobs + :on-push-to ',(uiop:ensure-list on-push-to) + :by-cron ',(uiop:ensure-list by-cron) + :on-pull-request ,on-pull-request + :cache ,cache))) + (let* ((workflow (,make-func-name) )) + (register-workflow workflow) + (on-workflow-redefinition workflow) + workflow)))) (defgeneric make-triggers (workflow) @@ -208,12 +213,20 @@ to the primary system with same package name as a package where defworkflow is used.") (:method ((workflow workflow)) - (let* ((system-name (string-downcase - (package-name *package*))) - (*current-system* - (asdf:find-system - (asdf:primary-system-name system-name))) - (system-path (40ants-ci/utils:make-github-workflows-path *current-system*)) + (let* ((system-path (40ants-ci/utils:make-github-workflows-path *current-system*)) (workflow-path (make-workflow-path system-path workflow))) (40ants-ci/github:generate workflow workflow-path)))) + + +(defun call-with-current-system (thunk) + (let* ((system-name (string-downcase + (package-name *package*))) + (*current-system* + (asdf:find-system + (asdf:primary-system-name system-name)))) + (funcall thunk))) + + +(defmacro with-current-system (() &body body) + `(call-with-current-system (lambda () ,@body))) From fc965e10dfdaa9de31de555c1eabddbd6ab0ac06 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 09:01:41 +0300 Subject: [PATCH 02/22] Fix additional steps. --- .github/workflows/ci.yml | 85 ----------------------------------- .github/workflows/docs.yml | 20 --------- .github/workflows/release.yml | 16 ------- src/jobs/job.lisp | 25 +++++++---- 4 files changed, 17 insertions(+), 129 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 129e8ff..d4019a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,38 +34,6 @@ "cache": "true" } }, - { - "name": "Checkout Code", - "uses": "actions/checkout@v4" - }, - { - "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@v4", - "with": { - "asdf-system": "40ants-ci", - "cache": "false" - } - }, - { - "name": "Change dist to Ultralisp if qlfile does not exist", - "run": "if [[ ! -e qlfile ]]; then echo 'dist ultralisp http://dist.ultralisp.org' > qlfile; fi", - "shell": "bash" - }, - { - "name": "Update Qlot", - "run": "qlot update --no-deps", - "shell": "bash" - }, - { - "name": "Install SBLint wrapper", - "run": "qlot exec ros install 40ants-asdf-system 40ants-linter", - "shell": "bash" - }, - { - "name": "Run Linter", - "run": "qlot exec 40ants-linter --system \"40ants-ci, 40ants-ci-tests\" --imports", - "shell": "bash" - }, { "name": "Change dist to Ultralisp if qlfile does not exist", "run": "if [[ ! -e qlfile ]]; then echo 'dist ultralisp http://dist.ultralisp.org' > qlfile; fi", @@ -108,38 +76,6 @@ "cache": "true" } }, - { - "name": "Checkout Code", - "uses": "actions/checkout@v4" - }, - { - "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@v4", - "with": { - "asdf-system": "40ants-ci", - "cache": "false" - } - }, - { - "name": "Change dist to Ultralisp", - "run": "echo 'dist ultralisp http://dist.ultralisp.org' > qlfile", - "shell": "bash" - }, - { - "name": "Update Qlot", - "run": "qlot update || qlot update", - "shell": "bash" - }, - { - "name": "Install LISP-CRITIC wrapper", - "run": "qlot exec ros install 40ants-critic", - "shell": "bash" - }, - { - "name": "Run Critic for \"40ants-ci\" system", - "run": "qlot exec lisp-critic --ignore function-too-long,check-prefix 40ants-ci", - "shell": "bash" - }, { "name": "Change dist to Ultralisp", "run": "echo 'dist ultralisp http://dist.ultralisp.org' > qlfile", @@ -200,27 +136,6 @@ "cache": "true" } }, - { - "name": "Checkout Code", - "uses": "actions/checkout@v4" - }, - { - "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@v4", - "with": { - "asdf-system": "40ants-ci", - "qlfile-template": "{% ifequal quicklisp_dist \"ultralisp\" %}\ndist ultralisp http://dist.ultralisp.org\n{% endifequal %}", - "cache": "false" - } - }, - { - "name": "Run Tests", - "uses": "40ants/run-tests@v2", - "with": { - "asdf-system": "40ants-ci", - "coveralls-token": "\n${{ matrix.lisp == 'sbcl-bin' &&\n matrix.os == 'ubuntu-latest' &&\n matrix.quicklisp == 'ultralisp' &&\n secrets.github_token }}" - } - }, { "name": "Run Tests", "uses": "40ants/run-tests@v2", diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 40b80f2..0cd1739 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -34,26 +34,6 @@ "cache": "true" } }, - { - "name": "Checkout Code", - "uses": "actions/checkout@v4" - }, - { - "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@v4", - "with": { - "asdf-system": "40ants-ci", - "cache": "false" - } - }, - { - "name": "Build Docs", - "uses": "40ants/build-docs@v1", - "with": { - "asdf-system": "40ants-ci", - "error-on-warnings": true - } - }, { "name": "Build Docs", "uses": "40ants/build-docs@v1", diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73b9a8a..4362396 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,22 +17,6 @@ "OS": "ubuntu-latest" }, "steps": [ - { - "name": "Checkout Code", - "uses": "actions/checkout@v4" - }, - { - "name": "Create release tag", - "uses": "butlerlogic/action-autotag@8bc1ad456dcdee34e8c6ffbce991cc31793578c2", - "with": { - "root": "ChangeLog.md", - "regex_pattern": "^## (?\\d+\\.\\d+\\.\\d+.*?)( |\\n).*$", - "tag_prefix": "v" - }, - "env": { - "GITHUB_TOKEN": "${{ secrets.GITHUB_TOKEN }}" - } - }, { "name": "Checkout Code", "uses": "actions/checkout@v4" diff --git a/src/jobs/job.lisp b/src/jobs/job.lisp index f422e45..ef46bf1 100644 --- a/src/jobs/job.lisp +++ b/src/jobs/job.lisp @@ -17,7 +17,8 @@ #:make-matrix #:make-env #:permissions - #:make-permissions)) + #:make-permissions + #:explicit-steps)) (in-package #:40ants-ci/jobs/job) @@ -33,7 +34,8 @@ :documentation "A list of plists denoting matrix combinations to be excluded.") (steps :initform nil :initarg :steps - :reader steps) + :documentation "This slot holds steps given as a STEPS argument to a job constructor. Depending on a job class, it might add additional steps around these explicit steps." + :reader explicit-steps) (permissions :initform nil :initarg :permissions :documentation "A plist of permissions need for running the job. @@ -53,18 +55,25 @@ (unless (slot-boundp job 'name) (setf (slot-value job 'name) (string-downcase - (class-name (class-of job))) - (slot-value job 'steps) + (class-name (class-of job)))) + (setf (slot-value job 'steps) (mapcar #'ensure-step - (steps job))))) + (slot-value job 'steps))))) + (defmethod os :around ((job job)) (uiop:ensure-list (call-next-method))) -(defmethod steps :around ((job job)) - (uiop:ensure-list - (call-next-method))) + +(defgeneric steps (job) + (:method ((job job)) + (explicit-steps job)) + + (:method :around ((job job)) + (uiop:ensure-list + (call-next-method)))) + (defmethod exclude :around ((job job)) (ensure-list-of-plists From ee40e0bb2d96ba0d2e2917b8a87fa6fe625be502 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 09:07:49 +0300 Subject: [PATCH 03/22] Add an example of a custom list of steps. --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ src/ci.lisp | 9 +++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4019a3..12ae979 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,33 @@ } } ] + }, + "lisp-job": { + "runs-on": "ubuntu-latest", + "env": { + "OS": "ubuntu-latest", + "QUICKLISP_DIST": "quicklisp", + "LISP": "ccl-bin" + }, + "steps": [ + { + "name": "Checkout Code", + "uses": "actions/checkout@v4" + }, + { + "name": "Setup Common Lisp Environment", + "uses": "40ants/setup-lisp@v4", + "with": { + "asdf-system": "40ants-ci", + "cache": "true" + } + }, + { + "name": "Show Roswell Config", + "run": "ros config", + "shell": "bash" + } + ] } } } \ No newline at end of file diff --git a/src/ci.lisp b/src/ci.lisp index d710e7b..90c4457 100644 --- a/src/ci.lisp +++ b/src/ci.lisp @@ -8,7 +8,9 @@ #:build-docs) (:import-from #:40ants-ci/workflow #:defworkflow) - (:import-from #:40ants-ci/jobs/autotag)) + (:import-from #:40ants-ci/jobs/autotag) + (:import-from #:40ants-ci/jobs/lisp-job + #:lisp-job)) (in-package 40ants-ci/ci) @@ -47,6 +49,9 @@ :coverage t :qlfile "{% ifequal quicklisp_dist \"ultralisp\" %} dist ultralisp http://dist.ultralisp.org - {% endifequal %}"))) + {% endifequal %}") + (40ants-ci/jobs/lisp-job:lisp-job :lisp "ccl-bin" + :steps ((40ants-ci/steps/sh:sh "Show Roswell Config" + "ros config"))))) From 2550776c8d82c861583537d1c35761a80e960c8e Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 09:08:03 +0300 Subject: [PATCH 04/22] Fix critique. --- src/workflow.lisp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workflow.lisp b/src/workflow.lisp index a614f18..587e69a 100644 --- a/src/workflow.lisp +++ b/src/workflow.lisp @@ -120,7 +120,7 @@ :by-cron ',(uiop:ensure-list by-cron) :on-pull-request ,on-pull-request :cache ,cache))) - (let* ((workflow (,make-func-name) )) + (let ((workflow (,make-func-name) )) (register-workflow workflow) (on-workflow-redefinition workflow) workflow)))) From 925dea2e6d112976891bddab951e5a97e240d412 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 09:17:42 +0300 Subject: [PATCH 05/22] Renamed custom job. --- .github/workflows/ci.yml | 2 +- src/ci.lisp | 3 ++- src/jobs/job.lisp | 10 ++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12ae979..8738ff4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,7 +146,7 @@ } ] }, - "lisp-job": { + "check-ros-config": { "runs-on": "ubuntu-latest", "env": { "OS": "ubuntu-latest", diff --git a/src/ci.lisp b/src/ci.lisp index 90c4457..649288d 100644 --- a/src/ci.lisp +++ b/src/ci.lisp @@ -50,7 +50,8 @@ :qlfile "{% ifequal quicklisp_dist \"ultralisp\" %} dist ultralisp http://dist.ultralisp.org {% endifequal %}") - (40ants-ci/jobs/lisp-job:lisp-job :lisp "ccl-bin" + (40ants-ci/jobs/lisp-job:lisp-job :name "check-ros-config" + :lisp "ccl-bin" :steps ((40ants-ci/steps/sh:sh "Show Roswell Config" "ros config"))))) diff --git a/src/jobs/job.lisp b/src/jobs/job.lisp index ef46bf1..cf07472 100644 --- a/src/jobs/job.lisp +++ b/src/jobs/job.lisp @@ -52,13 +52,15 @@ (defmethod initialize-instance :after ((job job) &rest initargs) (declare (ignore initargs)) + (unless (slot-boundp job 'name) (setf (slot-value job 'name) (string-downcase - (class-name (class-of job)))) - (setf (slot-value job 'steps) - (mapcar #'ensure-step - (slot-value job 'steps))))) + (class-name (class-of job))))) + + (setf (slot-value job 'steps) + (mapcar #'ensure-step + (slot-value job 'steps)))) (defmethod os :around ((job job)) From b22f64fb496f11380ea65e774fcc21af631c6237 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 09:18:41 +0300 Subject: [PATCH 06/22] Added docs on job name. --- src/jobs/job.lisp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/jobs/job.lisp b/src/jobs/job.lisp index cf07472..3642c8c 100644 --- a/src/jobs/job.lisp +++ b/src/jobs/job.lisp @@ -24,7 +24,8 @@ (defclass job () ((name :initarg :name - :reader name) + :reader name + :documentation "If this name was not given in constructor, then name will be lowercased name of the job class.") (os :initform "ubuntu-latest" :initarg :os :reader os) From c85fb435349a8b78c9408575d723f5cce1e4ffff Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 09:39:39 +0300 Subject: [PATCH 07/22] Update deps --- qlfile.lock | 2 +- src/ci.lisp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/qlfile.lock b/qlfile.lock index 982082b..ae996ea 100644 --- a/qlfile.lock +++ b/qlfile.lock @@ -5,7 +5,7 @@ ("ultralisp" . (:class qlot/source/dist:source-dist :initargs (:distribution "http://dist.ultralisp.org" :%version :latest) - :version "20231202040501")) + :version "20240227053500")) ("bordeaux-threads" . (:class qlot/source/github:source-github :initargs (:repos "svetlyak40wt/bordeaux-threads" :ref nil :branch "fix-apiv2-for-no-threads" :tag nil) diff --git a/src/ci.lisp b/src/ci.lisp index 649288d..54fc29b 100644 --- a/src/ci.lisp +++ b/src/ci.lisp @@ -50,6 +50,8 @@ :qlfile "{% ifequal quicklisp_dist \"ultralisp\" %} dist ultralisp http://dist.ultralisp.org {% endifequal %}") + ;; This is an example of a job with a custom + ;; step: (40ants-ci/jobs/lisp-job:lisp-job :name "check-ros-config" :lisp "ccl-bin" :steps ((40ants-ci/steps/sh:sh "Show Roswell Config" From 0f9fc8663840997da8aca8fccefa3617c6ccd28e Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 10:20:25 +0300 Subject: [PATCH 08/22] Add example of a job with a custom step + switched to autoapi. --- src/core.lisp | 78 +++++++++++++++++++++++------------------ src/jobs/autotag.lisp | 6 +++- src/jobs/critic.lisp | 4 ++- src/jobs/docs.lisp | 3 +- src/jobs/job.lisp | 3 +- src/jobs/linter.lisp | 4 ++- src/jobs/lisp-job.lisp | 10 ++++-- src/jobs/run-tests.lisp | 7 ++-- src/steps/action.lisp | 6 ++-- src/steps/sh.lisp | 9 ++--- src/steps/step.lisp | 4 ++- src/utils.lisp | 8 ++--- 12 files changed, 84 insertions(+), 58 deletions(-) diff --git a/src/core.lisp b/src/core.lisp index 6fedc82..13b25d0 100644 --- a/src/core.lisp +++ b/src/core.lisp @@ -37,10 +37,12 @@ "GIT" "CL" "CI" + "JSON" "OS" "SBL" "BSD" "TODO" + "ASD" "ASDF" "EXAMPLE")) " @@ -58,7 +60,8 @@ actions and [`SBLint`](https://github.com/cxxxr/sblint) to check code for compil (40ants-ci system) (@reasons section) (@quickstart section) - (@details section)) + (@details section) + (@api section)) (defsection-copy @readme @index) @@ -529,42 +532,49 @@ modified .github/workflows/docs.yml (defsection @details (:title "Details" - :ignore-words ("ASD")) + :ignore-words ("ASDF" + "CCL-BIN")) " TODO: I have to write a few chapters with details on additional job's parameters and a way how to create new job types. -" - (generate function) - - (40ants-ci/jobs/run-tests:run-tests function) - (40ants-ci/jobs/run-tests:run-tests class) - - (40ants-ci/jobs/docs:build-docs function) - (40ants-ci/jobs/docs:build-docs class) - - (40ants-ci/jobs/job:job class) - (40ants-ci/jobs/job:name (reader 40ants-ci/jobs/job:job)) - (40ants-ci/jobs/job:os (reader 40ants-ci/jobs/job:job)) - (40ants-ci/jobs/job:steps (reader 40ants-ci/jobs/job:job)) - (40ants-ci/jobs/job:permissions (reader 40ants-ci/jobs/job:job)) - (40ants-ci/jobs/job:make-env generic-function) - (40ants-ci/jobs/job:use-matrix-p generic-function) - (40ants-ci/jobs/job:make-matrix generic-function) - (40ants-ci/jobs/job:make-permissions generic-function) - - (40ants-ci/jobs/lisp-job:lisp-job class) - (40ants-ci/jobs/lisp-job:lisp (reader 40ants-ci/jobs/lisp-job:lisp-job)) - (40ants-ci/jobs/lisp-job:asdf-system (reader 40ants-ci/jobs/lisp-job:lisp-job)) - (40ants-ci/jobs/lisp-job:quicklisp (reader 40ants-ci/jobs/lisp-job:lisp-job)) - - (40ants-ci/jobs/linter:linter class) - (40ants-ci/jobs/linter:linter function) - - (40ants-ci/jobs/critic:critic class) - (40ants-ci/jobs/critic:critic function) - - (40ants-ci/jobs/autotag:autotag class) - (40ants-ci/jobs/autotag:autotag function)) + +But for now, I want to show a small example, how to define a workflow with a +job which takes care about lisp installation and then calls a custom step: + + +```lisp +(defworkflow ci + :on-push-to \"master\" + :by-cron \"0 10 * * 1\" + :on-pull-request t + :cache t + :jobs ((40ants-ci/jobs/lisp-job:lisp-job :name \"check-ros-config\" + :lisp \"ccl-bin\" + :steps ((40ants-ci/steps/sh:sh \"Show Roswell Config\" + \"ros config\"))))) +``` + +Here we are using the class 40ANTS-CI/JOBS/LISP-JOB:LISP-JOB which is base for most classes in this ASDF system +and pass a custom 40ANTS-CI/STEPS/SH:SH step to it. This step will be called after the repostory checkout and CCL-BIN lisp installation. +so, thus when this step will run `ros config` command, it will output something like that: + +``` +asdf.version=3.3.5.3 +ccl-bin.version=1.12.2 +setup.time=3918000017 +sbcl-bin.version=2.4.1 +default.lisp=ccl-bin + +Possible subcommands: +set +show +``` + +Pay attention to the NAME argument of 40ANTS-CI/JOBS/LISP-JOB:LISP-JOB class. If you omit it, then default \"lisp-job\" name will be used. +") + + +(40ants-doc/autodoc:defautodoc @api (:system "40ants-ci")) (defun generate (system &key path) diff --git a/src/jobs/autotag.lisp b/src/jobs/autotag.lisp index 9e4a576..1652d55 100644 --- a/src/jobs/autotag.lisp +++ b/src/jobs/autotag.lisp @@ -3,7 +3,11 @@ (:import-from #:40ants-ci/jobs/job) (:import-from #:40ants-ci/steps/action #:action) - (:export #:autotag)) + (:export #:autotag + #:filename + #:regex + #:tag-prefix + #:token-pattern)) (in-package 40ants-ci/jobs/autotag) (defparameter *default-filename* "ChangeLog.md") diff --git a/src/jobs/critic.lisp b/src/jobs/critic.lisp index 32942cf..091d75f 100644 --- a/src/jobs/critic.lisp +++ b/src/jobs/critic.lisp @@ -5,7 +5,9 @@ (:import-from #:40ants-ci/jobs/job) (:import-from #:40ants-ci/jobs/lisp-job #:asdf-system) - (:export #:critic)) + (:export #:critic + #:asdf-systems + #:ignore-critiques)) (in-package 40ants-ci/jobs/critic) diff --git a/src/jobs/docs.lisp b/src/jobs/docs.lisp index fd26ec3..021ae30 100644 --- a/src/jobs/docs.lisp +++ b/src/jobs/docs.lisp @@ -7,7 +7,8 @@ #:action) (:import-from #:40ants-ci/utils #:current-system-name) - (:export #:build-docs)) + (:export #:build-docs + #:error-on-warnings)) (in-package 40ants-ci/jobs/docs) diff --git a/src/jobs/job.lisp b/src/jobs/job.lisp index 3642c8c..e707f94 100644 --- a/src/jobs/job.lisp +++ b/src/jobs/job.lisp @@ -18,7 +18,8 @@ #:make-env #:permissions #:make-permissions - #:explicit-steps)) + #:explicit-steps + #:exclude)) (in-package #:40ants-ci/jobs/job) diff --git a/src/jobs/linter.lisp b/src/jobs/linter.lisp index 7895665..012bec9 100644 --- a/src/jobs/linter.lisp +++ b/src/jobs/linter.lisp @@ -7,7 +7,9 @@ (:import-from #:40ants-ci/jobs/job) (:import-from #:40ants-ci/utils #:current-system-name) - (:export #:linter)) + (:export #:linter + #:asdf-systems + #:check-imports)) (in-package 40ants-ci/jobs/linter) diff --git a/src/jobs/lisp-job.lisp b/src/jobs/lisp-job.lisp index 393b923..98f7e3c 100644 --- a/src/jobs/lisp-job.lisp +++ b/src/jobs/lisp-job.lisp @@ -17,7 +17,11 @@ (:export #:lisp-job #:asdf-system #:lisp - #:quicklisp)) + #:quicklisp + #:qlfile + #:asdf-version + #:roswell-version + #:qlot-version)) (in-package 40ants-ci/jobs/lisp-job) @@ -43,12 +47,12 @@ (roswell-version :initarg :roswell-version :initform nil :type (or null string) - :documentation "Roswell version to use when setting up Lisp environment. If NIL, then will be used version, pinned in SETUP-LISP github action." + :documentation "Roswell version to use when setting up Lisp environment. If NIL, then will be used version, pinned in `setup-lisp` github action." :reader roswell-version) (qlot-version :initarg :qlot-version :initform nil :type (or null string) - :documentation "Qlot version to use when setting up Lisp environment. If NIL, then will be used version, pinned in SETUP-LISP github action." + :documentation "Qlot version to use when setting up Lisp environment. If NIL, then will be used version, pinned in `setup-lisp` github action." :reader qlot-version)) (:documentation "This job checkouts the sources, installs Roswell and Qlot. Also, it caches results between runs.")) diff --git a/src/jobs/run-tests.lisp b/src/jobs/run-tests.lisp index 996dade..6564692 100644 --- a/src/jobs/run-tests.lisp +++ b/src/jobs/run-tests.lisp @@ -10,9 +10,10 @@ #:current-system-name) (:import-from #:alexandria #:when-let) - (:export - #:run-tests)) -(in-package 40ants-ci/jobs/run-tests) + (:export #:run-tests + #:coverage + #:custom)) +(in-package #:40ants-ci/jobs/run-tests) (defclass run-tests (40ants-ci/jobs/lisp-job:lisp-job) diff --git a/src/steps/action.lisp b/src/steps/action.lisp index ce11210..6c2cd12 100644 --- a/src/steps/action.lisp +++ b/src/steps/action.lisp @@ -4,8 +4,10 @@ (:import-from #:40ants-ci/github) (:import-from #:alexandria #:remove-from-plistf) - (:export #:action)) -(in-package 40ants-ci/steps/action) + (:export #:action + #:uses + #:action-args)) +(in-package #:40ants-ci/steps/action) (defclass action (40ants-ci/steps/step:step) diff --git a/src/steps/sh.lisp b/src/steps/sh.lisp index 6622402..29c297f 100644 --- a/src/steps/sh.lisp +++ b/src/steps/sh.lisp @@ -6,10 +6,11 @@ #:remove-from-plistf) (:import-from #:40ants-ci/utils #:dedent) - (:export - #:sh - #:sections)) -(in-package 40ants-ci/steps/sh) + (:export #:sh + #:sections + #:command + #:shell)) +(in-package #:40ants-ci/steps/sh) (defvar *default-shell* "bash") diff --git a/src/steps/step.lisp b/src/steps/step.lisp index d761de1..ba5074d 100644 --- a/src/steps/step.lisp +++ b/src/steps/step.lisp @@ -7,7 +7,9 @@ #:plistp) (:export #:step #:step-id - #:step-name)) + #:step-name + #:env + #:step-if)) (in-package 40ants-ci/steps/step) diff --git a/src/utils.lisp b/src/utils.lisp index fda34a2..3c93aec 100644 --- a/src/utils.lisp +++ b/src/utils.lisp @@ -43,10 +43,6 @@ CL-USER> (docs-builder/utils:system-packages :docs-builder) # #) ``` - -This function can be used by builder to find pieces of documentation. -For example, DOCS-BUILDER/BUILDERS/MGL-PAX/GUESSER:@INDEX -builder uses it to find documentation sections. ") (:method ((system string)) (system-packages (asdf:find-system system))) @@ -169,9 +165,9 @@ it will output HELLO-WORLD.\" (defun alistp (list) - "Test wheather LIST is a properly formed alist. + "Test wheather LIST argument is a properly formed alist. - In this library, ALIST has always a string as a key. + In this library, alist has always a string as a key. Because we need them to have this form to serialize to JSON propertly. From 577b8da81aff60342def318b2fe72c00e59b2753 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 10:20:56 +0300 Subject: [PATCH 09/22] Fixed import of defautodoc. --- src/core.lisp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core.lisp b/src/core.lisp index 13b25d0..7a54d6c 100644 --- a/src/core.lisp +++ b/src/core.lisp @@ -16,6 +16,8 @@ (:import-from #:40ants-ci/jobs/autotag) (:import-from #:docs-config #:docs-config) + (:import-from #:40ants-doc/autodoc + #:defautodoc) (:export #:generate #:@index #:@readme)) @@ -574,7 +576,7 @@ Pay attention to the NAME argument of 40ANTS-CI/JOBS/LISP-JOB:LISP-JOB class. If ") -(40ants-doc/autodoc:defautodoc @api (:system "40ants-ci")) +(defautodoc @api (:system "40ants-ci")) (defun generate (system &key path) From 38a9c679706144232c3d2e38d14104447ad590bf Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 22:10:23 +0300 Subject: [PATCH 10/22] Turn off cache to check if the problem is in the cached qlot. --- .github/workflows/ci.yml | 8 ++++---- src/ci.lisp | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8738ff4..c663c70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ "uses": "40ants/setup-lisp@v4", "with": { "asdf-system": "40ants-ci", - "cache": "true" + "cache": "false" } }, { @@ -73,7 +73,7 @@ "uses": "40ants/setup-lisp@v4", "with": { "asdf-system": "40ants-ci", - "cache": "true" + "cache": "false" } }, { @@ -133,7 +133,7 @@ "with": { "asdf-system": "40ants-ci", "qlfile-template": "{% ifequal quicklisp_dist \"ultralisp\" %}\ndist ultralisp http://dist.ultralisp.org\n{% endifequal %}", - "cache": "true" + "cache": "false" } }, { @@ -163,7 +163,7 @@ "uses": "40ants/setup-lisp@v4", "with": { "asdf-system": "40ants-ci", - "cache": "true" + "cache": "false" } }, { diff --git a/src/ci.lisp b/src/ci.lisp index 54fc29b..01b75d4 100644 --- a/src/ci.lisp +++ b/src/ci.lisp @@ -23,7 +23,7 @@ :on-push-to "master" :on-pull-request t :by-cron "0 10 * * 1" - :cache t + :cache nil :jobs ((40ants-ci/jobs/docs:build-docs))) @@ -31,7 +31,7 @@ :on-push-to "master" :by-cron "0 10 * * 1" :on-pull-request t - :cache t + :cache nil :jobs ((40ants-ci/jobs/linter:linter :asdf-systems ("40ants-ci" "40ants-ci-tests") From 017e4ac9c8a7ba73466632c238d6dfe3e92e709d Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 22:34:29 +0300 Subject: [PATCH 11/22] Moved docs into a separate ASDF system. --- .gitignore | 4 +- 40ants-ci-docs.asd | 11 + 40ants-ci.asd | 8 +- {src => docs}/changelog.lisp | 4 +- docs/index.lisp | 567 ++++++++++++++++++++++++++++++++++ src/ci.lisp | 3 +- src/core.lisp | 577 +---------------------------------- 7 files changed, 592 insertions(+), 582 deletions(-) create mode 100644 40ants-ci-docs.asd rename {src => docs}/changelog.lisp (97%) create mode 100644 docs/index.lisp diff --git a/.gitignore b/.gitignore index 4f2b437..b7d2acf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -/docs/ +/docs/build/ /env/ /.qlot -/.DS_Store *.fasl +.DS_Store diff --git a/40ants-ci-docs.asd b/40ants-ci-docs.asd new file mode 100644 index 0000000..a51867c --- /dev/null +++ b/40ants-ci-docs.asd @@ -0,0 +1,11 @@ +(defsystem "40ants-ci-docs" + :author "Alexander Artemenko " + :license "Unlicense" + :class :package-inferred-system + :description "Provides documentation for 40ants-ci." + :source-control (:git "https://github.com/40ants/ci") + :bug-tracker "https://github.com/40ants/ci/issues" + :homepage "https://40ants.com/ci/" + :pathname "docs" + :depends-on ("40ants-ci" + "40ants-ci-docs/index")) diff --git a/40ants-ci.asd b/40ants-ci.asd index b12ce70..6f1ebc6 100644 --- a/40ants-ci.asd +++ b/40ants-ci.asd @@ -6,8 +6,12 @@ :defsystem-depends-on ("40ants-asdf-system") :pathname "src" :depends-on ("40ants-ci/core" - "40ants-ci/ci" - "40ants-ci/changelog") + "40ants-ci/jobs/job" + "40ants-ci/jobs/docs" + "40ants-ci/jobs/linter" + "40ants-ci/jobs/critic" + "40ants-ci/jobs/run-tests" + "40ants-ci/jobs/autotag") :description "A tool simplify continuous deployment for Common Lisp projects." :homepage "https://40ants.com/ci/" :source-control (:git "https://github.com/40ants/ci") diff --git a/src/changelog.lisp b/docs/changelog.lisp similarity index 97% rename from src/changelog.lisp rename to docs/changelog.lisp index 807c16f..8a5cfb0 100644 --- a/src/changelog.lisp +++ b/docs/changelog.lisp @@ -1,8 +1,8 @@ -(uiop:define-package #:40ants-ci/changelog +(uiop:define-package #:40ants-ci-docs/changelog (:use #:cl) (:import-from #:40ants-doc/changelog #:defchangelog)) -(in-package #:40ants-ci/changelog) +(in-package #:40ants-ci-docs/changelog) (defchangelog (:ignore-words ("40ANTS-DOC" diff --git a/docs/index.lisp b/docs/index.lisp new file mode 100644 index 0000000..64f013d --- /dev/null +++ b/docs/index.lisp @@ -0,0 +1,567 @@ +(uiop:define-package #:40ants-ci-docs/index + (:use #:cl) + (:import-from #:40ants-doc + #:defsection + #:reader + #:section + #:defsection-copy) + (:import-from #:docs-config + #:docs-config) + (:import-from #:40ants-doc/autodoc + #:defautodoc)) +(in-package #:40ants-ci-docs/index) + + +(defmethod docs-config ((system (eql (asdf:find-system "40ants-ci")))) + ;; 40ANTS-DOC-THEME-40ANTS system will bring + ;; as dependency a full 40ANTS-DOC but we don't want + ;; unnecessary dependencies here: + (uiop:symbol-call :ql :quickload :40ants-doc-theme-40ants) + (list :theme + (find-symbol "40ANTS-THEME" + (find-package "40ANTS-DOC-THEME-40ANTS")))) + + +(defsection @index (:title "40Ants-CI - Github Workflow Generator" + :ignore-words ("YAML" + "GIT" + "CL" + "CI" + "JSON" + "OS" + "SBL" + "BSD" + "TODO" + "ASD" + "ASDF" + "EXAMPLE")) + " +[![](https://github-actions.40ants.com/40ants/ci/matrix.svg)](https://github.com/40ants/ci/actions) + +![Quicklisp](http://quickdocs.org/badge/ci.svg) + +This is a small utility, which can generate GitHub workflows for Common Lisp +projects. + +It generates workflow for running tests and building docs. These workflows +use [40ants/run-tests](https://40ants.com/run-tests) and [40ants/build-docs](https://40ants.com/build-doc) +actions and [`SBLint`](https://github.com/cxxxr/sblint) to check code for compilation errors. +" + (40ants-ci system) + (@reasons section) + (@quickstart section) + (@details section) + (@api section)) + + +(defsection-copy @readme @index) + + +(defsection @reasons (:title "Reasons to Use") + " +- This system hides all entrails related to caching. +- Includes a few ready to use job types. +- Custom job types can be defined and distributed as separate ASDF systems. +- You don't have to write YAML anymore! +") + + +(defsection @quickstart (:title "Quickstart") + " +This system allows you to define workflows in the lisp code. The best way is to make these +definitions a part of your ASDF system. This way 40ANTS-CI will be able to +automatically understand for which system it builds a workflow. + +Each workflow consists of jobs and each job is a number of steps. + +There are three predefine types of jobs and you can create your own. Predefined jobs +allows to reuse steps in multiple CL libraries. + +In next examples, I'll presume you are writing code in a file which is the part +of the package inferred ASDF system `EXAMPLE/CI`. A file should have the following header: + +```lisp +(defpackage #:example/ci + (:use #:cl) + (:import-from #:40ants-ci/workflow + #:defworkflow) + (:import-from #:40ants-ci/jobs/linter) + (:import-from #:40ants-ci/jobs/run-tests) + (:import-from #:40ants-ci/jobs/docs)) +``` +" + (@job-types section) + (@caching section)) + + +(defsection @job-types (:title "Job Types") + (@autotag section) + (@linter section) + (@critic section) + (@run-tests section) + (@build-docs section)) + + +(defsection @linter (:title "Linter") + " +The simplest job type is linter. It loads a + +```lisp +(defworkflow linter + :on-pull-request t + :jobs ((40ants-ci/jobs/linter:linter))) +``` + +When you'll hit `C-c C-c` on this definition, +it will generate `.github/workflows/linter.yml` with following content: + +```json +{ + \"name\": \"LINTER\", + \"on\": { + \"pull_request\": null + }, + \"jobs\": { + \"linter\": { + \"runs-on\": \"ubuntu-latest\", + \"env\": { + \"OS\": \"ubuntu-latest\", + \"QUICKLISP_DIST\": \"quicklisp\", + \"LISP\": \"sbcl-bin\" + }, + \"steps\": [ + { + \"name\": \"Checkout Code\", + \"uses\": \"actions/checkout@v4\" + }, + { + \"name\": \"Setup Common Lisp Environment\", + \"uses\": \"40ants/setup-lisp@v4\", + \"with\": { + \"asdf-system\": \"example\" + } + }, + { + \"name\": \"Install SBLint\", + \"run\": \"qlot exec ros install cxxxr/sblint\", + \"shell\": \"bash\" + }, + { + \"name\": \"Run Linter\", + \"run\": \"qlot exec sblint example.asd\", + \"shell\": \"bash\" + } + ] + } + } +} +``` + +Here you can see, a few steps in the job: + +1. Checkout the code. +2. Install Roswell & Qlot using [40ants/setup-lisp](https://40ants.com/setup-lisp/) action. +3. Install [`SBLint`](https://github.com/cxxxr/sblint). +4. Run linter for `example.asd`. + +Another interesting thing is that this workflow automatically uses `ubuntu-latest` OS, +`Quicklisp` and `sbcl-bin` Lisp implementation. Later I'll show you how to redefine these settings. +" + (40ants-ci/jobs/linter:linter class)) + + +(defsection @critic (:title "Critic") + " +This job is similar to linter, but instead of SBLint it runs +[Lisp Critic](https://40ants.com/40ants-critic). + +Lisp Critic is a program which advices how to make you Common Lisp code more +idiomatic, readable and performant. Also, sometimes it might catch logical +errors in the code. + +Here is how you can add this job type in your workflow: + +```lisp +(defworkflow ci + :on-pull-request t + :jobs ((40ants-ci/jobs/critic:critic))) +``` + +Also, you might combine this job together with others, for example, +with linter: + +```lisp +(defworkflow ci + :on-pull-request t + :jobs ((40ants-ci/jobs/linter:linter) + (40ants-ci/jobs/critic:critic))) +``` + +and they will be executed in parallel. See docs on 40ANTS-CI/JOBS/CRITIC:CRITIC function +to learn about supported arguments.") + + +(defsection @autotag (:title "Autotag") + " +This job is automates git tag placement on the commit where you have changed the ChangeLog.md. + +This can be a useful to automate package deployment and releases. You update the changelog, +a job pushes a new git tag and the next action triggers on this tag and build a release. + +Or you if you publish your library at Quicklisp distribution, then you can change +it's source type to the `latest-github-tag` to provide more stable releases to your +users. This way you commits into master will be ignored until you change the changelog and +git tag will be pushed. Here is an [example](https://github.com/quicklisp/quicklisp-projects/blob/ee133271c81caf5d8bbf8cef3054544ff47b64c6/projects/alexa/source.txt) how to setup this kind of quicklisp project source. + +```lisp +(defworkflow release + :on-push-to \"master\" + :jobs ((40ants-ci/jobs/autotag:autotag))) +``` +" + (40ants-ci/jobs/autotag:autotag function) + (40ants-ci/jobs/autotag:autotag class)) + + +(defsection @run-tests (:title "Running Tests" + :ignore-words ("ASDF:TEST-SYSTEM")) + " +Another interesting job type is 40ANTS-CI/JOBS/RUN-TESTS:RUN-TESTS. + +When using this job type, make sure, your system +runs tests on `(ASDF:TEST-SYSTEM :system-name)` call +and signals error if something went wrong. + +```lisp + +(defworkflow ci + :on-push-to \"master\" + :by-cron \"0 10 * * 1\" + :on-pull-request t + :jobs ((40ants-ci/jobs/run-tests:run-tests + :coverage t))) + +``` + +Here I've added a few options to the workflow: + +- `by-cron` - sets a schedule. +- `on-push-to` - defines a branch or branches to track. + +It will generate `.github/workflows/ci.yml` with following content: + +```json +{ + \"name\": \"CI\", + \"on\": { + \"push\": { + \"branches\": [ + \"master\" + ] + }, + \"pull_request\": null, + \"schedule\": [ + { + \"cron\": \"0 10 * * 1\" + } + ] + }, + \"jobs\": { + + \"run-tests\": { + \"runs-on\": \"ubuntu-latest\", + \"env\": { + \"OS\": \"ubuntu-latest\", + \"QUICKLISP_DIST\": \"quicklisp\", + \"LISP\": \"sbcl-bin\" + }, + \"steps\": [ + { + \"name\": \"Checkout Code\", + \"uses\": \"actions/checkout@v4\" + }, + { + \"name\": \"Setup Common Lisp Environment\", + \"uses\": \"40ants/setup-lisp@v4\", + \"with\": { + \"asdf-system\": \"example\" + } + }, + { + \"name\": \"Run Tests\", + \"uses\": \"40ants/run-tests@v2\", + \"with\": { + \"asdf-system\": \"example\", + \"coveralls-token\": \"${{ secrets.github_token }}\" + } + } + ] + } + } +} + +``` + +The result is similar to the workflow generated for Linter, +but uses [40ants/setup-lisp](https://40ants.com/run-tests/) action +at the final step. + +Also, I've passed an option `:coverage t` to the job. Thus coverage +report will be uploaded to [Coveralls.io](https://coveralls.io/) automatically. +" + (@matrix section) + (@multiple-jobs section)) + + +(defsection @matrix (:title "Defining a test Matrix") + " +Lisp has many implementations and can be used on multiple platforms. Thus +it is a good idea to test our software on many combinations of OS and lisp +implementations. Workflow generator makes this very easy. + +Here is an example of workflow definition with three dimentional matrix. +It not only tests a library under different lisps and OS, but also checks +if it works with the latest Quicklisp and Ultralisp distributions: + +```lisp +(defworkflow ci + :on-pull-request t + :jobs ((run-tests + :os (\"ubuntu-latest\" + \"macos-latest\") + :quicklisp (\"quicklisp\" + \"ultralisp\") + :lisp (\"sbcl-bin\" + \"ccl-bin\" + \"allegro\" + \"clisp\" + \"cmucl\") + :exclude (;; Seems allegro is does not support 64bit OSX. + ;; Unable to install it using Roswell: + ;; alisp is not executable. Missing 32bit glibc? + (:os \"macos-latest\" :lisp \"allegro\"))))) +``` +") + + +(defsection @multiple-jobs (:title "Multiple jobs") + " +Besides a build matrix, you might specify a multiple jobs of the same type, +but with different parameters: + +```lisp +(defworkflow ci + :on-push-to \"master\" + :on-pull-request t + :jobs ((run-tests + :lisp \"sbcl-bin\") + (run-tests + :lisp \"ccl-bin\") + (run-tests + :lisp \"allegro\"))) +``` + +This will generate a workflow with three jobs: \"run-tests\", \"run-tests-2\" and \"run-tests-3\". + +Meaningful names might be specified as well: + +```lisp +(defworkflow ci + :on-push-to \"master\" + :on-pull-request t + :jobs ((run-tests + :name \"test-on-sbcl\" + :lisp \"sbcl-bin\") + (run-tests + :name \"test-on-ccl\" + :lisp \"ccl-bin\") + (run-tests + :name \"test-on-allegro\" + :lisp \"allegro\"))) +``` + +Here is how these jobs will look like in the GitHub interface: + +![](https://user-images.githubusercontent.com/24827/151619261-2d49e2a6-bc5c-42db-aec5-674d9229a1b0.png) + +") + + +(defsection @build-docs (:title "Building Docs") + " +Third predefined job type is 40ANTS-CI/JOBS/DOCS:BUILD-DOCS. +It uses [40ants/build-docs](https://40ants.com/build-docs/) +action and will work only if your ASDF system uses a documentation builder supported by +[40ants/docs-builder](https://40ants.com/docs-builder/). + +To build docs on every push to master, just use this code: + +```lisp + +(defworkflow docs + :on-push-to \"master\" + :jobs ((40ants-ci/jobs/docs:build-docs))) + +``` + +It will generate `.github/workflows/docs.yml` with following content: + +```json + +{ + \"name\": \"DOCS\", + \"on\": { + \"push\": { + \"branches\": [ + \"master\" + ] + } + }, + \"jobs\": { + \"build-docs\": { + \"runs-on\": \"ubuntu-latest\", + \"env\": { + \"OS\": \"ubuntu-latest\", + \"QUICKLISP_DIST\": \"quicklisp\", + \"LISP\": \"sbcl-bin\" + }, + \"steps\": [ + { + \"name\": \"Checkout Code\", + \"uses\": \"actions/checkout@v4\" + }, + { + \"name\": \"Setup Common Lisp Environment\", + \"uses\": \"40ants/setup-lisp@v4\", + \"with\": { + \"asdf-system\": \"example\", + \"qlfile-template\": \"\" + } + }, + { + \"name\": \"Build Docs\", + \"uses\": \"40ants/build-docs@v1\", + \"with\": { + \"asdf-system\": \"example\" + } + } + ] + } + } +} + +``` + +") + + +(defsection @caching (:title "Caching") + " +To significantly speed up our tests, we can cache installed Roswell, +Qlot and Common Lisp fasl files. + +To accomplish this task, you don't need to dig into GitHub's docs anymore! +Just add one line `:cache t` to your workflow definition: + +```lisp +(defworkflow docs + :on-push-to \"master\" + :cache t + :jobs ((40ants-ci/jobs/docs:build-docs))) +``` + +Here is the diff of the generated workflow file. It shows steps, added automatically: + + +```diff +modified .github/workflows/docs.yml +@@ -20,13 +20,40 @@ + \"name\": \"Checkout Code\", + \"uses\": \"actions/checkout@v4\" + }, ++ { ++ \"name\": \"Grant All Perms to Make Cache Restoring Possible\", ++ \"run\": \"sudo mkdir -p /usr/local/etc/roswell\\n sudo chown \\\"${USER}\\\" /usr/local/etc/roswell\\n # Here the ros binary will be restored:\\n sudo chown \\\"${USER}\\\" /usr/local/bin\", ++ \"shell\": \"bash\" ++ }, ++ { ++ \"name\": \"Get Current Month\", ++ \"id\": \"current-month\", ++ \"run\": \"echo \\\"::set-output name=value::$(date -u \\\"+%Y-%m\\\")\\\"\", ++ \"shell\": \"bash\" ++ }, ++ { ++ \"name\": \"Cache Roswell Setup\", ++ \"id\": \"cache\", ++ \"uses\": \"actions/cache@v3\", ++ \"with\": { ++ \"path\": \"qlfile\\n qlfile.lock\\n /usr/local/bin/ros\\n ~/.cache/common-lisp/\\n ~/.roswell\\n /usr/local/etc/roswell\\n .qlot\", ++ \"key\": \"${{ steps.current-month.outputs.value }}-${{ env.cache-name }}-ubuntu-latest-quicklisp-sbcl-bin-${{ hashFiles('qlfile.lock') }}\" ++ } ++ }, ++ { ++ \"name\": \"Restore Path To Cached Files\", ++ \"run\": \"echo $HOME/.roswell/bin >> $GITHUB_PATH\\n echo .qlot/bin >> $GITHUB_PATH\", ++ \"shell\": \"bash\", ++ \"if\": \"steps.cache.outputs.cache-hit == 'true'\" ++ }, + { + \"name\": \"Setup Common Lisp Environment\", + \"uses\": \"40ants/setup-lisp@v4\", + \"with\": { + \"asdf-system\": \"40ants-ci\", + \"qlfile-template\": \"\" +- } ++ }, ++ \"if\": \"steps.cache.outputs.cache-hit != 'true'\" + }, + { +``` + +") + + +(defsection @details (:title "Details" + :ignore-words ("ASDF" + "CCL-BIN")) + " +TODO: I have to write a few chapters with details on additional job's parameters +and a way how to create new job types. + +But for now, I want to show a small example, how to define a workflow with a +job which takes care about lisp installation and then calls a custom step: + + +```lisp +(defworkflow ci + :on-push-to \"master\" + :by-cron \"0 10 * * 1\" + :on-pull-request t + :cache t + :jobs ((40ants-ci/jobs/lisp-job:lisp-job :name \"check-ros-config\" + :lisp \"ccl-bin\" + :steps ((40ants-ci/steps/sh:sh \"Show Roswell Config\" + \"ros config\"))))) +``` + +Here we are using the class 40ANTS-CI/JOBS/LISP-JOB:LISP-JOB which is base for most classes in this ASDF system +and pass a custom 40ANTS-CI/STEPS/SH:SH step to it. This step will be called after the repostory checkout and CCL-BIN lisp installation. +so, thus when this step will run `ros config` command, it will output something like that: + +``` +asdf.version=3.3.5.3 +ccl-bin.version=1.12.2 +setup.time=3918000017 +sbcl-bin.version=2.4.1 +default.lisp=ccl-bin + +Possible subcommands: +set +show +``` + +Pay attention to the NAME argument of 40ANTS-CI/JOBS/LISP-JOB:LISP-JOB class. If you omit it, then default \"lisp-job\" name will be used. +") + + +(defautodoc @api (:system "40ants-ci")) diff --git a/src/ci.lisp b/src/ci.lisp index 01b75d4..5395d15 100644 --- a/src/ci.lisp +++ b/src/ci.lisp @@ -24,7 +24,8 @@ :on-pull-request t :by-cron "0 10 * * 1" :cache nil - :jobs ((40ants-ci/jobs/docs:build-docs))) + :jobs ((40ants-ci/jobs/docs:build-docs + :asdf-system "40ants-ci-docs"))) (defworkflow ci diff --git a/src/core.lisp b/src/core.lisp index 7a54d6c..4066ed8 100644 --- a/src/core.lisp +++ b/src/core.lisp @@ -1,582 +1,9 @@ (uiop:define-package #:40ants-ci (:nicknames #:40ants-ci/core) (:use #:cl) - (:import-from #:40ants-doc - #:defsection - #:reader - #:section - #:defsection-copy) (:import-from #:40ants-ci/github) - (:import-from #:40ants-ci/jobs/job) - (:import-from #:40ants-ci/jobs/lisp-job) - (:import-from #:40ants-ci/jobs/docs) - (:import-from #:40ants-ci/jobs/linter) - (:import-from #:40ants-ci/jobs/critic) - (:import-from #:40ants-ci/jobs/run-tests) - (:import-from #:40ants-ci/jobs/autotag) - (:import-from #:docs-config - #:docs-config) - (:import-from #:40ants-doc/autodoc - #:defautodoc) - (:export #:generate - #:@index - #:@readme)) -(in-package 40ants-ci) - - -(defmethod docs-config ((system (eql (asdf:find-system "40ants-ci")))) - ;; 40ANTS-DOC-THEME-40ANTS system will bring - ;; as dependency a full 40ANTS-DOC but we don't want - ;; unnecessary dependencies here: - (uiop:symbol-call :ql :quickload :40ants-doc-theme-40ants) - (list :theme - (find-symbol "40ANTS-THEME" - (find-package "40ANTS-DOC-THEME-40ANTS")))) - - -(defsection @index (:title "40Ants-CI - Github Workflow Generator" - :ignore-words ("YAML" - "GIT" - "CL" - "CI" - "JSON" - "OS" - "SBL" - "BSD" - "TODO" - "ASD" - "ASDF" - "EXAMPLE")) - " -[![](https://github-actions.40ants.com/40ants/ci/matrix.svg)](https://github.com/40ants/ci/actions) - -![Quicklisp](http://quickdocs.org/badge/ci.svg) - -This is a small utility, which can generate GitHub workflows for Common Lisp -projects. - -It generates workflow for running tests and building docs. These workflows -use [40ants/run-tests](https://40ants.com/run-tests) and [40ants/build-docs](https://40ants.com/build-doc) -actions and [`SBLint`](https://github.com/cxxxr/sblint) to check code for compilation errors. -" - (40ants-ci system) - (@reasons section) - (@quickstart section) - (@details section) - (@api section)) - - -(defsection-copy @readme @index) - - -(defsection @reasons (:title "Reasons to Use") - " -- This system hides all entrails related to caching. -- Includes a few ready to use job types. -- Custom job types can be defined and distributed as separate ASDF systems. -- You don't have to write YAML anymore! -") - - -(defsection @quickstart (:title "Quickstart") - " -This system allows you to define workflows in the lisp code. The best way is to make these -definitions a part of your ASDF system. This way 40ANTS-CI will be able to -automatically understand for which system it builds a workflow. - -Each workflow consists of jobs and each job is a number of steps. - -There are three predefine types of jobs and you can create your own. Predefined jobs -allows to reuse steps in multiple CL libraries. - -In next examples, I'll presume you are writing code in a file which is the part -of the package inferred ASDF system `EXAMPLE/CI`. A file should have the following header: - -```lisp -(defpackage #:example/ci - (:use #:cl) - (:import-from #:40ants-ci/workflow - #:defworkflow) - (:import-from #:40ants-ci/jobs/linter) - (:import-from #:40ants-ci/jobs/run-tests) - (:import-from #:40ants-ci/jobs/docs)) -``` -" - (@job-types section) - (@caching section)) - - -(defsection @job-types (:title "Job Types") - (@autotag section) - (@linter section) - (@critic section) - (@run-tests section) - (@build-docs section)) - - -(defsection @linter (:title "Linter") - " -The simplest job type is linter. It loads a - -```lisp -(defworkflow linter - :on-pull-request t - :jobs ((40ants-ci/jobs/linter:linter))) -``` - -When you'll hit `C-c C-c` on this definition, -it will generate `.github/workflows/linter.yml` with following content: - -```json -{ - \"name\": \"LINTER\", - \"on\": { - \"pull_request\": null - }, - \"jobs\": { - \"linter\": { - \"runs-on\": \"ubuntu-latest\", - \"env\": { - \"OS\": \"ubuntu-latest\", - \"QUICKLISP_DIST\": \"quicklisp\", - \"LISP\": \"sbcl-bin\" - }, - \"steps\": [ - { - \"name\": \"Checkout Code\", - \"uses\": \"actions/checkout@v4\" - }, - { - \"name\": \"Setup Common Lisp Environment\", - \"uses\": \"40ants/setup-lisp@v4\", - \"with\": { - \"asdf-system\": \"example\" - } - }, - { - \"name\": \"Install SBLint\", - \"run\": \"qlot exec ros install cxxxr/sblint\", - \"shell\": \"bash\" - }, - { - \"name\": \"Run Linter\", - \"run\": \"qlot exec sblint example.asd\", - \"shell\": \"bash\" - } - ] - } - } -} -``` - -Here you can see, a few steps in the job: - -1. Checkout the code. -2. Install Roswell & Qlot using [40ants/setup-lisp](https://40ants.com/setup-lisp/) action. -3. Install [`SBLint`](https://github.com/cxxxr/sblint). -4. Run linter for `example.asd`. - -Another interesting thing is that this workflow automatically uses `ubuntu-latest` OS, -`Quicklisp` and `sbcl-bin` Lisp implementation. Later I'll show you how to redefine these settings. -" - (40ants-ci/jobs/linter:linter class)) - - -(defsection @critic (:title "Critic") - " -This job is similar to linter, but instead of SBLint it runs -[Lisp Critic](https://40ants.com/40ants-critic). - -Lisp Critic is a program which advices how to make you Common Lisp code more -idiomatic, readable and performant. Also, sometimes it might catch logical -errors in the code. - -Here is how you can add this job type in your workflow: - -```lisp -(defworkflow ci - :on-pull-request t - :jobs ((40ants-ci/jobs/critic:critic))) -``` - -Also, you might combine this job together with others, for example, -with linter: - -```lisp -(defworkflow ci - :on-pull-request t - :jobs ((40ants-ci/jobs/linter:linter) - (40ants-ci/jobs/critic:critic))) -``` - -and they will be executed in parallel. See docs on 40ANTS-CI/JOBS/CRITIC:CRITIC function -to learn about supported arguments.") - - -(defsection @autotag (:title "Autotag") - " -This job is automates git tag placement on the commit where you have changed the ChangeLog.md. - -This can be a useful to automate package deployment and releases. You update the changelog, -a job pushes a new git tag and the next action triggers on this tag and build a release. - -Or you if you publish your library at Quicklisp distribution, then you can change -it's source type to the `latest-github-tag` to provide more stable releases to your -users. This way you commits into master will be ignored until you change the changelog and -git tag will be pushed. Here is an [example](https://github.com/quicklisp/quicklisp-projects/blob/ee133271c81caf5d8bbf8cef3054544ff47b64c6/projects/alexa/source.txt) how to setup this kind of quicklisp project source. - -```lisp -(defworkflow release - :on-push-to \"master\" - :jobs ((40ants-ci/jobs/autotag:autotag))) -``` -" - (40ants-ci/jobs/autotag:autotag function) - (40ants-ci/jobs/autotag:autotag class)) - - -(defsection @run-tests (:title "Running Tests" - :ignore-words ("ASDF:TEST-SYSTEM")) - " -Another interesting job type is 40ANTS-CI/JOBS/RUN-TESTS:RUN-TESTS. - -When using this job type, make sure, your system -runs tests on `(ASDF:TEST-SYSTEM :system-name)` call -and signals error if something went wrong. - -```lisp - -(defworkflow ci - :on-push-to \"master\" - :by-cron \"0 10 * * 1\" - :on-pull-request t - :jobs ((40ants-ci/jobs/run-tests:run-tests - :coverage t))) - -``` - -Here I've added a few options to the workflow: - -- `by-cron` - sets a schedule. -- `on-push-to` - defines a branch or branches to track. - -It will generate `.github/workflows/ci.yml` with following content: - -```json -{ - \"name\": \"CI\", - \"on\": { - \"push\": { - \"branches\": [ - \"master\" - ] - }, - \"pull_request\": null, - \"schedule\": [ - { - \"cron\": \"0 10 * * 1\" - } - ] - }, - \"jobs\": { - - \"run-tests\": { - \"runs-on\": \"ubuntu-latest\", - \"env\": { - \"OS\": \"ubuntu-latest\", - \"QUICKLISP_DIST\": \"quicklisp\", - \"LISP\": \"sbcl-bin\" - }, - \"steps\": [ - { - \"name\": \"Checkout Code\", - \"uses\": \"actions/checkout@v4\" - }, - { - \"name\": \"Setup Common Lisp Environment\", - \"uses\": \"40ants/setup-lisp@v4\", - \"with\": { - \"asdf-system\": \"example\" - } - }, - { - \"name\": \"Run Tests\", - \"uses\": \"40ants/run-tests@v2\", - \"with\": { - \"asdf-system\": \"example\", - \"coveralls-token\": \"${{ secrets.github_token }}\" - } - } - ] - } - } -} - -``` - -The result is similar to the workflow generated for Linter, -but uses [40ants/setup-lisp](https://40ants.com/run-tests/) action -at the final step. - -Also, I've passed an option `:coverage t` to the job. Thus coverage -report will be uploaded to [Coveralls.io](https://coveralls.io/) automatically. -" - (@matrix section) - (@multiple-jobs section)) - - -(defsection @matrix (:title "Defining a test Matrix") - " -Lisp has many implementations and can be used on multiple platforms. Thus -it is a good idea to test our software on many combinations of OS and lisp -implementations. Workflow generator makes this very easy. - -Here is an example of workflow definition with three dimentional matrix. -It not only tests a library under different lisps and OS, but also checks -if it works with the latest Quicklisp and Ultralisp distributions: - -```lisp -(defworkflow ci - :on-pull-request t - :jobs ((run-tests - :os (\"ubuntu-latest\" - \"macos-latest\") - :quicklisp (\"quicklisp\" - \"ultralisp\") - :lisp (\"sbcl-bin\" - \"ccl-bin\" - \"allegro\" - \"clisp\" - \"cmucl\") - :exclude (;; Seems allegro is does not support 64bit OSX. - ;; Unable to install it using Roswell: - ;; alisp is not executable. Missing 32bit glibc? - (:os \"macos-latest\" :lisp \"allegro\"))))) -``` -") - - -(defsection @multiple-jobs (:title "Multiple jobs") - " -Besides a build matrix, you might specify a multiple jobs of the same type, -but with different parameters: - -```lisp -(defworkflow ci - :on-push-to \"master\" - :on-pull-request t - :jobs ((run-tests - :lisp \"sbcl-bin\") - (run-tests - :lisp \"ccl-bin\") - (run-tests - :lisp \"allegro\"))) -``` - -This will generate a workflow with three jobs: \"run-tests\", \"run-tests-2\" and \"run-tests-3\". - -Meaningful names might be specified as well: - -```lisp -(defworkflow ci - :on-push-to \"master\" - :on-pull-request t - :jobs ((run-tests - :name \"test-on-sbcl\" - :lisp \"sbcl-bin\") - (run-tests - :name \"test-on-ccl\" - :lisp \"ccl-bin\") - (run-tests - :name \"test-on-allegro\" - :lisp \"allegro\"))) -``` - -Here is how these jobs will look like in the GitHub interface: - -![](https://user-images.githubusercontent.com/24827/151619261-2d49e2a6-bc5c-42db-aec5-674d9229a1b0.png) - -") - - -(defsection @build-docs (:title "Building Docs") - " -Third predefined job type is 40ANTS-CI/JOBS/DOCS:BUILD-DOCS. -It uses [40ants/build-docs](https://40ants.com/build-docs/) -action and will work only if your ASDF system uses a documentation builder supported by -[40ants/docs-builder](https://40ants.com/docs-builder/). - -To build docs on every push to master, just use this code: - -```lisp - -(defworkflow docs - :on-push-to \"master\" - :jobs ((40ants-ci/jobs/docs:build-docs))) - -``` - -It will generate `.github/workflows/docs.yml` with following content: - -```json - -{ - \"name\": \"DOCS\", - \"on\": { - \"push\": { - \"branches\": [ - \"master\" - ] - } - }, - \"jobs\": { - \"build-docs\": { - \"runs-on\": \"ubuntu-latest\", - \"env\": { - \"OS\": \"ubuntu-latest\", - \"QUICKLISP_DIST\": \"quicklisp\", - \"LISP\": \"sbcl-bin\" - }, - \"steps\": [ - { - \"name\": \"Checkout Code\", - \"uses\": \"actions/checkout@v4\" - }, - { - \"name\": \"Setup Common Lisp Environment\", - \"uses\": \"40ants/setup-lisp@v4\", - \"with\": { - \"asdf-system\": \"example\", - \"qlfile-template\": \"\" - } - }, - { - \"name\": \"Build Docs\", - \"uses\": \"40ants/build-docs@v1\", - \"with\": { - \"asdf-system\": \"example\" - } - } - ] - } - } -} - -``` - -") - - -(defsection @caching (:title "Caching") - " -To significantly speed up our tests, we can cache installed Roswell, -Qlot and Common Lisp fasl files. - -To accomplish this task, you don't need to dig into GitHub's docs anymore! -Just add one line `:cache t` to your workflow definition: - -```lisp -(defworkflow docs - :on-push-to \"master\" - :cache t - :jobs ((40ants-ci/jobs/docs:build-docs))) -``` - -Here is the diff of the generated workflow file. It shows steps, added automatically: - - -```diff -modified .github/workflows/docs.yml -@@ -20,13 +20,40 @@ - \"name\": \"Checkout Code\", - \"uses\": \"actions/checkout@v4\" - }, -+ { -+ \"name\": \"Grant All Perms to Make Cache Restoring Possible\", -+ \"run\": \"sudo mkdir -p /usr/local/etc/roswell\\n sudo chown \\\"${USER}\\\" /usr/local/etc/roswell\\n # Here the ros binary will be restored:\\n sudo chown \\\"${USER}\\\" /usr/local/bin\", -+ \"shell\": \"bash\" -+ }, -+ { -+ \"name\": \"Get Current Month\", -+ \"id\": \"current-month\", -+ \"run\": \"echo \\\"::set-output name=value::$(date -u \\\"+%Y-%m\\\")\\\"\", -+ \"shell\": \"bash\" -+ }, -+ { -+ \"name\": \"Cache Roswell Setup\", -+ \"id\": \"cache\", -+ \"uses\": \"actions/cache@v3\", -+ \"with\": { -+ \"path\": \"qlfile\\n qlfile.lock\\n /usr/local/bin/ros\\n ~/.cache/common-lisp/\\n ~/.roswell\\n /usr/local/etc/roswell\\n .qlot\", -+ \"key\": \"${{ steps.current-month.outputs.value }}-${{ env.cache-name }}-ubuntu-latest-quicklisp-sbcl-bin-${{ hashFiles('qlfile.lock') }}\" -+ } -+ }, -+ { -+ \"name\": \"Restore Path To Cached Files\", -+ \"run\": \"echo $HOME/.roswell/bin >> $GITHUB_PATH\\n echo .qlot/bin >> $GITHUB_PATH\", -+ \"shell\": \"bash\", -+ \"if\": \"steps.cache.outputs.cache-hit == 'true'\" -+ }, - { - \"name\": \"Setup Common Lisp Environment\", - \"uses\": \"40ants/setup-lisp@v4\", - \"with\": { - \"asdf-system\": \"40ants-ci\", - \"qlfile-template\": \"\" -- } -+ }, -+ \"if\": \"steps.cache.outputs.cache-hit != 'true'\" - }, - { -``` - -") - - -(defsection @details (:title "Details" - :ignore-words ("ASDF" - "CCL-BIN")) - " -TODO: I have to write a few chapters with details on additional job's parameters -and a way how to create new job types. - -But for now, I want to show a small example, how to define a workflow with a -job which takes care about lisp installation and then calls a custom step: - - -```lisp -(defworkflow ci - :on-push-to \"master\" - :by-cron \"0 10 * * 1\" - :on-pull-request t - :cache t - :jobs ((40ants-ci/jobs/lisp-job:lisp-job :name \"check-ros-config\" - :lisp \"ccl-bin\" - :steps ((40ants-ci/steps/sh:sh \"Show Roswell Config\" - \"ros config\"))))) -``` - -Here we are using the class 40ANTS-CI/JOBS/LISP-JOB:LISP-JOB which is base for most classes in this ASDF system -and pass a custom 40ANTS-CI/STEPS/SH:SH step to it. This step will be called after the repostory checkout and CCL-BIN lisp installation. -so, thus when this step will run `ros config` command, it will output something like that: - -``` -asdf.version=3.3.5.3 -ccl-bin.version=1.12.2 -setup.time=3918000017 -sbcl-bin.version=2.4.1 -default.lisp=ccl-bin - -Possible subcommands: -set -show -``` - -Pay attention to the NAME argument of 40ANTS-CI/JOBS/LISP-JOB:LISP-JOB class. If you omit it, then default \"lisp-job\" name will be used. -") - - -(defautodoc @api (:system "40ants-ci")) + (:export #:generate)) +(in-package #:40ants-ci) (defun generate (system &key path) From cd0920a5db8d79d9b758b7854678f0db6c76e9f3 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Tue, 27 Feb 2024 22:34:57 +0300 Subject: [PATCH 12/22] Updated docs workflow. --- .github/workflows/docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0cd1739..ae399d5 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,15 +30,15 @@ "name": "Setup Common Lisp Environment", "uses": "40ants/setup-lisp@v4", "with": { - "asdf-system": "40ants-ci", - "cache": "true" + "asdf-system": "40ants-ci-docs", + "cache": "false" } }, { "name": "Build Docs", "uses": "40ants/build-docs@v1", "with": { - "asdf-system": "40ants-ci", + "asdf-system": "40ants-ci-docs", "error-on-warnings": true } } From ed7b1249e6e774009f06028de34db936e24596de Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Wed, 28 Feb 2024 00:03:30 +0300 Subject: [PATCH 13/22] Export sections. --- docs/index.lisp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/index.lisp b/docs/index.lisp index 64f013d..8795af9 100644 --- a/docs/index.lisp +++ b/docs/index.lisp @@ -8,11 +8,16 @@ (:import-from #:docs-config #:docs-config) (:import-from #:40ants-doc/autodoc - #:defautodoc)) + #:defautodoc) + (:import-from #:40ants-logging-docs/changelog + #:@changelog) + (:export #:@index + #:@readme + #:@changelog)) (in-package #:40ants-ci-docs/index) -(defmethod docs-config ((system (eql (asdf:find-system "40ants-ci")))) +(defmethod docs-config ((system (eql (asdf:find-system "40ants-ci-docs")))) ;; 40ANTS-DOC-THEME-40ANTS system will bring ;; as dependency a full 40ANTS-DOC but we don't want ;; unnecessary dependencies here: From 86c9312035b2069e7604bb8ad2b7b698dc94577a Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Wed, 28 Feb 2024 00:30:12 +0300 Subject: [PATCH 14/22] Enable cache again. --- .github/workflows/ci.yml | 18 +++++++++--------- .github/workflows/docs.yml | 6 +++--- src/ci.lisp | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c663c70..4e7f439 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,10 +28,10 @@ }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@v4", + "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", "with": { "asdf-system": "40ants-ci", - "cache": "false" + "cache": "true" } }, { @@ -70,10 +70,10 @@ }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@v4", + "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", "with": { "asdf-system": "40ants-ci", - "cache": "false" + "cache": "true" } }, { @@ -129,11 +129,11 @@ }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@v4", + "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", "with": { "asdf-system": "40ants-ci", "qlfile-template": "{% ifequal quicklisp_dist \"ultralisp\" %}\ndist ultralisp http://dist.ultralisp.org\n{% endifequal %}", - "cache": "false" + "cache": "true" } }, { @@ -160,10 +160,10 @@ }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@v4", + "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", "with": { "asdf-system": "40ants-ci", - "cache": "false" + "cache": "true" } }, { @@ -174,4 +174,4 @@ ] } } -} \ No newline at end of file +} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ae399d5..94e4132 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,10 +28,10 @@ }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@v4", + "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", "with": { "asdf-system": "40ants-ci-docs", - "cache": "false" + "cache": "true" } }, { @@ -45,4 +45,4 @@ ] } } -} \ No newline at end of file +} diff --git a/src/ci.lisp b/src/ci.lisp index 5395d15..e93211f 100644 --- a/src/ci.lisp +++ b/src/ci.lisp @@ -23,7 +23,7 @@ :on-push-to "master" :on-pull-request t :by-cron "0 10 * * 1" - :cache nil + :cache t :jobs ((40ants-ci/jobs/docs:build-docs :asdf-system "40ants-ci-docs"))) @@ -32,7 +32,7 @@ :on-push-to "master" :by-cron "0 10 * * 1" :on-pull-request t - :cache nil + :cache t :jobs ((40ants-ci/jobs/linter:linter :asdf-systems ("40ants-ci" "40ants-ci-tests") From a960399a7a6a8aaad9e09d6512de30931eb1b329 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Wed, 28 Feb 2024 00:47:03 +0300 Subject: [PATCH 15/22] Push. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e7f439..cdff8a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ }, "steps": [ { - "name": "Checkout Code", + "name": "Checkout Code 2", "uses": "actions/checkout@v4" }, { @@ -65,7 +65,7 @@ }, "steps": [ { - "name": "Checkout Code", + "name": "Checkout Code 2", "uses": "actions/checkout@v4" }, { @@ -124,7 +124,7 @@ }, "steps": [ { - "name": "Checkout Code", + "name": "Checkout Code 2", "uses": "actions/checkout@v4" }, { @@ -155,7 +155,7 @@ }, "steps": [ { - "name": "Checkout Code", + "name": "Checkout Code 2", "uses": "actions/checkout@v4" }, { From b3075bd9f1300355abc3b3f8ccf2318fbf2c9b02 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Wed, 28 Feb 2024 01:30:24 +0300 Subject: [PATCH 16/22] Return back to setup-lisp@v4 --- .github/workflows/ci.yml | 18 +++++++++--------- .github/workflows/docs.yml | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdff8a8..8738ff4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,12 +23,12 @@ }, "steps": [ { - "name": "Checkout Code 2", + "name": "Checkout Code", "uses": "actions/checkout@v4" }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", + "uses": "40ants/setup-lisp@v4", "with": { "asdf-system": "40ants-ci", "cache": "true" @@ -65,12 +65,12 @@ }, "steps": [ { - "name": "Checkout Code 2", + "name": "Checkout Code", "uses": "actions/checkout@v4" }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", + "uses": "40ants/setup-lisp@v4", "with": { "asdf-system": "40ants-ci", "cache": "true" @@ -124,12 +124,12 @@ }, "steps": [ { - "name": "Checkout Code 2", + "name": "Checkout Code", "uses": "actions/checkout@v4" }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", + "uses": "40ants/setup-lisp@v4", "with": { "asdf-system": "40ants-ci", "qlfile-template": "{% ifequal quicklisp_dist \"ultralisp\" %}\ndist ultralisp http://dist.ultralisp.org\n{% endifequal %}", @@ -155,12 +155,12 @@ }, "steps": [ { - "name": "Checkout Code 2", + "name": "Checkout Code", "uses": "actions/checkout@v4" }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", + "uses": "40ants/setup-lisp@v4", "with": { "asdf-system": "40ants-ci", "cache": "true" @@ -174,4 +174,4 @@ ] } } -} +} \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 94e4132..8c8fb1b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,7 +28,7 @@ }, { "name": "Setup Common Lisp Environment", - "uses": "40ants/setup-lisp@fix-recreating-of-qlfile-inside-cache", + "uses": "40ants/setup-lisp@v4", "with": { "asdf-system": "40ants-ci-docs", "cache": "true" @@ -45,4 +45,4 @@ ] } } -} +} \ No newline at end of file From 6932a6b20c2cdc8420684553b15d8af79297dd91 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Wed, 28 Feb 2024 10:24:42 +0300 Subject: [PATCH 17/22] Add custom env job. --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8738ff4..176b514 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,14 @@ "name": "Show Roswell Config", "run": "ros config", "shell": "bash" + }, + { + "name": "Custom ENV", + "run": "echo $CUSTOM_ENV", + "shell": "bash", + "env": { + "CUSTOM-ENV": "Hello world!" + } } ] } From 572016c18671f1f8b50fd502a283ca8eef6e131a Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Wed, 28 Feb 2024 10:42:12 +0300 Subject: [PATCH 18/22] Custom steps. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 176b514..787fd27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,7 +146,7 @@ } ] }, - "check-ros-config": { + "custom-steps": { "runs-on": "ubuntu-latest", "env": { "OS": "ubuntu-latest", @@ -176,7 +176,7 @@ "run": "echo $CUSTOM_ENV", "shell": "bash", "env": { - "CUSTOM-ENV": "Hello world!" + "CUSTOM_ENV": "Hello world!" } } ] From 370c945884479c86e7d92a43a4340eff4c6c4d71 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Sat, 2 Mar 2024 10:38:16 +0300 Subject: [PATCH 19/22] Refactor workflow, job and step env argument processing. --- .github/workflows/docs.yml | 4 +++ docs/changelog.lisp | 13 ++++++++ docs/index.lisp | 44 +++++++++++++++++++++++++++ src/ci.lisp | 15 ++++++---- src/jobs/docs.lisp | 16 +++++----- src/jobs/job.lisp | 44 +++++++++++++++++++-------- src/jobs/linter.lisp | 3 +- src/steps/sh.lisp | 7 +---- src/steps/step.lisp | 29 +++++++++++------- src/utils.lisp | 26 ++++++++++++++++ src/workflow.lisp | 61 ++++++++++++++++++++++++++++++-------- 11 files changed, 206 insertions(+), 56 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8c8fb1b..439fc78 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,10 +13,14 @@ } ] }, + "env": { + "CUSTOM_ENV_HELLO": "Hello" + }, "jobs": { "build-docs": { "runs-on": "ubuntu-latest", "env": { + "CUSTOM_ENV_WORLD": "World!", "OS": "ubuntu-latest", "QUICKLISP_DIST": "quicklisp", "LISP": "sbcl-bin" diff --git a/docs/changelog.lisp b/docs/changelog.lisp index 8a5cfb0..5716506 100644 --- a/docs/changelog.lisp +++ b/docs/changelog.lisp @@ -11,6 +11,19 @@ "GITHUB_TOKEN" "OSX") :external-docs ("https://40ants.com/40ants-asdf-system/")) + (0.15.0 2024-03-02 + " +New +=== + +* Now you can specify ENV argument to 40ANTS-CI:DEFWORKFLOW and any job. This should be an alist where keys are strings and values are evaluated during GitHub workflow generation phase. Read more in 40ANTS-CI-DOCS/INDEX::@ENV section. + +Backward incompatible changes +============================= + +* When additional keyword arguments to 40ANTS-CI/STEPS/SH:SH function are given, they are transformed into env variables. Previously, their names were taken as is. Now they are uppercased and dash symbols are replaced with underscores. + +") (0.14.0 2024-02-25 " Changed diff --git a/docs/index.lisp b/docs/index.lisp index 8795af9..c6f0ad9 100644 --- a/docs/index.lisp +++ b/docs/index.lisp @@ -569,4 +569,48 @@ Pay attention to the NAME argument of 40ANTS-CI/JOBS/LISP-JOB:LISP-JOB class. If ") +(defsection @env (:title "Adding env variables") + " +You can specify additional environment variables on any level of the GitHub workflow: for workflow itself, for a job or for a step. + +To specify env for workflow or a job, just add an ENV argument with alist or plist value like this: + +```lisp +(defworkflow release + :on-push-to \"master\" + :env (:github-token \"${{ secrets.autotag_token }}\") + :jobs ((40ants-ci/jobs/autotag:autotag))) +``` + +or as alist: + +```lisp +(defworkflow release + :on-push-to \"master\" + :env ((\"github_token\" . \"${{ secrets.autotag_token }}\")) + :jobs ((40ants-ci/jobs/autotag:autotag))) +``` + +or for the job itself: + +```lisp +(defworkflow release + :on-push-to \"master\" + :jobs ((40ants-ci/jobs/autotag:autotag + :env (:github-token \"${{ secrets.autotag_token }}\")))) +``` + +the same way it can be specified on a custom step: + +```lisp +(40ants-ci/steps/sh:sh \"Custom env-var example\" + \"echo $CUSTOM_VAR\" + :env (:custom-var \"Hello world!\")) +``` + +Note - environment variable names are always transformed to uppercase and dashes are replaced with underscores. + +") + + (defautodoc @api (:system "40ants-ci")) diff --git a/src/ci.lisp b/src/ci.lisp index e93211f..095c7e4 100644 --- a/src/ci.lisp +++ b/src/ci.lisp @@ -24,8 +24,10 @@ :on-pull-request t :by-cron "0 10 * * 1" :cache t + :env ((:custom-env-hello . "Hello")) :jobs ((40ants-ci/jobs/docs:build-docs - :asdf-system "40ants-ci-docs"))) + :asdf-system "40ants-ci-docs" + :env ((:custom-env-world . "World!"))))) (defworkflow ci @@ -51,11 +53,12 @@ :qlfile "{% ifequal quicklisp_dist \"ultralisp\" %} dist ultralisp http://dist.ultralisp.org {% endifequal %}") - ;; This is an example of a job with a custom - ;; step: - (40ants-ci/jobs/lisp-job:lisp-job :name "check-ros-config" + ;; This is an example of a job with a custom steps: + (40ants-ci/jobs/lisp-job:lisp-job :name "custom-steps" :lisp "ccl-bin" :steps ((40ants-ci/steps/sh:sh "Show Roswell Config" - "ros config"))))) - + "ros config") + (40ants-ci/steps/sh:sh "Custom ENV" + "echo $CUSTOM_ENV" + :env (:custom-env "Hello world!")))))) diff --git a/src/jobs/docs.lisp b/src/jobs/docs.lisp index 021ae30..ff0a930 100644 --- a/src/jobs/docs.lisp +++ b/src/jobs/docs.lisp @@ -19,14 +19,14 @@ (:documentation "Builds documentation and uploads it to GitHub using [\"40ants/build-docs\" github action](https://40ants.com/build-docs/).")) -(defun build-docs (&key asdf-system - asdf-version - (error-on-warnings t)) - "Creates a job of class BUILD-DOCS." - (make-instance 'build-docs - :asdf-system asdf-system - :error-on-warnings error-on-warnings - :asdf-version asdf-version)) +;; (defun build-docs (&key asdf-system +;; asdf-version +;; (error-on-warnings t)) +;; "Creates a job of class BUILD-DOCS." +;; (make-instance 'build-docs +;; :asdf-system asdf-system +;; :error-on-warnings error-on-warnings +;; :asdf-version asdf-version)) (defmethod 40ants-ci/jobs/job:steps ((job build-docs)) diff --git a/src/jobs/job.lisp b/src/jobs/job.lisp index e707f94..1cd67da 100644 --- a/src/jobs/job.lisp +++ b/src/jobs/job.lisp @@ -1,9 +1,11 @@ (uiop:define-package #:40ants-ci/jobs/job (:use #:cl) (:import-from #:40ants-ci/utils + #:to-env-alist #:ensure-list-of-plists) (:import-from #:40ants-ci/github) (:import-from #:serapeum + #:soft-alist-of #:length<) (:import-from #:alexandria #:length=) @@ -19,7 +21,8 @@ #:permissions #:make-permissions #:explicit-steps - #:exclude)) + #:exclude + #:job-env)) (in-package #:40ants-ci/jobs/job) @@ -34,6 +37,11 @@ :initarg :exclude :reader exclude :documentation "A list of plists denoting matrix combinations to be excluded.") + (env :initform nil + :type (soft-alist-of string string) + :initarg :env + :documentation "An alist of environment variables and their values to be added on job level. Values are evaluated in runtime." + :reader job-env) (steps :initform nil :initarg :steps :documentation "This slot holds steps given as a STEPS argument to a job constructor. Depending on a job class, it might add additional steps around these explicit steps." @@ -52,17 +60,26 @@ :reader permissions))) -(defmethod initialize-instance :after ((job job) &rest initargs) - (declare (ignore initargs)) - - (unless (slot-boundp job 'name) - (setf (slot-value job 'name) - (string-downcase - (class-name (class-of job))))) - - (setf (slot-value job 'steps) - (mapcar #'ensure-step - (slot-value job 'steps)))) +(defmethod initialize-instance :around ((job job) &rest initargs) + (let* ((initargs (copy-list initargs)) + (env (getf initargs :env))) + (when env + (setf (getf initargs :env) + (to-env-alist env))) + + (unless (getf initargs :name) + (setf (getf initargs :name) + (string-downcase + (class-name (class-of job))))) + + + (setf (getf initargs :steps) + (mapcar #'ensure-step + (getf initargs :steps))) + + (apply #'call-next-method + job + initargs))) (defmethod os :around ((job job)) @@ -103,6 +120,9 @@ (defgeneric make-env (job) (:method ((job job)) (append + (when (job-env job) + (job-env job)) + (cond ((length< 1 (os job)) `(("OS" . "${{ matrix.os }}"))) diff --git a/src/jobs/linter.lisp b/src/jobs/linter.lisp index 012bec9..9ab7ea5 100644 --- a/src/jobs/linter.lisp +++ b/src/jobs/linter.lisp @@ -32,7 +32,7 @@ (current-system-name)))) -(defun linter (&key asdf-systems asdf-version check-imports) +(defun linter (&key asdf-systems asdf-version check-imports env) "Creates a job which will run SBLint for given ASDF systems. If no ASD files given, it will use all ASD files from @@ -41,6 +41,7 @@ :asdf-system (first asdf-systems) :asdf-systems asdf-systems :asdf-version asdf-version + :env env :check-imports check-imports)) diff --git a/src/steps/sh.lisp b/src/steps/sh.lisp index 29c297f..12cca34 100644 --- a/src/steps/sh.lisp +++ b/src/steps/sh.lisp @@ -25,12 +25,7 @@ ;; ignore-critiques: if-no-else -(defun sh (name command &rest env &key - id - if - (shell *default-shell*) - &allow-other-keys) - (remove-from-plistf env :id :if :shell) +(defun sh (name command &key id if (shell *default-shell*) env) (make-instance 'sh :name name :command command diff --git a/src/steps/step.lisp b/src/steps/step.lisp index ba5074d..ad4f223 100644 --- a/src/steps/step.lisp +++ b/src/steps/step.lisp @@ -3,8 +3,11 @@ (:shadow #:step) (:import-from #:40ants-ci/github) (:import-from #:40ants-ci/utils + #:to-env-alist #:alistp #:plistp) + (:import-from #:serapeum + #:soft-alist-of) (:export #:step #:step-id #:step-name @@ -22,25 +25,29 @@ :reader step-name) (env :initarg :env :initform nil + :type (soft-alist-of string string) + :documentation "An alist of environment variables." :reader env) (if :initarg :if :initform nil :reader step-if))) +(defmethod initialize-instance :around ((step step) &rest initargs) + (let* ((initargs (copy-list initargs)) + (env (getf initargs :env))) + (when env + (setf (getf initargs :env) + (to-env-alist env))) + + (apply #'call-next-method + step + initargs))) + + (defgeneric make-env (step) (:method ((step step)) - (let ((env (env step))) - (cond - ((plistp env) - (loop for (key value) on env by #'cddr - collect (cons (symbol-name key) - value))) - ((alistp env) - env) - (t - (error "~A is not alisp or plist" - env)))))) + (env step))) (defmethod 40ants-ci/github:prepare-data ((step step)) diff --git a/src/utils.lisp b/src/utils.lisp index 3c93aec..539e203 100644 --- a/src/utils.lisp +++ b/src/utils.lisp @@ -261,3 +261,29 @@ it will output HELLO-WORLD.\" (defmethod yason:encode ((object (eql nil)) &optional (stream yason::*json-output*)) (write-string "[]" stream)) + + +(deftype allowed-env-name-type () + '(or string keyword)) + + +(deftype env-alist-type () + '(serapeum:soft-alist-of allowed-env-name-type string)) + + +(defun to-env-alist (env) + (flet ((make-env-name (name) + (str:replace-all "-" "_" + (string-upcase name)))) + (cond + ((plistp env) + (loop for (key value) on env by #'cddr + collect (cons (make-env-name key) + value))) + ((typep env 'env-alist-type) + (loop for (key . value) in env + collect (cons (make-env-name key) + value))) + (t + (error "~A is not alist or plist" + env))))) diff --git a/src/workflow.lisp b/src/workflow.lisp index 587e69a..2638e34 100644 --- a/src/workflow.lisp +++ b/src/workflow.lisp @@ -1,16 +1,26 @@ -(defpackage #:40ants-ci/workflow +(uiop:define-package #:40ants-ci/workflow (:use #:cl) (:import-from #:40ants-ci/github #:*current-system*) (:import-from #:40ants-ci/jobs/job) (:import-from #:40ants-ci/utils + #:to-env-alist #:ensure-primary-system) (:import-from #:40ants-ci/vars) (:import-from #:alexandria #:with-output-to-file) (:import-from #:40ants-ci/jobs/job) - (:export - #:defworkflow)) + (:import-from #:serapeum + #:soft-alist-of) + (:export #:defworkflow + #:workflow-env + #:workflow + #:name + #:on-push-to + #:on-pull-request + #:by-cron + #:cache-p + #:jobs)) (in-package 40ants-ci/workflow) @@ -34,10 +44,32 @@ (cache :initform t :initarg :cache :reader cache-p) + (env :initform nil + :type (soft-alist-of string string) + :initarg :env + :documentation "An alist of environment variables and their values to be added on workflow level. Values are evaluated in runtime." + :reader workflow-env) (jobs :initform nil :initarg :jobs :reader jobs))) + +(defmethod initialize-instance :around ((workflow workflow) &rest initargs) + (let* ((initargs (copy-list initargs)) + (env (getf initargs :env))) + (when env + (setf (getf initargs :env) + (to-env-alist env))) + + (setf (getf initargs :jobs) + (mapcar #'make-job + (getf initargs :jobs))) + + (apply #'call-next-method + workflow + initargs))) + + (defmethod on-push-to :around ((workflow workflow)) (uiop:ensure-list (call-next-method))) @@ -105,6 +137,7 @@ by-cron on-pull-request cache + env jobs) (let ((make-func-name (intern (format nil "MAKE-~A-WORKFLOW" name)))) @@ -112,14 +145,14 @@ (defclass ,name (workflow) ()) (defun ,make-func-name () - (let ((jobs (mapcar #'make-job ',jobs))) - (make-instance ',name - :name ',name - :jobs jobs - :on-push-to ',(uiop:ensure-list on-push-to) - :by-cron ',(uiop:ensure-list by-cron) - :on-pull-request ,on-pull-request - :cache ,cache))) + (make-instance ',name + :name ',name + :jobs ',jobs + :env ',env + :on-push-to ',(uiop:ensure-list on-push-to) + :by-cron ',(uiop:ensure-list by-cron) + :on-pull-request ,on-pull-request + :cache ,cache)) (let ((workflow (,make-func-name) )) (register-workflow workflow) (on-workflow-redefinition workflow) @@ -145,6 +178,7 @@ (triggers (make-triggers workflow)) (jobs (uiop:ensure-list (jobs workflow))) + (env (workflow-env workflow)) (used-job-names (make-hash-table :test 'equal))) (flet ((ensure-unique (name) @@ -159,10 +193,13 @@ (append `(("name" . ,(symbol-name (name workflow)))) - + (when triggers `(("on" . ,triggers))) + (when env + `(("env" . ,env))) + (when jobs `(("jobs" . ,(loop for job in jobs From 60a197ea76f05121181f9ecd99c81648b132116f Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Sat, 2 Mar 2024 10:52:43 +0300 Subject: [PATCH 20/22] Add env argument to all job building functions. --- docs/changelog.lisp | 2 ++ src/jobs/autotag.lisp | 15 ++++++++++----- src/jobs/critic.lisp | 5 +++-- src/jobs/docs.lisp | 18 ++++++++++-------- src/jobs/run-tests.lisp | 6 ++++-- src/workflow.lisp | 4 +++- 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/docs/changelog.lisp b/docs/changelog.lisp index 5716506..5d13f77 100644 --- a/docs/changelog.lisp +++ b/docs/changelog.lisp @@ -17,12 +17,14 @@ New === * Now you can specify ENV argument to 40ANTS-CI:DEFWORKFLOW and any job. This should be an alist where keys are strings and values are evaluated during GitHub workflow generation phase. Read more in 40ANTS-CI-DOCS/INDEX::@ENV section. +* 40ANTS-CI/JOBS/AUTOTAG:AUTOTAG function now ignores TOKEN-PATTERN argument if ENV argument was given and has GITHUB_TOKEN value for whole job. Backward incompatible changes ============================= * When additional keyword arguments to 40ANTS-CI/STEPS/SH:SH function are given, they are transformed into env variables. Previously, their names were taken as is. Now they are uppercased and dash symbols are replaced with underscores. + ") (0.14.0 2024-02-25 " diff --git a/src/jobs/autotag.lisp b/src/jobs/autotag.lisp index 1652d55..0218870 100644 --- a/src/jobs/autotag.lisp +++ b/src/jobs/autotag.lisp @@ -1,6 +1,7 @@ (uiop:define-package #:40ants-ci/jobs/autotag (:use #:cl) - (:import-from #:40ants-ci/jobs/job) + (:import-from #:40ants-ci/jobs/job + #:job-env) (:import-from #:40ants-ci/steps/action #:action) (:export #:autotag @@ -48,13 +49,15 @@ (defun autotag (&key (filename *default-filename*) (regex *default-regex*) (tag-prefix *default-tag-prefix*) - (token-pattern *default-token-pattern*)) + (token-pattern *default-token-pattern*) + env) "Creates a job which will run autotagger to create a new git tag for release." (make-instance 'autotag :filename filename :regex regex :tag-prefix tag-prefix - :token-pattern token-pattern)) + :token-pattern token-pattern + :env env)) (defmethod 40ants-ci/jobs/job:steps ((job autotag)) @@ -67,6 +70,8 @@ :root (filename job) :regex_pattern (regex job) :tag_prefix (tag-prefix job) - :env (list :github_token - (token-pattern job)))) + :env (unless (assoc "GITHUB_TOKEN" (job-env job) + :test #'string=) + (list :github_token + (token-pattern job))))) (call-next-method))) diff --git a/src/jobs/critic.lisp b/src/jobs/critic.lisp index 091d75f..1ec746f 100644 --- a/src/jobs/critic.lisp +++ b/src/jobs/critic.lisp @@ -21,7 +21,7 @@ :reader ignore-critiques))) -(defun critic (&key asdf-systems asdf-version ignore-critiques) +(defun critic (&key asdf-systems asdf-version ignore-critiques env) "Creates a job which will run Lisp Critic for given ASDF systems. If argument ASDF-SYSTEMS is NIL, it will use ASDF system @@ -36,7 +36,8 @@ :asdf-system (first asdf-systems) :asdf-systems asdf-systems :asdf-version asdf-version - :ignore-critiques ignore-critiques))) + :ignore-critiques ignore-critiques + :env env))) (defmethod 40ants-ci/jobs/job:steps ((job critic)) diff --git a/src/jobs/docs.lisp b/src/jobs/docs.lisp index ff0a930..6eababc 100644 --- a/src/jobs/docs.lisp +++ b/src/jobs/docs.lisp @@ -19,14 +19,16 @@ (:documentation "Builds documentation and uploads it to GitHub using [\"40ants/build-docs\" github action](https://40ants.com/build-docs/).")) -;; (defun build-docs (&key asdf-system -;; asdf-version -;; (error-on-warnings t)) -;; "Creates a job of class BUILD-DOCS." -;; (make-instance 'build-docs -;; :asdf-system asdf-system -;; :error-on-warnings error-on-warnings -;; :asdf-version asdf-version)) +(defun build-docs (&key asdf-system + asdf-version + (error-on-warnings t) + env) + "Creates a job of class BUILD-DOCS." + (make-instance 'build-docs + :asdf-system asdf-system + :error-on-warnings error-on-warnings + :asdf-version asdf-version + :env env)) (defmethod 40ants-ci/jobs/job:steps ((job build-docs)) diff --git a/src/jobs/run-tests.lisp b/src/jobs/run-tests.lisp index 6564692..9e0131e 100644 --- a/src/jobs/run-tests.lisp +++ b/src/jobs/run-tests.lisp @@ -34,10 +34,12 @@ quicklisp lisp exclude - custom) + custom + env) "Creates a job step of class RUN-TESTS." (declare (ignore coverage qlfile os quicklisp lisp - asdf-system asdf-version exclude)) + asdf-system asdf-version exclude + env)) (check-type custom (or null string list)) (apply #'make-instance 'run-tests diff --git a/src/workflow.lisp b/src/workflow.lisp index 2638e34..762557c 100644 --- a/src/workflow.lisp +++ b/src/workflow.lisp @@ -96,7 +96,9 @@ - (foo 1 2 3) -> result of foo call. " (if (and (listp arg) - (not (symbolp (first arg)))) + (or (not (symbolp (first arg))) + ;; We don't want to eval plists + (keywordp (first arg)))) arg (eval arg))) From 0e693f24bac61c1e553b42a3fe5e5b3daf0bc298 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Sat, 2 Mar 2024 10:57:28 +0300 Subject: [PATCH 21/22] Add dependency on serapeum. --- src/utils.lisp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils.lisp b/src/utils.lisp index 539e203..363d3ee 100644 --- a/src/utils.lisp +++ b/src/utils.lisp @@ -5,6 +5,8 @@ #:join #:split) (:import-from #:yason) + (:import-from #:serapeum + #:soft-alist-of) (:export #:to-json #:ensure-primary-system @@ -268,7 +270,7 @@ it will output HELLO-WORLD.\" (deftype env-alist-type () - '(serapeum:soft-alist-of allowed-env-name-type string)) + '(soft-alist-of allowed-env-name-type string)) (defun to-env-alist (env) From 5bb242642ebb0da70c0865236850a477fe6e49a2 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Sat, 2 Mar 2024 11:02:18 +0300 Subject: [PATCH 22/22] Removed unused import. --- src/steps/sh.lisp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/steps/sh.lisp b/src/steps/sh.lisp index 12cca34..f8ed7e4 100644 --- a/src/steps/sh.lisp +++ b/src/steps/sh.lisp @@ -2,8 +2,6 @@ (:use #:cl) (:import-from #:40ants-ci/steps/step) (:import-from #:40ants-ci/github) - (:import-from #:alexandria - #:remove-from-plistf) (:import-from #:40ants-ci/utils #:dedent) (:export #:sh