Refactor workflow, job and step env argument processing.

.github/workflows/docs.yml

@@ -13,10 +13,14 @@
       }
     ]
   },
+  "env": {
+    "CUSTOM_ENV_HELLO": "Hello"
+  },
   "jobs": {
     "build-docs": {
       "runs-on": "ubuntu-latest",
       "env": {
+        "CUSTOM_ENV_WORLD": "World!",
         "OS": "ubuntu-latest",
         "QUICKLISP_DIST": "quicklisp",
         "LISP": "sbcl-bin"

docs/changelog.lisp

@@ -11,6 +11,19 @@
 			      "GITHUB_TOKEN"
                               "OSX")
                :external-docs ("https://40ants.com/40ants-asdf-system/"))
+  (0.15.0 2024-03-02
+          "
+New
+===
+
+* Now you can specify ENV argument to 40ANTS-CI:DEFWORKFLOW and any job. This should be an alist where keys are strings and values are evaluated during GitHub workflow generation phase. Read more in 40ANTS-CI-DOCS/INDEX::@ENV section.
+
+Backward incompatible changes
+=============================
+
+* When additional keyword arguments to 40ANTS-CI/STEPS/SH:SH function are given, they are transformed into env variables. Previously, their names were taken as is. Now they are uppercased and dash symbols are replaced with underscores.
+
+")
   (0.14.0 2024-02-25
           "
 Changed

docs/index.lisp

@@ -569,4 +569,48 @@ Pay attention to the NAME argument of 40ANTS-CI/JOBS/LISP-JOB:LISP-JOB class. If
 ")
 
 
+(defsection @env (:title "Adding env variables")
+  "
+You can specify additional environment variables on any level of the GitHub workflow: for workflow itself, for a job or for a step.
+
+To specify env for workflow or a job, just add an ENV argument with alist or plist value like this:
+
+```lisp
+(defworkflow release
+  :on-push-to \"master\"
+  :env (:github-token \"${{ secrets.autotag_token }}\")
+  :jobs ((40ants-ci/jobs/autotag:autotag)))
+```
+
+or as alist:
+
+```lisp
+(defworkflow release
+  :on-push-to \"master\"
+  :env ((\"github_token\" . \"${{ secrets.autotag_token }}\"))
+  :jobs ((40ants-ci/jobs/autotag:autotag)))
+```
+
+or for the job itself:
+
+```lisp
+(defworkflow release
+  :on-push-to \"master\"
+  :jobs ((40ants-ci/jobs/autotag:autotag
+           :env (:github-token \"${{ secrets.autotag_token }}\"))))
+```
+
+the same way it can be specified on a custom step:
+
+```lisp
+(40ants-ci/steps/sh:sh \"Custom env-var example\"
+                       \"echo $CUSTOM_VAR\"
+                       :env (:custom-var \"Hello world!\"))
+```
+
+Note - environment variable names are always transformed to uppercase and dashes are replaced with underscores.
+
+")
+
+
 (defautodoc @api (:system "40ants-ci"))

src/ci.lisp

@@ -24,8 +24,10 @@
   :on-pull-request t
   :by-cron "0 10 * * 1"
   :cache t
+  :env ((:custom-env-hello . "Hello"))
   :jobs ((40ants-ci/jobs/docs:build-docs
-          :asdf-system "40ants-ci-docs")))
+          :asdf-system "40ants-ci-docs"
+          :env ((:custom-env-world . "World!")))))
 
 
 (defworkflow ci
@@ -51,11 +53,12 @@
           :qlfile "{% ifequal quicklisp_dist \"ultralisp\" %}
                    dist ultralisp http://dist.ultralisp.org
                    {% endifequal %}")
-         ;; This is an example of a job with a custom
-         ;; step:
-         (40ants-ci/jobs/lisp-job:lisp-job :name "check-ros-config"
+         ;; This is an example of a job with a custom steps:
+         (40ants-ci/jobs/lisp-job:lisp-job :name "custom-steps"
                                            :lisp "ccl-bin"
                                            :steps ((40ants-ci/steps/sh:sh "Show Roswell Config"
-                                                                          "ros config")))))
-
+                                                                          "ros config")
+                                                   (40ants-ci/steps/sh:sh "Custom ENV"
+                                                                          "echo $CUSTOM_ENV"
+                                                                          :env (:custom-env "Hello world!"))))))
 

src/jobs/docs.lisp

@@ -19,14 +19,14 @@
   (:documentation "Builds documentation and uploads it to GitHub using [\"40ants/build-docs\" github action](https://40ants.com/build-docs/)."))
 
 
-(defun build-docs (&key asdf-system
-                     asdf-version
-                     (error-on-warnings t))
-  "Creates a job of class BUILD-DOCS."
-  (make-instance 'build-docs
-                 :asdf-system asdf-system
-                 :error-on-warnings error-on-warnings
-                 :asdf-version asdf-version))
+;; (defun build-docs (&key asdf-system
+;;                      asdf-version
+;;                      (error-on-warnings t))
+;;   "Creates a job of class BUILD-DOCS."
+;;   (make-instance 'build-docs
+;;                  :asdf-system asdf-system
+;;                  :error-on-warnings error-on-warnings
+;;                  :asdf-version asdf-version))
 
 
 (defmethod 40ants-ci/jobs/job:steps ((job build-docs))

src/jobs/job.lisp

@@ -1,9 +1,11 @@
 (uiop:define-package #:40ants-ci/jobs/job
   (:use #:cl)
   (:import-from #:40ants-ci/utils
+                #:to-env-alist
                 #:ensure-list-of-plists)
   (:import-from #:40ants-ci/github)
   (:import-from #:serapeum
+                #:soft-alist-of
                 #:length<)
   (:import-from #:alexandria
                 #:length=)
@@ -19,7 +21,8 @@
            #:permissions
            #:make-permissions
            #:explicit-steps
-           #:exclude))
+           #:exclude
+           #:job-env))
 (in-package #:40ants-ci/jobs/job)
 
 
@@ -34,6 +37,11 @@
             :initarg :exclude
             :reader exclude
             :documentation "A list of plists denoting matrix combinations to be excluded.")
+   (env :initform nil
+        :type (soft-alist-of string string)
+        :initarg :env
+        :documentation "An alist of environment variables and their values to be added on job level. Values are evaluated in runtime."
+        :reader job-env)
    (steps :initform nil
           :initarg :steps
           :documentation "This slot holds steps given as a STEPS argument to a job constructor. Depending on a job class, it might add additional steps around these explicit steps."
@@ -52,17 +60,26 @@
                 :reader permissions)))
 
 
-(defmethod initialize-instance :after ((job job) &rest initargs)
-  (declare (ignore initargs))
-
-  (unless (slot-boundp job 'name)
-    (setf (slot-value job 'name)
-          (string-downcase
-           (class-name (class-of job)))))
-
-  (setf (slot-value job 'steps)
-        (mapcar #'ensure-step
-                (slot-value job 'steps))))
+(defmethod initialize-instance :around ((job job) &rest initargs)
+  (let* ((initargs (copy-list initargs))
+         (env (getf initargs :env)))
+    (when env
+      (setf (getf initargs :env)
+            (to-env-alist env)))
+
+    (unless (getf initargs :name)
+      (setf (getf initargs :name)
+            (string-downcase
+             (class-name (class-of job)))))
+
+
+    (setf (getf initargs :steps)
+          (mapcar #'ensure-step
+                  (getf initargs :steps)))
+
+    (apply #'call-next-method
+           job
+           initargs)))
 
 
 (defmethod os :around ((job job))
@@ -103,6 +120,9 @@
 (defgeneric make-env (job)
   (:method ((job job))
     (append
+     (when (job-env job)
+       (job-env job))
+
      (cond
        ((length< 1 (os job))
         `(("OS" . "${{ matrix.os }}")))

src/jobs/linter.lisp

@@ -32,7 +32,7 @@
        (current-system-name))))
 
 
-(defun linter (&key asdf-systems asdf-version check-imports)
+(defun linter (&key asdf-systems asdf-version check-imports env)
   "Creates a job which will run SBLint for given ASDF systems.
 
    If no ASD files given, it will use all ASD files from
@@ -41,6 +41,7 @@
                  :asdf-system (first asdf-systems)
                  :asdf-systems asdf-systems
                  :asdf-version asdf-version
+                 :env env
                  :check-imports check-imports))
 
 

src/steps/sh.lisp

@@ -25,12 +25,7 @@
 
 
 ;; ignore-critiques: if-no-else
-(defun sh (name command &rest env &key
-                                    id
-                                    if
-                                    (shell *default-shell*)
-           &allow-other-keys)
-  (remove-from-plistf env :id :if :shell)
+(defun sh (name command &key id if (shell *default-shell*) env)
   (make-instance 'sh
                  :name name
                  :command command

src/steps/step.lisp

@@ -3,8 +3,11 @@
   (:shadow #:step)
   (:import-from #:40ants-ci/github)
   (:import-from #:40ants-ci/utils
+                #:to-env-alist
                 #:alistp
                 #:plistp)
+  (:import-from #:serapeum
+                #:soft-alist-of)
   (:export #:step
            #:step-id
            #:step-name
@@ -22,25 +25,29 @@
          :reader step-name)
    (env :initarg :env
         :initform nil
+        :type (soft-alist-of string string)
+        :documentation "An alist of environment variables."
         :reader env)
    (if :initarg :if
        :initform nil
        :reader step-if)))
 
 
+(defmethod initialize-instance :around ((step step) &rest initargs)
+  (let* ((initargs (copy-list initargs))
+         (env (getf initargs :env)))
+    (when env
+      (setf (getf initargs :env)
+            (to-env-alist env)))
+
+    (apply #'call-next-method
+           step
+           initargs)))
+
+
 (defgeneric make-env (step)
   (:method ((step step))
-    (let ((env (env step)))
-      (cond
-        ((plistp env)
-         (loop for (key value) on env by #'cddr
-               collect (cons (symbol-name key)
-                             value)))
-        ((alistp env)
-         env)
-        (t
-         (error "~A is not alisp or plist"
-                env))))))
+    (env step)))
 
 
 (defmethod 40ants-ci/github:prepare-data ((step step))

src/utils.lisp

@@ -261,3 +261,29 @@ it will output HELLO-WORLD.\"
 
 (defmethod yason:encode ((object (eql nil)) &optional (stream yason::*json-output*))
   (write-string "[]" stream))
+
+
+(deftype allowed-env-name-type ()
+  '(or string keyword))
+
+
+(deftype env-alist-type ()
+  '(serapeum:soft-alist-of allowed-env-name-type string))
+
+
+(defun to-env-alist (env)
+  (flet ((make-env-name (name)
+           (str:replace-all "-" "_"
+                            (string-upcase name))))
+    (cond
+      ((plistp env)
+       (loop for (key value) on env by #'cddr
+             collect (cons (make-env-name key)
+                           value)))
+      ((typep env 'env-alist-type)
+       (loop for (key . value) in env
+             collect (cons (make-env-name key)
+                           value)))
+      (t
+       (error "~A is not alist or plist"
+              env)))))