Support running local pipeline

deps.edn

@@ -3,8 +3,8 @@
  :deps
  {org.clojure/clojure  {:mvn/version "1.11.0"}
   nostromo/nostromo    {:git/url "https://github.com/nosana-ci/nostromo"
-                        :sha     "f5e8f268806282ec6283bd4e9cfdd5a297ec9033"}
-  ;; nostromo/nostromo    {:local/root "../nostromo"}
+                        :sha     "641fc912df5ace0da364e7ad2d26341d4b126f3c"}
+  nostromo/nostromo    {:local/root "../nostromo"}
 
   org.clojure/tools.cli {:mvn/version "1.0.214"}
 

src/nosana_node/main.clj

@@ -3,6 +3,7 @@
             nosana-node.handler
             nos.ops.git
             [nosana-node.system :refer [start-system use-jetty use-when] :as nos-sys]
+            [nosana-node.pipeline :as pipeline]
             nosana-node.gitlab
             nos.ops.docker
             [nos.store :as store]
@@ -35,6 +36,8 @@
     :parse-fn #(Integer/parseInt %)
     :validate [#(< 0 % 0x10000) "Must be a number between 0 and 65536"]]
    [nil "--ipfs-url URL" "IPFS url"]
+   [nil "--podman URI" "Podman connection URI"
+    :id :podman-conn-uri]
    ["-v" nil "Verbosity level"
     :id :verbosity
     :default 0
@@ -59,23 +62,33 @@
 (defn use-cli
   "Parse CLI arguments using `tools.cli` and add to system map."
   [{:keys [cli-args] :as sys}]
-  (let [{:keys [errors options arguments summary]}
+
+  (let [{:keys [errors options arguments summary] :as res}
         (cli/parse-opts cli-args cli-options)
 
         state (cond
                 (:help options)
-                {:exit-message (usage summary :ok? true)}
+                {:exit-message (usage summary) :ok? true}
                 ;; if action is pipeline we don't run a server
-                (= "pipeline" (first arguments)) (assoc options :run-server? false)
+                (= "pipeline" (first arguments))
+                (assoc options :run-server? false)
                 :else options)]
     (if (:exit-message state)
-      (do (println (:exit-message state))
-          (System/exit (if (:ok? state) 0 1)))
-      (merge sys state))))
+      (do
+        (println (:exit-message state))
+        (System/exit (if (:ok? state) 0 1)))
+      (do
+        (assoc options :run-server? false)
+        (update sys :nos/vault merge state)))))
 
-(defn -main [& args]
-  (reset! system {:cli-args args})
+(defn use-pipeline [system]
+  (log/set-min-level! :error)
+  (let [conf (nosana/make-config system)
+        dir  (System/getProperty "user.dir")]
+    (pipeline/run-local-pipeline dir (:nos/vault system))
+    (println "=> Done running pipeline!")))
 
+(defn -main [& args]
   (start-system
    system
    {:http/handler      #'nos-sys/handler
@@ -86,8 +99,11 @@
                                   use-nostromo
                                   use-nosana
                                   nos-sys/use-wrap-ctx
-                                  use-jetty)]
+                                  use-jetty)
+                        (use-when #(not (:run-server? %))
+                                  use-pipeline)]
     :system/profile    :prod
+    :cli-args          args
     :nos/log-dir       "/tmp/logs"
     :nos/store-path    "/tmp/store"
     :nos/vault-path    (io/resource "config.edn")}))

src/nosana_node/pipeline.clj

@@ -2,7 +2,10 @@
   (:require [clojure.edn :as edn]
             [taoensso.timbre :refer [log]]
             [clj-http.client :as http]
+            [clojure.java.shell :as sh]
             [cheshire.core :as json]
+            [clojure.core.async :refer [chan >!! <!! put!]]
+            [konserve.memory :refer [new-mem-store]]
             [nosana-node.nosana :refer [finish-flow create-flow ipfs-upload]]
             [clojure.java.io :as io]
             [nos.core :as nos]
@@ -44,6 +47,7 @@
        (map (fn [[k v]]
               (case (:type v)
                 "nosana/secret" [k [:nosana/secret (:endpoint v) (:value v)]]
+                "nos/vault"     [k [:nos/vault (:value v)]]
                 "string"        [k v]
                 [k v])))
        (into {"TERM"        "xterm-color"
@@ -130,6 +134,50 @@
         nos/build
         (assoc :default-args (:nos-default-args conf)))))
 
+(defn make-local-git-artifact! [dir artifact-name flow-id]
+  (println "Creating artifact " )
+  (let [artifact               (str "/tmp/nos-artifacts/" flow-id "/" artifact-name)
+        _                      (io/make-parents artifact)
+        stash                  (-> (sh/sh "git" "-C" dir "stash" "create")
+                                   :out
+                                   (string/replace "\n" ""))
+        _                      (println "Created stash " stash " for working directory")
+        {:keys [err out exit]} (sh/sh "git"
+                                      "-C" dir
+                                      "archive"
+                                      "--prefix" "project/"
+                                      "--format" "tar"
+                                      "--output" artifact
+                                      stash)]
+    (cond (pos? exit) (throw (ex-info "Failed to make artifact" {:err err}))
+          :else
+          true)))
+
+(defn get-git-sha [dir]
+  (-> (sh/sh "git" "-C" dir "rev-parse" "HEAD")
+      :out
+      (string/replace "\n" "")))
+
+(defn run-local-pipeline [dir vault]
+  (let [yaml     (slurp (str dir "/.nosana-ci.yml"))
+        commit   (get-git-sha dir)
+        pipeline (-> yaml yaml/parse-string)
+        flow
+        (-> {:ops []}
+            (update :ops #(into [] (concat % (pipeline->flow-ops pipeline))))
+            (assoc-in [:state :input/commit-sha] commit)
+            nos/build
+            (assoc :default-args  {:container/run
+                                   {:conn         {:uri [:nos/vault :podman-conn-uri]}
+                                    :inline-logs? true
+                                    :stdout?      true}}))]
+    (println "Flow ID is " (:id flow))
+    (make-local-git-artifact! dir "checkout" (:id flow))
+    (let [flow-engine {:store     (<!! (new-mem-store))
+                       :chan      (chan)
+                       :nos/vault vault}]
+      (nos/run-flow! flow-engine flow))))
+
 (defmethod finish-flow "Pipeline" [flow conf]
   (let [results    (:state flow)
         job-result {:nos-id      (:id flow)

src/nosana_node/system.clj

@@ -17,7 +17,7 @@
 (defn use-when [f & components]
   (fn [sys]
     (if (f sys)
-      (update sys :biff/components #(concat components %))
+      (update sys :system/components #(concat components %))
       sys)))
 
 (defn start-system [system-atom init]