Create multi-yml actions config read and OS packages

.github/workflows/pal-ci.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Run Linters
         run: |
-          make install-linters
+          make install-deps
           echo
           make lint
 

.gitignore

@@ -251,8 +251,6 @@ modules.xml
 ### WebStorm+iml Patch ###
 # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
 
-
-
 # Created by https://www.gitignore.io/api/macos
 # Edit at https://www.gitignore.io/?templates=macos
 
@@ -291,4 +289,7 @@ upload/*
 !upload/.gitkeep
 *.pem
 *.key
-vendor
+vendor
+*.deb
+*.rpm
+.vagrant

Dockerfile

@@ -10,10 +10,25 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-COPY pal pal.yml ./entrypoint.sh ./test/pal-actions.yml localhost.key localhost.pem /pal/
+COPY pal pal.yml ./entrypoint.sh localhost.key localhost.pem /pal/
+COPY ./test/*.yml /pal/actions/
+
+RUN sed -i "s|listen:.*|listen: 0.0.0.0:8443|" /pal/pal.yml && \
+    sed -i "s|  key:.*|  key: /pal/localhost.key|" /pal/pal.yml && \
+    sed -i "s|cert:.*|cert: /pal/localhost.pem|" /pal/pal.yml && \
+    sed -i "s|upload_dir:.*|upload_dir: /pal/upload|" /pal/pal.yml && \
+    sed -i "s|path:.*|path: /pal/pal.db|" /pal/pal.yml && \
+    sed -i "s|debug:.*|debug: false|" /pal/pal.yml && \
+    mkdir -p /pal/pal.db /pal/upload
 
 WORKDIR /pal
 
+RUN addgroup --gid 101010 --system pal && \
+    adduser --uid 101010 --system --ingroup pal --home /pal --shell /sbin/nologin --comment "pal Service Account" pal && \
+    chown -Rf pal:pal /pal
+
+USER pal
+
 EXPOSE 8443
 
 CMD ["/pal/entrypoint.sh"]

Makefile

@@ -4,7 +4,9 @@ PACKAGES:=$(shell go list ./... | grep -v /vendor/)
 BUILT_ON := $(shell date -u)
 COMMIT_HASH:=$(shell git log -n 1 --pretty=format:"%H")
 GO_LINUX := GOOS=linux GOARCH=amd64
-LDFLAGS := '-s -w -X "main.builtOn=$(BUILT_ON)" -X "main.commitHash=$(COMMIT_HASH)"'
+GO_ARM := GOOS=linux GOARCH=arm64
+VERSION := 1.0.0
+LDFLAGS := '-s -w -X "main.builtOn=$(BUILT_ON)" -X "main.commitHash=$(COMMIT_HASH)" -X "main.version=$(VERSION)"'
 
 
 .PHONY: test
@@ -17,9 +19,19 @@ build:
 linux:
 	CGO_ENABLED=0 $(GO_LINUX) go build -a -installsuffix cgo -o $(MAIN_PACKAGE) -ldflags $(LDFLAGS) .
 
+arm64:
+	CGO_ENABLED=0 $(GO_ARM) go build -a -installsuffix cgo -o $(MAIN_PACKAGE) -ldflags $(LDFLAGS) .
+
 clean:
 	find . -name *_gen.go -type f -delete
 	rm -f ./$(MAIN_PACKAGE)
+	rm -f ./localhost.*
+	rm -f ./*.deb
+	rm -f ./*.rpm
+
+cleanall: clean
+	docker rm -f pal || true
+	rm -rf ./pal.db
 
 fmt:
 	go fmt ./...
@@ -28,30 +40,44 @@ lint: fmt
 	$(GOPATH)/bin/staticcheck $(PACKAGES)
 	$(GOPATH)/bin/golangci-lint run
 	$(GOPATH)/bin/gosec -quiet -no-fail ./...
+	if command -v shellcheck; then find . -name "*.sh" -type f -exec shellcheck {} \;; fi
 
 run:
-	./pal -c ./pal.yml -a ./test/pal-actions.yml
+	./pal -c ./pal.yml -d ./test
 
 test:
 	./test/test.sh
 
-install-linters:
+install-deps:
 	go install honnef.co/go/tools/cmd/[email protected]
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.61.0
 	curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b $(GOPATH)/bin v2.21.4
+	go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest
 
 update-deps:
 	go get -u ./...
 	go mod tidy
 
 certs:
-	openssl req -x509 -newkey rsa:4096 -nodes -keyout localhost.key -out localhost.pem -days 365 -sha256 -subj '/CN=localhost' -addext 'subjectAltName=IP:127.0.0.1'
+	openssl req -x509 -newkey rsa:4096 -nodes -keyout localhost.key -out localhost.pem -days 365 -sha256 -subj '/CN=localhost' -addext "subjectAltName=IP:127.0.0.1,DNS:localhost"
 
 docker:
 	sudo docker build -t pal:latest .
 	sudo docker rm -f pal || true
 	mkdir -p ./pal.db
-	sudo docker run -d --name=pal -p 8443:8443 -e HTTP_LISTEN='0.0.0.0:8443' \
-	-v $(PWD)/upload:/pal/upload:rw -v $(PWD)/pal.db:/pal/pal.db:rw \
+	sudo docker run -d --name=pal -p 8443:8443 -e HTTP_UI_BASIC_AUTH='admin p@LLy' \
+	-e HTTP_AUTH_HEADER='X-Pal-Auth PaLLy!@#890-' -e HTTP_SESSION_SECRET='P@llY^S3$$h' -e DB_ENCRYPT_KEY='8c755319-fd2a-4a89-b0d9-ae7b8d26' \
 	--health-cmd 'curl -sfk https://127.0.0.1:8443/v1/pal/health || exit 1' --restart=unless-stopped pal:latest
 
+pkg: linux
+	VERSION=$(VERSION) ARCH=amd64 nfpm pkg --packager deb --target ./
+	VERSION=$(VERSION) ARCH=amd64 nfpm pkg --packager rpm --target ./
+	$(MAKE) arm64
+	VERSION=$(VERSION) ARCH=arm64 nfpm pkg --packager deb --target ./
+	VERSION=$(VERSION) ARCH=arm64 nfpm pkg --packager rpm --target ./
+
+vagrant: pkg
+	vagrant destroy -f || true
+	vagrant up
+	sleep 10
+	$(MAKE) test

README.md

@@ -9,6 +9,8 @@
 - [Quick Start](#quick-start)
   - [Local Development](#local-development)
   - [Docker](#docker)
+  - [Vagrant](#vagrant)
+  - [DEB & RPM Builds](#deb--rpm-builds)
 - [YAML Definitions Configuration](#yaml-definitions-configuration)
 - [API Endpoints](#api-endpoints)
   - [Command Execution](#command-execution)
@@ -23,7 +25,7 @@
   - [Env Variables](#env-variables)
   - [Notification Variables](#notification-variables)
 - [YAML Server Configurations](#yaml-server-configurations)
-- [Example `pal-actions.yml`](#example-pal-actionsyml)
+- [Example Action Definition YML](#example-action-definition-yml)
 
 ## Use Cases
 
@@ -55,28 +57,61 @@
 ```bash
 make
 make certs
-./pal -c ./pal.yml -d ./test/pal-actions.yml
+./pal -c ./pal.yml -d ./test
 ```
 
 ### Docker
 
+#### Run default insecure test configs
+
 ```bash
 make linux
 make certs
-make docker # Default configurations
+# Default insecure test configurations
+make docker
+```
+
+#### Generate random secrets for one-time use
+
+```bash
+sudo docker run -d --name=pal -p 8443:8443 \
+	--health-cmd 'curl -sfk https://127.0.0.1:8443/v1/pal/health || exit 1' --restart=unless-stopped pal:latest
+
+# See generated random secrets
+sudo docker logs pal
 ```
 
 **Available Docker Run Env Variables:**
 
 ```bash
-# Default values
+# Default insecure test values
 -e HTTP_LISTEN="127.0.0.1:8443"
 -e HTTP_TIMEOUT_MIN="10"
 -e HTTP_BODY_LIMIT="12M"
 -e HTTP_CORS_ALLOW_ORIGINS='["*"]'
 -e HTTP_AUTH_HEADER='X-Pal-Auth PaLLy!@#890-'
 -e HTTP_UI_BASIC_AUTH='admin p@LLy5'
+-e HTTP_SESSION_SECRET='P@llY^S3$$h'
 -e DB_ENCRYPT_KEY='8c755319-fd2a-4a89-b0d9-ae7b8d26'
+-e GLOBAL_DEBUG='true'
+```
+
+### Vagrant
+
+```bash
+# Need nfpm to build RPMs / Debs
+make install-deps
+make vagrant
+# If you want to ignore debs/rpm builds and installs just run:
+# vagrant up
+```
+
+### DEB & RPM Builds
+
+```bash
+# Need nfpm to build RPM / DEB files
+make install-deps
+make pkg
 ```
 
 **Default Access:** `https://127.0.0.1:8443` (See [Configurations](#configurations) to customize)
@@ -171,7 +206,6 @@ DELETE             /v1/pal/db/delete?key={{ key_name }}
 
 ```bash
 curl -vsk -H'X-Pal-Auth: PaLLy!@#890-' -XPUT -d 'pal' 'https://127.0.0.1:8443/v1/pal/db/put?key=name'
-
 ```
 
 ### Health Check
@@ -250,10 +284,10 @@ GET /v1/pal/action?group={{ group }}&action={{ action }}
 
 ```
 Usage: pal [options] <args>
-  -a,	Set action definitions file path location, default is ./pal-actions.yml
   -c,	Set configuration file path location, default is ./pal.yml
+  -d,	Set action definitions file directory location, default is ./actions
 
-Example: pal -a ./pal-actions.yml -c ./pal.yml
+Example: pal -c ./pal.yml -d ./actions
 ```
 
 ## Built-In Variables
@@ -283,6 +317,7 @@ Every cmd run includes the below built-in env variables.
 ```
 
 ### Notification Variables
+
 When `OnError.Notification` is configured for the action, you can use available substitution variables in the notification message:
 
 `$PAL_GROUP` - Group name
@@ -297,7 +332,7 @@ When `OnError.Notification` is configured for the action, you can use available
 
 **See latest example reference, here:** [https://github.com/marshyski/pal/blob/main/pal.yml](https://github.com/marshyski/pal/blob/main/pal.yml)
 
-## Example `pal-actions.yml`
+## Example Action Definition YML
 
 ```yaml
 monitor:
@@ -336,4 +371,6 @@ monitor:
 curl -sk -H'X-Monitor-System: q1w2e3r4t5' 'https://127.0.0.1:8443/v1/pal/run/monitor/system'
 ```
 
-**For a more complete example, see:** [https://github.com/marshyski/pal/blob/main/test/pal-actions.yml](https://github.com/marshyski/pal/blob/main/test/pal-actions.yml)
+All actions can be defined in one file, or split into multiple `.yml` files. The `-d` CLI argument tells the program what directory to verify valid action yml files.
+
+**For more complete examples, see:** [https://github.com/marshyski/pal/tree/main/test](https://github.com/marshyski/pal/tree/main/test)

Vagrantfile

@@ -0,0 +1,52 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+    config.vm.box = "debian/bookworm64"
+    config.vm.hostname = "debian12"
+    config.vm.provider "virtualbox" do |v|
+        v.name = "debian12"
+        v.memory = 2048
+        v.cpus = 1
+        v.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
+        v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+        v.customize ["modifyvm", :id, "--uartmode1", "file", File::NULL]
+    end
+    config.vm.network "forwarded_port", guest: 8443, host: 8443
+    config.vm.synced_folder ".", "/vagrant", SharedFoldersEnableSymlinksCreate: true
+    config.vm.provision "shell", inline: <<-SHELL
+# Setup Base Packages
+ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get update && \
+    apt-get upgrade -y && \
+    apt-get dist-upgrade -y && \
+    apt-get install -y --no-install-recommends \
+        curl \
+        ca-certificates \
+        jq && \
+    apt-get clean
+
+# Setup pal
+dpkg -i /vagrant/pal*amd64.deb
+
+# Create Self-Signed Certs
+cd /pal
+openssl req -x509 -newkey rsa:4096 -nodes -keyout /pal/localhost.key -out /pal/localhost.pem -days 365 -sha256 -subj '/CN=localhost' -addext "subjectAltName=IP:127.0.0.1,DNS:localhost"
+chown -Rf pal:pal /pal
+
+# Copy Insecure Test Configs
+cp -f /vagrant/pal.yml /pal/
+cp -f /vagrant/test/*.yml /pal/actions/
+
+# Configure Paths for /pal In pal.yml
+sed -i "s|listen:.*|listen: 0.0.0.0:8443|" /pal/pal.yml
+sed -i "s|  key:.*|  key: /pal/localhost.key|" /pal/pal.yml
+sed -i "s|cert:.*|cert: /pal/localhost.pem|" /pal/pal.yml
+sed -i "s|upload_dir:.*|upload_dir: /pal/upload|" /pal/pal.yml
+sed -i "s|path:.*|path: /pal/pal.db|" /pal/pal.yml
+
+# Run pal Systemd Service
+systemctl daemon-reload
+systemctl enable pal
+systemctl restart pal
+SHELL
+end

config/config.go

@@ -17,18 +17,54 @@ var (
 	configMap = cmap.New()
 )
 
-func ValidateDefs(res map[string][]data.ActionData) {
+func validateDefs(res map[string][]data.ActionData) bool {
 	validate := validator.New(validator.WithRequiredStructEnabled())
 
 	for _, v := range res {
 		for _, e := range v {
 			err := validate.Struct(e)
 			if err != nil {
 				log.Println(err)
-				log.Fatalln("error panic definitions are invalid")
+				return false
 			}
 		}
 	}
+
+	return true
+}
+
+func ReadConfig(dir string) map[string][]data.ActionData {
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		log.Fatalln("Error reading directory:", err)
+	}
+
+	groups := make(map[string][]data.ActionData)
+
+	for _, file := range files {
+		if filepath.Ext(file.Name()) == ".yml" {
+			fileLoc := filepath.Clean(dir + "/" + file.Name())
+			defs, err := os.ReadFile(fileLoc)
+			if err != nil {
+				log.Println("Error reading file:", err)
+				continue // Skip to the next file
+			}
+
+			var groupData map[string][]data.ActionData
+			err = yaml.Unmarshal(defs, &groupData)
+			if err != nil {
+				log.Println("Error unmarshaling YAML:", err)
+				continue
+			}
+
+			if validateDefs(groupData) {
+				for k, v := range groupData {
+					groups[k] = v
+				}
+			}
+		}
+	}
+	return groups
 }
 
 func InitConfig(location string) error {