From f38a0640427cc807abc350d2ec6496b13403cd7c Mon Sep 17 00:00:00 2001 From: marshyski Date: Sun, 20 Oct 2024 00:50:12 -0400 Subject: [PATCH] Create multi-yml actions config read and OS packages --- .github/workflows/pal-ci.yml | 2 +- .gitignore | 7 +-- Dockerfile | 17 +++++- Makefile | 38 +++++++++++--- README.md | 55 +++++++++++++++---- Vagrantfile | 52 ++++++++++++++++++ config/config.go | 40 +++++++++++++- entrypoint.sh | 30 ++++++++++- main.go | 31 ++++------- nfpm.yaml | 84 ++++++++++++++++++++++++++++++ pal.service | 8 +-- pkg-install.sh | 32 ++++++++++++ test/build.yml | 15 ++++++ test/html.yml | 15 ++++++ test/json.yml | 29 +++++++++++ test/{pal-actions.yml => test.yml} | 62 ---------------------- test/vagrant.yml | 8 +++ 17 files changed, 414 insertions(+), 111 deletions(-) create mode 100644 Vagrantfile create mode 100644 nfpm.yaml create mode 100755 pkg-install.sh create mode 100644 test/build.yml create mode 100644 test/html.yml create mode 100644 test/json.yml rename test/{pal-actions.yml => test.yml} (63%) create mode 100644 test/vagrant.yml diff --git a/.github/workflows/pal-ci.yml b/.github/workflows/pal-ci.yml index 527abf3..11ff3c9 100644 --- a/.github/workflows/pal-ci.yml +++ b/.github/workflows/pal-ci.yml @@ -25,7 +25,7 @@ jobs: - name: Run Linters run: | - make install-linters + make install-deps echo make lint diff --git a/.gitignore b/.gitignore index 363c67a..7aeb279 100644 --- a/.gitignore +++ b/.gitignore @@ -251,8 +251,6 @@ modules.xml ### WebStorm+iml Patch ### # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 - - # Created by https://www.gitignore.io/api/macos # Edit at https://www.gitignore.io/?templates=macos @@ -291,4 +289,7 @@ upload/* !upload/.gitkeep *.pem *.key -vendor \ No newline at end of file +vendor +*.deb +*.rpm +.vagrant \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 85206d2..b7ae332 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,25 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -COPY pal pal.yml ./entrypoint.sh ./test/pal-actions.yml localhost.key localhost.pem /pal/ +COPY pal pal.yml ./entrypoint.sh localhost.key localhost.pem /pal/ +COPY ./test/*.yml /pal/actions/ + +RUN sed -i "s|listen:.*|listen: 0.0.0.0:8443|" /pal/pal.yml && \ + sed -i "s| key:.*| key: /pal/localhost.key|" /pal/pal.yml && \ + sed -i "s|cert:.*|cert: /pal/localhost.pem|" /pal/pal.yml && \ + sed -i "s|upload_dir:.*|upload_dir: /pal/upload|" /pal/pal.yml && \ + sed -i "s|path:.*|path: /pal/pal.db|" /pal/pal.yml && \ + sed -i "s|debug:.*|debug: false|" /pal/pal.yml && \ + mkdir -p /pal/pal.db /pal/upload WORKDIR /pal +RUN addgroup --gid 101010 --system pal && \ + adduser --uid 101010 --system --ingroup pal --home /pal --shell /sbin/nologin --comment "pal Service Account" pal && \ + chown -Rf pal:pal /pal + +USER pal + EXPOSE 8443 CMD ["/pal/entrypoint.sh"] diff --git a/Makefile b/Makefile index 6dcbe34..8fae72e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,9 @@ PACKAGES:=$(shell go list ./... | grep -v /vendor/) BUILT_ON := $(shell date -u) COMMIT_HASH:=$(shell git log -n 1 --pretty=format:"%H") GO_LINUX := GOOS=linux GOARCH=amd64 -LDFLAGS := '-s -w -X "main.builtOn=$(BUILT_ON)" -X "main.commitHash=$(COMMIT_HASH)"' +GO_ARM := GOOS=linux GOARCH=arm64 +VERSION := 1.0.0 +LDFLAGS := '-s -w -X "main.builtOn=$(BUILT_ON)" -X "main.commitHash=$(COMMIT_HASH)" -X "main.version=$(VERSION)"' .PHONY: test @@ -17,9 +19,19 @@ build: linux: CGO_ENABLED=0 $(GO_LINUX) go build -a -installsuffix cgo -o $(MAIN_PACKAGE) -ldflags $(LDFLAGS) . +arm64: + CGO_ENABLED=0 $(GO_ARM) go build -a -installsuffix cgo -o $(MAIN_PACKAGE) -ldflags $(LDFLAGS) . + clean: find . -name *_gen.go -type f -delete rm -f ./$(MAIN_PACKAGE) + rm -f ./localhost.* + rm -f ./*.deb + rm -f ./*.rpm + +cleanall: clean + docker rm -f pal || true + rm -rf ./pal.db fmt: go fmt ./... @@ -28,30 +40,44 @@ lint: fmt $(GOPATH)/bin/staticcheck $(PACKAGES) $(GOPATH)/bin/golangci-lint run $(GOPATH)/bin/gosec -quiet -no-fail ./... + if command -v shellcheck; then find . -name "*.sh" -type f -exec shellcheck {} \;; fi run: - ./pal -c ./pal.yml -a ./test/pal-actions.yml + ./pal -c ./pal.yml -d ./test test: ./test/test.sh -install-linters: +install-deps: go install honnef.co/go/tools/cmd/staticcheck@2024.1.1 curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.61.0 curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b $(GOPATH)/bin v2.21.4 + go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest update-deps: go get -u ./... go mod tidy certs: - openssl req -x509 -newkey rsa:4096 -nodes -keyout localhost.key -out localhost.pem -days 365 -sha256 -subj '/CN=localhost' -addext 'subjectAltName=IP:127.0.0.1' + openssl req -x509 -newkey rsa:4096 -nodes -keyout localhost.key -out localhost.pem -days 365 -sha256 -subj '/CN=localhost' -addext "subjectAltName=IP:127.0.0.1,DNS:localhost" docker: sudo docker build -t pal:latest . sudo docker rm -f pal || true mkdir -p ./pal.db - sudo docker run -d --name=pal -p 8443:8443 -e HTTP_LISTEN='0.0.0.0:8443' \ - -v $(PWD)/upload:/pal/upload:rw -v $(PWD)/pal.db:/pal/pal.db:rw \ + sudo docker run -d --name=pal -p 8443:8443 -e HTTP_UI_BASIC_AUTH='admin p@LLy' \ + -e HTTP_AUTH_HEADER='X-Pal-Auth PaLLy!@#890-' -e HTTP_SESSION_SECRET='P@llY^S3$$h' -e DB_ENCRYPT_KEY='8c755319-fd2a-4a89-b0d9-ae7b8d26' \ --health-cmd 'curl -sfk https://127.0.0.1:8443/v1/pal/health || exit 1' --restart=unless-stopped pal:latest +pkg: linux + VERSION=$(VERSION) ARCH=amd64 nfpm pkg --packager deb --target ./ + VERSION=$(VERSION) ARCH=amd64 nfpm pkg --packager rpm --target ./ + $(MAKE) arm64 + VERSION=$(VERSION) ARCH=arm64 nfpm pkg --packager deb --target ./ + VERSION=$(VERSION) ARCH=arm64 nfpm pkg --packager rpm --target ./ + +vagrant: pkg + vagrant destroy -f || true + vagrant up + sleep 10 + $(MAKE) test diff --git a/README.md b/README.md index beb5f65..d92e8ac 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ - [Quick Start](#quick-start) - [Local Development](#local-development) - [Docker](#docker) + - [Vagrant](#vagrant) + - [DEB & RPM Builds](#deb--rpm-builds) - [YAML Definitions Configuration](#yaml-definitions-configuration) - [API Endpoints](#api-endpoints) - [Command Execution](#command-execution) @@ -23,7 +25,7 @@ - [Env Variables](#env-variables) - [Notification Variables](#notification-variables) - [YAML Server Configurations](#yaml-server-configurations) -- [Example `pal-actions.yml`](#example-pal-actionsyml) +- [Example Action Definition YML](#example-action-definition-yml) ## Use Cases @@ -55,28 +57,61 @@ ```bash make make certs -./pal -c ./pal.yml -d ./test/pal-actions.yml +./pal -c ./pal.yml -d ./test ``` ### Docker +#### Run default insecure test configs + ```bash make linux make certs -make docker # Default configurations +# Default insecure test configurations +make docker +``` + +#### Generate random secrets for one-time use + +```bash +sudo docker run -d --name=pal -p 8443:8443 \ + --health-cmd 'curl -sfk https://127.0.0.1:8443/v1/pal/health || exit 1' --restart=unless-stopped pal:latest + +# See generated random secrets +sudo docker logs pal ``` **Available Docker Run Env Variables:** ```bash -# Default values +# Default insecure test values -e HTTP_LISTEN="127.0.0.1:8443" -e HTTP_TIMEOUT_MIN="10" -e HTTP_BODY_LIMIT="12M" -e HTTP_CORS_ALLOW_ORIGINS='["*"]' -e HTTP_AUTH_HEADER='X-Pal-Auth PaLLy!@#890-' -e HTTP_UI_BASIC_AUTH='admin p@LLy5' +-e HTTP_SESSION_SECRET='P@llY^S3$$h' -e DB_ENCRYPT_KEY='8c755319-fd2a-4a89-b0d9-ae7b8d26' +-e GLOBAL_DEBUG='true' +``` + +### Vagrant + +```bash +# Need nfpm to build RPMs / Debs +make install-deps +make vagrant +# If you want to ignore debs/rpm builds and installs just run: +# vagrant up +``` + +### DEB & RPM Builds + +```bash +# Need nfpm to build RPM / DEB files +make install-deps +make pkg ``` **Default Access:** `https://127.0.0.1:8443` (See [Configurations](#configurations) to customize) @@ -171,7 +206,6 @@ DELETE /v1/pal/db/delete?key={{ key_name }} ```bash curl -vsk -H'X-Pal-Auth: PaLLy!@#890-' -XPUT -d 'pal' 'https://127.0.0.1:8443/v1/pal/db/put?key=name' - ``` ### Health Check @@ -250,10 +284,10 @@ GET /v1/pal/action?group={{ group }}&action={{ action }} ``` Usage: pal [options] - -a, Set action definitions file path location, default is ./pal-actions.yml -c, Set configuration file path location, default is ./pal.yml + -d, Set action definitions file directory location, default is ./actions -Example: pal -a ./pal-actions.yml -c ./pal.yml +Example: pal -c ./pal.yml -d ./actions ``` ## Built-In Variables @@ -283,6 +317,7 @@ Every cmd run includes the below built-in env variables. ``` ### Notification Variables + When `OnError.Notification` is configured for the action, you can use available substitution variables in the notification message: `$PAL_GROUP` - Group name @@ -297,7 +332,7 @@ When `OnError.Notification` is configured for the action, you can use available **See latest example reference, here:** [https://github.com/marshyski/pal/blob/main/pal.yml](https://github.com/marshyski/pal/blob/main/pal.yml) -## Example `pal-actions.yml` +## Example Action Definition YML ```yaml monitor: @@ -336,4 +371,6 @@ monitor: curl -sk -H'X-Monitor-System: q1w2e3r4t5' 'https://127.0.0.1:8443/v1/pal/run/monitor/system' ``` -**For a more complete example, see:** [https://github.com/marshyski/pal/blob/main/test/pal-actions.yml](https://github.com/marshyski/pal/blob/main/test/pal-actions.yml) +All actions can be defined in one file, or split into multiple `.yml` files. The `-d` CLI argument tells the program what directory to verify valid action yml files. + +**For more complete examples, see:** [https://github.com/marshyski/pal/tree/main/test](https://github.com/marshyski/pal/tree/main/test) diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..57c7f9d --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,52 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure(2) do |config| + config.vm.box = "debian/bookworm64" + config.vm.hostname = "debian12" + config.vm.provider "virtualbox" do |v| + v.name = "debian12" + v.memory = 2048 + v.cpus = 1 + v.customize ["modifyvm", :id, "--natdnsproxy1", "on"] + v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + v.customize ["modifyvm", :id, "--uartmode1", "file", File::NULL] + end + config.vm.network "forwarded_port", guest: 8443, host: 8443 + config.vm.synced_folder ".", "/vagrant", SharedFoldersEnableSymlinksCreate: true + config.vm.provision "shell", inline: <<-SHELL +# Setup Base Packages +ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get upgrade -y && \ + apt-get dist-upgrade -y && \ + apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + jq && \ + apt-get clean + +# Setup pal +dpkg -i /vagrant/pal*amd64.deb + +# Create Self-Signed Certs +cd /pal +openssl req -x509 -newkey rsa:4096 -nodes -keyout /pal/localhost.key -out /pal/localhost.pem -days 365 -sha256 -subj '/CN=localhost' -addext "subjectAltName=IP:127.0.0.1,DNS:localhost" +chown -Rf pal:pal /pal + +# Copy Insecure Test Configs +cp -f /vagrant/pal.yml /pal/ +cp -f /vagrant/test/*.yml /pal/actions/ + +# Configure Paths for /pal In pal.yml +sed -i "s|listen:.*|listen: 0.0.0.0:8443|" /pal/pal.yml +sed -i "s| key:.*| key: /pal/localhost.key|" /pal/pal.yml +sed -i "s|cert:.*|cert: /pal/localhost.pem|" /pal/pal.yml +sed -i "s|upload_dir:.*|upload_dir: /pal/upload|" /pal/pal.yml +sed -i "s|path:.*|path: /pal/pal.db|" /pal/pal.yml + +# Run pal Systemd Service +systemctl daemon-reload +systemctl enable pal +systemctl restart pal +SHELL +end diff --git a/config/config.go b/config/config.go index 3e9f9d7..08fca6c 100644 --- a/config/config.go +++ b/config/config.go @@ -17,7 +17,7 @@ var ( configMap = cmap.New() ) -func ValidateDefs(res map[string][]data.ActionData) { +func validateDefs(res map[string][]data.ActionData) bool { validate := validator.New(validator.WithRequiredStructEnabled()) for _, v := range res { @@ -25,10 +25,46 @@ func ValidateDefs(res map[string][]data.ActionData) { err := validate.Struct(e) if err != nil { log.Println(err) - log.Fatalln("error panic definitions are invalid") + return false } } } + + return true +} + +func ReadConfig(dir string) map[string][]data.ActionData { + files, err := os.ReadDir(dir) + if err != nil { + log.Fatalln("Error reading directory:", err) + } + + groups := make(map[string][]data.ActionData) + + for _, file := range files { + if filepath.Ext(file.Name()) == ".yml" { + fileLoc := filepath.Clean(dir + "/" + file.Name()) + defs, err := os.ReadFile(fileLoc) + if err != nil { + log.Println("Error reading file:", err) + continue // Skip to the next file + } + + var groupData map[string][]data.ActionData + err = yaml.Unmarshal(defs, &groupData) + if err != nil { + log.Println("Error unmarshaling YAML:", err) + continue + } + + if validateDefs(groupData) { + for k, v := range groupData { + groups[k] = v + } + } + } + } + return groups } func InitConfig(location string) error { diff --git a/entrypoint.sh b/entrypoint.sh index 402dd76..b7c4123 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,7 +1,13 @@ #!/bin/sh +PASS=$(tr Hello, World!

$PAL_INPUT Hello, World!

" diff --git a/test/json.yml b/test/json.yml new file mode 100644 index 0000000..936729e --- /dev/null +++ b/test/json.yml @@ -0,0 +1,29 @@ +json: + # curl -sk -XPOST -d '{"hello":"world"}' 'https://127.0.0.1:8443/v1/pal/run/json/parse' + # curl -sk 'https://127.0.0.1:8443/v1/pal/run/json/parse?input=%7B%22hello%22%3A%22world%22%7D' + - action: parse + desc: Run jq On Input JSON + output: true + concurrent: true + headers: + - header: Content-Type + value: application/json + # input_validate: required,json + tags: + - json + on_error: + notification: "$PAL_GROUP/$PAL_ACTION: error failed parsing JSON: $PAL_OUTPUT: input $PAL_INPUT" + cmd: echo "$PAL_INPUT" | jq + # curl -sk -XPOST -H 'X-PAL-TEST: true' -d 'test_body' 'https://127.0.0.1:8443/v1/pal/run/json/parse_pal_request?test=true' + - action: parse_pal_request + desc: Run jq On PAL_REQUEST JSON + output: true + concurrent: true + headers: + - header: Content-Type + value: application/json + tags: + - json + on_error: + notification: "json/parse_pal_request: error failed parsing PAL_REQUEST" + cmd: echo "$PAL_REQUEST" | jq diff --git a/test/pal-actions.yml b/test/test.yml similarity index 63% rename from test/pal-actions.yml rename to test/test.yml index ea9db1e..2811007 100644 --- a/test/pal-actions.yml +++ b/test/test.yml @@ -107,65 +107,3 @@ test: retries: 2 retry_interval: 10 cmd: echo "$PAL_INPUT retry fail" && exit 1 - -build: - - action: go-pal - desc: Lint & Build Pal - output: true - on_error: - notification: "$PAL_GROUP/$PAL_ACTION lint or build failed: $PAL_OUTPUT" - tags: - - build - cmd: | - make lint - make - file ./pal - du -sh ./pal - sha256sum ./pal - ./pal -h - -json: - # curl -sk -XPOST -d '{"hello":"world"}' 'https://127.0.0.1:8443/v1/pal/run/json/parse' - # curl -sk 'https://127.0.0.1:8443/v1/pal/run/json/parse?input=%7B%22hello%22%3A%22world%22%7D' - - action: parse - desc: Run jq On Input JSON - output: true - concurrent: true - headers: - - header: Content-Type - value: application/json - # input_validate: required,json - tags: - - json - on_error: - notification: "$PAL_GROUP/$PAL_ACTION: error failed parsing JSON: $PAL_OUTPUT: input $PAL_INPUT" - cmd: echo "$PAL_INPUT" | jq - # curl -sk -XPOST -H 'X-PAL-TEST: true' -d 'test_body' 'https://127.0.0.1:8443/v1/pal/run/json/parse_pal_request?test=true' - - action: parse_pal_request - desc: Run jq On PAL_REQUEST JSON - output: true - concurrent: true - headers: - - header: Content-Type - value: application/json - tags: - - json - on_error: - notification: "json/parse_pal_request: error failed parsing PAL_REQUEST" - cmd: echo "$PAL_REQUEST" | jq - -html: - # curl -sk -o /dev/null -w '%{content_type}' 'https://127.0.0.1:8443/v1/pal/run/html/index_cache' - - action: index_cache - desc: Sets Max-Age Cache Control & Renders HTML Content-Type - output: true - concurrent: true - headers: - - header: Cache-Control - value: max-age=3600 - - header: Content-Type - value: text/html - tags: - - html - cmd: | - echo "Hello, World!

$PAL_INPUT Hello, World!

" diff --git a/test/vagrant.yml b/test/vagrant.yml new file mode 100644 index 0000000..aebf69b --- /dev/null +++ b/test/vagrant.yml @@ -0,0 +1,8 @@ +vagrant: + - action: pal + desc: Setup vagrant env - run this pal as 8444 due to port conflict with vagrant on 8443 + output: true + timeout: 1200 + on_error: + notification: "$PAL_GROUP/$PAL_ACTION vagrant up failed: $PAL_OUTPUT" + cmd: make vagrant