From 48de8bfd534dc6c9857dc8e305c1765197564cbb Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Wed, 13 Nov 2024 11:23:45 +0200 Subject: [PATCH 01/65] environment setup fixes --- lib/auth.sh | 105 ++++++++++++++++++++++++++++++++++++++++++- lib/cleanup.sh | 85 ++++++++++++++++++++--------------- lib/environment.sh | 40 +++++++---------- lib/secrets.sh | 19 ++++++-- lib/worker_config.sh | 10 ++--- 5 files changed, 188 insertions(+), 71 deletions(-) diff --git a/lib/auth.sh b/lib/auth.sh index 9cf834d1..8ecfe39b 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -3,6 +3,9 @@ # shellcheck source=/usr/local/lib/utils.sh disable=SC1091 source /usr/local/lib/utils.sh +# Array to track configured providers +declare -a configured_providers=() + # Function to authenticate actors authenticate_actors() { local actors_json="$1" # Expect the extracted actors JSON as a parameter @@ -48,6 +51,8 @@ authenticate_actors() { log_error "Authentication failed for provider $provider" return 1 fi + # Add provider to configured list if authentication succeeds + configured_providers+=("$provider") else log_error "Authentication function $auth_function not found for provider $provider" return 1 @@ -88,5 +93,103 @@ authenticate_provider() { return 0 } -# Example usage (expects actors JSON to be passed): +# Function to clean up actors based on the providers configured during authentication +cleanup_actors() { + log_info "Starting cleanup of actors" + + # Track if any actual cleanup was performed + local any_cleanup=false + + # Loop through each configured provider only + for provider in "${configured_providers[@]}"; do + case "$provider" in + azure) + cleanup_provider "az" "az logout" "az account show" "Azure" && any_cleanup=true + ;; + gcp) + cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" && any_cleanup=true + ;; + aws) + cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" && any_cleanup=true + ;; + bitwarden) + cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" && any_cleanup=true + ;; + *) + log_warn "Unsupported or unavailable actor type for cleanup: $provider" + ;; + esac + done + + # Log a summary if no cleanup actions were needed + if [[ "$any_cleanup" == false ]]; then + log_info "No active sessions found for any configured providers." + fi +} + +# Generic function to clean up authentication for any provider +cleanup_provider() { + local provider=$1 + local logout_cmd=$2 + local list_cmd=$3 + local name=$4 + local cleaned_up=false + + # Check if the provider's CLI is available + if ! command -v "$provider" > /dev/null; then + return 0 # Skip silently if CLI is not available + fi + + # Check if there are active sessions/accounts to clean up + if ! eval "$list_cmd" > /dev/null 2>&1; then + return 0 # Skip silently if no active sessions are found + fi + + log_info "Cleaning up $name authentication" + + # Run the logout command and capture any output or errors + local logout_output + logout_output=$(eval "$logout_cmd" 2>&1) + local logout_status=$? + + # Check if the logout was successful or if an expected error message was returned + if [[ $logout_status -ne 0 ]]; then + if echo "$logout_output" | grep -q -E "No credentials available to revoke|No active sessions|No active accounts"; then + log_info "No active $name credentials to revoke." + else + log_error "Failed to log out of $name: $logout_output" + return 1 + fi + else + log_info "$name authentication cleaned up successfully." + cleaned_up=true + fi + + # Return the cleanup status for summary reporting + if [[ "$cleaned_up" == true ]]; then + return 0 + else + return 1 + fi +} + +# Function to clean up sensitive environment variables based on a pattern +cleanup_sensitive_env_vars() { + log_info "Cleaning up sensitive environment variables" + + # Define a pattern for sensitive environment variables (e.g., AZURE_CREDS, GCP_CREDS, etc.) + local pattern="_CREDS" + + # Loop through environment variables that match the pattern + for var in $(env | grep "${pattern}" | cut -d'=' -f1); do + unset "$var" + log_info "Unset sensitive environment variable: $var" + done + + log_info "Sensitive environment variables cleaned up successfully." +} + +# Example usage: # authenticate_actors "$actors_json" +# cleanup_actors +# cleanup_sensitive_env_vars diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 4af48ec4..d6c26624 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -12,68 +12,81 @@ cleanup_provider() { local logout_cmd=$2 local list_cmd=$3 local name=$4 - - log_info "Cleaning up $name authentication" - + local cleaned_up=false + + # Check if the provider's CLI is available if ! command -v "$provider" > /dev/null; then - log_warn "$name CLI not found. Skipping $name cleanup." - return 0 + return 0 # Skip silently if CLI is not available fi + # Check if there are active sessions/accounts to clean up if ! eval "$list_cmd" > /dev/null 2>&1; then - log_info "No active $name accounts or sessions found." - return 0 + return 0 # Skip silently if no active sessions are found fi - if ! eval "$logout_cmd"; then - log_error "Failed to log out of $name" - return 1 + log_info "Cleaning up $name authentication" + + # Run the logout command and capture any output or errors + local logout_output + logout_output=$(eval "$logout_cmd" 2>&1) + local logout_status=$? + + # Check if the logout was successful or if an expected error message was returned + if [[ $logout_status -ne 0 ]]; then + if echo "$logout_output" | grep -q -E "No credentials available to revoke|No active sessions|No active accounts"; then + log_info "No active $name credentials to revoke." + else + log_error "Failed to log out of $name: $logout_output" + return 1 + fi + else + log_info "$name authentication cleaned up successfully." + cleaned_up=true fi - log_info "$name authentication cleaned up successfully." + # Return the cleanup status for summary reporting + if [[ "$cleaned_up" == true ]]; then + return 0 + else + return 1 + fi } -# Function to clean up actors based on the worker configuration +# Function to clean up actors based on the providers configured during authentication cleanup_actors() { log_info "Starting cleanup of actors" - local worker_config - if ! worker_config=$(get_worker_config_path); then - log_error "Failed to retrieve worker configuration path." - return 1 - fi - - local actors_json - actors_json=$(yq e -o=json '.config.actors' "$worker_config" 2>/dev/null) - - if [[ -z "$actors_json" || "$actors_json" == "null" ]]; then - log_info "No actors found for cleanup." - return 0 - fi - - # Process each actor type - echo "$actors_json" | jq -c '.[]' | while IFS= read -r actor; do - local type - type=$(echo "$actor" | jq -r '.type') + # Accept configured providers as arguments + local configured_providers=("$@") + + # Track if any actual cleanup was performed + local any_cleanup=false - case "$type" in + # Loop through each configured provider only + for provider in "${configured_providers[@]}"; do + case "$provider" in azure) - cleanup_provider "az" "az logout" "az account show" "Azure" + cleanup_provider "az" "az logout" "az account show" "Azure" && any_cleanup=true ;; gcp) - cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" + cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" && any_cleanup=true ;; aws) - cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" + cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" && any_cleanup=true ;; bitwarden) - cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" + cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" && any_cleanup=true ;; *) - log_warn "Unsupported or unavailable actor type for cleanup: $type" + log_warn "Unsupported or unavailable actor type for cleanup: $provider" ;; esac done + + # Log a summary if no cleanup actions were needed + if [[ "$any_cleanup" == false ]]; then + log_info "No active sessions found for any configured providers." + fi } # Function to clean up sensitive environment variables based on a pattern diff --git a/lib/environment.sh b/lib/environment.sh index 8b59f359..b2f258e7 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -10,11 +10,9 @@ source /usr/local/lib/worker_config.sh # Main function to coordinate environment setup configure_environment() { - # Load and resolve the worker configuration local resolved_config resolved_config=$(load_and_resolve_worker_config) - # Directly check the command success, not using $? if [ -z "$resolved_config" ]; then log_error "Failed to resolve worker configuration." return 1 @@ -29,34 +27,28 @@ configure_environment() { fi log_info "Config file found: $config_path" - # Extract actors section from the resolved configuration + # Extract actors section and authenticate local actors actors=$(get_worker_section "$resolved_config" "config.actors") - # log_debug "Extracted actors: $actors" - if [ -z "$actors" ]; then - log_error "No actors found in the configuration." - return 1 - fi - - # Authenticate actors using the extracted actors section - if ! authenticate_actors "$actors"; then - log_error "Failed to authenticate actors." - return 1 + if [[ $? -eq 0 && -n "$actors" ]]; then + if ! authenticate_actors "$actors"; then + log_error "Failed to authenticate actors." + return 1 + fi + else + log_info "No actors defined or required for authentication." fi - # Extract secrets section from the resolved configuration + # Extract secrets section and fetch secrets if available local secrets secrets=$(get_worker_section "$resolved_config" "config.secrets") - # log_debug "Extracted secrets: $secrets" - if [ -z "$secrets" ]; then - log_error "No secrets found in the configuration." - return 1 - fi - - # Fetch secrets using the resolved configuration - if ! fetch_secrets "$secrets"; then - log_error "Failed to fetch secrets." - return 1 + if [[ $? -eq 0 && -n "$secrets" ]]; then + if ! fetch_secrets "$secrets"; then + log_error "Failed to fetch secrets." + return 1 + fi + else + log_info "No secrets found or required in the configuration." fi # Clean up actors and sensitive environment variables diff --git a/lib/secrets.sh b/lib/secrets.sh index aae52f4a..fdd860ad 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -21,14 +21,21 @@ source_provider_module() { # Fetch secrets and set them as environment variables fetch_secrets() { local secrets_json="$1" - + log_info "Fetching secrets and setting them as environment variables." + # Exit early if secrets_json is empty, null, or invalid JSON if [[ -z "$secrets_json" || "$secrets_json" == "null" ]]; then log_info "No worker secrets found in the configuration." return 0 fi + # Confirm secrets_json is valid JSON before processing + if ! echo "$secrets_json" | jq empty > /dev/null 2>&1; then + log_error "Invalid JSON format for secrets configuration." + return 1 + fi + # Create a temporary file to store environment variables local secrets_env_file secrets_env_file=$(mktemp /tmp/secret_vars.XXXXXX) @@ -40,22 +47,26 @@ fetch_secrets() { name=$(echo "$secret" | jq -r '.key') url=$(resolve_env_vars "$(echo "$secret" | jq -r '.value')") + # Check if the secret has a valid name and URL + if [[ -z "$name" || -z "$url" ]]; then + log_error "Secret name or URL is missing or empty." + continue + fi + # Extract provider from the URL (first part before '/') provider=$(echo "$url" | cut -d '/' -f 1) # Handle secrets based on the provider case "$provider" in gcp) - # Extract secret name and pass to GCP resolver key_vault_name=$(echo "$url" | cut -d '/' -f 2) secret_name=$(echo "$url" | cut -d '/' -f 3) if [[ -z "$secret_name" ]]; then - log_error "Invalid GCP secret name: $url" + log_error "Invalid GCP secret name format: $url" continue fi ;; azure|bitwarden) - # Extract key vault and secret name key_vault_name=$(echo "$url" | cut -d '/' -f 2) secret_name=$(echo "$url" | cut -d '/' -f 3) if [[ -z "$key_vault_name" || -z "$secret_name" ]]; then diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 038e5c07..c4aeaa88 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -56,15 +56,13 @@ get_worker_section() { return 1 fi + # Attempt to extract the section and handle missing/null cases local extracted_section - if ! extracted_section=$(echo "$config_json" | jq -r ".${section}"); then - log_error "Failed to extract section '$section' from JSON." - return 1 - fi + extracted_section=$(echo "$config_json" | jq -r ".${section} // empty") + # Return success if the section is empty or null if [[ -z "$extracted_section" || "$extracted_section" == "null" ]]; then - log_error "Section '$section' is empty or null." - return 1 + return 0 # No error, section simply doesn't exist fi echo "$extracted_section" From 21096be526f269ffcd96cd6ff561b40102c0de17 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Wed, 13 Nov 2024 11:24:25 +0200 Subject: [PATCH 02/65] worker config update --- src/configs/worker.yml | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/src/configs/worker.yml b/src/configs/worker.yml index 1ff0e0f6..7faa866d 100644 --- a/src/configs/worker.yml +++ b/src/configs/worker.yml @@ -5,26 +5,12 @@ config: variables: DOCKER_IMAGE_NAME: 'udx-worker' - secrets: - NEW_RELIC_API_KEY: 'gcp/udx-worker-project/new_relic_api_key' - - # Supported: - # NEW_RELIC_API_KEY: "azure/kv-udx-worker/new_relic_api_key" - - # To be supported: - # NEW_RELIC_API_KEY: "aws/secrets-manager/new_relic_api_key" - # NEW_RELIC_API_KEY: "bitwarden/collection/new_relic_api_key" - actors: - type: gcp creds: '${GCP_CREDS}' - - # Supported: - # - type: azure - # creds: "${AZURE_CREDS}" - - # To be supported: - # - type: aws - # creds: "${AWS_CREDS}" - # - type: bitwarden - # creds: "${BITWARDEN_CREDS}" + - type: azure + creds: '${AZURE_CREDS}' + - type: aws + creds: '${AWS_CREDS}' + - type: bitwarden + creds: '${BITWARDEN_CREDS}' From f5640cf858b431de54fc9fee72d1524102c651aa Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Wed, 13 Nov 2024 11:24:44 +0200 Subject: [PATCH 03/65] .dockerignore tweak --- .dockerignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index a0c0a5b1..1c9975e5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -32,9 +32,6 @@ Makefile.variables *.bak *.old -# Unnecessary configuration or text files -etc/home/logo.txt - # Readme files in the secrets and auth directories lib/auth/readme.md lib/secrets/readme.md From 0a3539ee519df452af5644247b36066a43f02a39 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Wed, 13 Nov 2024 11:32:46 +0200 Subject: [PATCH 04/65] fixes --- .yamllint | 2 +- lib/auth.sh | 47 +++++++++++------------------------------------ 2 files changed, 12 insertions(+), 37 deletions(-) diff --git a/.yamllint b/.yamllint index 315d44fc..9f34b2e7 100644 --- a/.yamllint +++ b/.yamllint @@ -2,7 +2,7 @@ extends: default rules: line-length: - max: 105 # Set a longer line length if 80 is too restrictive + max: 150 # Set a longer line length if 80 is too restrictive level: warning # Level should be either "error" or "warning" truthy: diff --git a/lib/auth.sh b/lib/auth.sh index 8ecfe39b..38d517f2 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -15,8 +15,17 @@ authenticate_actors() { return 0 fi - # Process each actor in the configuration - echo "$actors_json" | jq -c '.[]' | while IFS= read -r actor; do + # Use a temporary file to avoid a subshell + local actors_file + actors_file=$(mktemp) + echo "$actors_json" | jq -c '.[]' > "$actors_file" + + # Read all lines into an array, then delete the file + mapfile -t actors_array < "$actors_file" + rm -f "$actors_file" + + # Process each actor in the array + for actor in "${actors_array[@]}"; do local type provider creds auth_script auth_function # Extract the type and provider from the actor data @@ -93,40 +102,6 @@ authenticate_provider() { return 0 } -# Function to clean up actors based on the providers configured during authentication -cleanup_actors() { - log_info "Starting cleanup of actors" - - # Track if any actual cleanup was performed - local any_cleanup=false - - # Loop through each configured provider only - for provider in "${configured_providers[@]}"; do - case "$provider" in - azure) - cleanup_provider "az" "az logout" "az account show" "Azure" && any_cleanup=true - ;; - gcp) - cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" && any_cleanup=true - ;; - aws) - cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" && any_cleanup=true - ;; - bitwarden) - cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" && any_cleanup=true - ;; - *) - log_warn "Unsupported or unavailable actor type for cleanup: $provider" - ;; - esac - done - - # Log a summary if no cleanup actions were needed - if [[ "$any_cleanup" == false ]]; then - log_info "No active sessions found for any configured providers." - fi -} - # Generic function to clean up authentication for any provider cleanup_provider() { local provider=$1 From c8432259f99983c80d99c1fe37d0ceb3a6e68363 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 15:14:17 +0200 Subject: [PATCH 05/65] initiated aws secret manager module --- lib/secrets/aws.sh | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/secrets/aws.sh b/lib/secrets/aws.sh index ed381229..b89fc012 100644 --- a/lib/secrets/aws.sh +++ b/lib/secrets/aws.sh @@ -1,6 +1,29 @@ #!/bin/bash # Function to resolve AWS secret -resolve_aws_secret(){ - echo "Not supported yet. Is in progress" +resolve_aws_secret() { + local secret_name="$1" + local region="$2" + local secret_value + + if [[ -z "$secret_name" || -z "$region" ]]; then + echo "[ERROR] Invalid AWS secret name or region" >&2 + return 1 + fi + + # Retrieve the secret value using AWS CLI with detailed logging + echo "[INFO] Retrieving secret from AWS Secrets Manager: secret_name=$secret_name, region=$region" >&2 + if ! secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query 'SecretString' --output text --region "$region" 2>&1); then + echo "[ERROR] Failed to retrieve secret from AWS Secrets Manager: secret_name=$secret_name, region=$region" >&2 + echo "[DEBUG] AWS CLI output: $secret_value" >&2 + return 1 + fi + + if [ -z "$secret_value" ]; then + echo "[ERROR] Secret value is empty for $secret_name in region $region" >&2 + return 1 + fi + + printf "%s" "$secret_value" + return 0 } \ No newline at end of file From 6d8e9789181c17daf1494257f2eee9fa6bada695 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 15:15:17 +0200 Subject: [PATCH 06/65] initiated bitwarden secret manager module --- lib/secrets/bitwarden.sh | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/secrets/bitwarden.sh b/lib/secrets/bitwarden.sh index 6c3f1921..f6665568 100644 --- a/lib/secrets/bitwarden.sh +++ b/lib/secrets/bitwarden.sh @@ -1,6 +1,29 @@ #!/bin/bash # Function to resolve Bitwarden secret -resolve_bitwarden_secret(){ - echo "Not supported yet. Is in progress" +resolve_bitwarden_secret() { + local collection_name="$1" + local secret_name="$2" + local secret_value + + if [[ -z "$collection_name" || -z "$secret_name" ]]; then + echo "[ERROR] Invalid Bitwarden collection name or secret name" >&2 + return 1 + fi + + # Retrieve the secret using Bitwarden CLI + echo "[INFO] Retrieving secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" >&2 + if ! secret_value=$(bw get item "$secret_name" --organizationid "$collection_name" --output text 2>&1); then + echo "[ERROR] Failed to retrieve secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" >&2 + echo "[DEBUG] Bitwarden CLI output: $secret_value" >&2 + return 1 + fi + + if [ -z "$secret_value" ]; then + echo "[ERROR] Secret value is empty for $secret_name in collection $collection_name" >&2 + return 1 + fi + + printf "%s" "$secret_value" + return 0 } \ No newline at end of file From e566d00934fb692d3417157dad68471606c47a6a Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 15:35:42 +0200 Subject: [PATCH 07/65] release workflow improvements --- .github/workflows/release.yml | 150 ++++++++++++---------------------- 1 file changed, 54 insertions(+), 96 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d35f87bd..9fb049a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,4 @@ ---- -name: Release UDX Worker +name: Release on: push: @@ -7,11 +6,11 @@ on: - "latest" jobs: - test-pipeline: + docker-release: runs-on: ubuntu-latest permissions: id-token: write - contents: read + contents: write outputs: semVer: ${{ steps.gitversion.outputs.semVer }} changelog: ${{ steps.changelog.outputs.changelog }} @@ -26,17 +25,6 @@ jobs: with: driver: docker-container - - name: Prepare Docker cache directory - run: mkdir -p /tmp/.buildx-cache - - - name: Cache Docker layers - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Clear GitVersion Cache run: rm -rf .git/gitversion_cache @@ -52,25 +40,39 @@ jobs: useConfigFile: true configFilePath: ci/git-version.yml - - name: Multi-arch build - id: build + - name: Generate changelog + id: changelog + run: | + git log $(git describe --tags --abbrev=0)..HEAD -- . \ + --pretty=format:"- %s" > changelog.txt + CHANGELOG=$(cat changelog.txt | jq -sRr @uri) + echo "changelog<> $GITHUB_ENV + echo "$CHANGELOG" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Multi-arch build and push to Docker Hub + id: build-push uses: docker/build-push-action@v6 with: context: . file: ./Dockerfile platforms: linux/amd64,linux/arm64 - push: false - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache,mode=max + push: true tags: | usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + usabilitydynamics/udx-worker:latest - name: Generate SBOM + id: generate-sbom run: | export TRIVY_DISABLE_VEX_NOTICE=true trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} 2>/dev/null - # Save SBOM for later upload - id: generate-sbom - name: Upload SBOM Artifact uses: actions/upload-artifact@v4 @@ -78,19 +80,33 @@ jobs: name: sbom path: sbom.json - - name: Generate changelog - id: changelog + - name: Install Cosign + uses: sigstore/cosign-installer@v3.7.0 + + - name: Sign Docker image with Cosign + env: + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - git log $(git describe --tags --abbrev=0)..HEAD -- . \ - --pretty=format:"- %s" > changelog.txt - CHANGELOG=$(cat changelog.txt | jq -sRr @uri) - echo "changelog<> $GITHUB_ENV - echo "$CHANGELOG" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV + cosign sign \ + --key env://COSIGN_PRIVATE_KEY \ + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + + - name: Sign SBOM with Cosign + env: + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + run: | + cosign attest \ + --key env://COSIGN_PRIVATE_KEY \ + --type sbom \ + --predicate sbom.json \ + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + + - name: Log out from Docker Hub + run: docker logout github-release: runs-on: ubuntu-latest - needs: [test-pipeline] + needs: docker-release permissions: contents: write steps: @@ -108,8 +124,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} run: | - git tag ${{ needs.test-pipeline.outputs.semVer }} - git push origin ${{ needs.test-pipeline.outputs.semVer }} + git tag ${{ needs.docker-release.outputs.semVer }} + git push origin ${{ needs.docker-release.outputs.semVer }} - name: Download SBOM Artifact uses: actions/download-artifact@v4 @@ -119,71 +135,13 @@ jobs: - name: Create GitHub release uses: softprops/action-gh-release@v2 with: - tag_name: ${{ needs.test-pipeline.outputs.semVer }} - body: ${{ needs.test-pipeline.outputs.changelog }} + tag_name: ${{ needs.docker-release.outputs.semVer }} + body: | + Release version ${{ needs.docker-release.outputs.semVer }}. + [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). + ${{ needs.docker-release.outputs.changelog }} draft: false prerelease: false files: sbom.json env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - docker-release: - runs-on: ubuntu-latest - needs: [test-pipeline] - permissions: - contents: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker-container - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Load Docker cache - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} - - - name: Push Docker image - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - cache-from: type=local,src=/tmp/.buildx-cache - tags: | - usabilitydynamics/udx-worker:${{ needs.test-pipeline.outputs.semVer }} - usabilitydynamics/udx-worker:latest - - - name: Download SBOM Artifact - uses: actions/download-artifact@v4 - with: - name: sbom - - - name: Install Cosign - uses: sigstore/cosign-installer@v3.7.0 - - - name: Sign SBOM with Cosign - env: - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - run: | - cosign attest \ - --key env://COSIGN_PRIVATE_KEY \ - --type sbom \ - --predicate sbom.json \ - usabilitydynamics/udx-worker:${{ needs.test-pipeline.outputs.semVer }} - - - name: Log out from Docker Hub - run: docker logout From c44a452c8c209f803afd83672945b254aaa058e1 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 15:41:46 +0200 Subject: [PATCH 08/65] added non-latest config to git-version --- ci/git-version.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ci/git-version.yml b/ci/git-version.yml index 5bed307f..b5419f77 100644 --- a/ci/git-version.yml +++ b/ci/git-version.yml @@ -8,4 +8,13 @@ branches: increment: Minor source-branches: [] + other: + regex: ^(?!latest).*$ + tag: 'beta' + increment: Patch + source-branches: [] + tracks-release-branches: false + is-release-branch: false + prevent-increment-of-merged-branch-version: false + assembly-versioning-scheme: MajorMinorPatch From 77f00c8624c0bcecd1efab3d93a6e4006a780343 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 16:06:02 +0200 Subject: [PATCH 09/65] disabled trivy caching on build workflow --- .github/workflows/build-and-test.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 0bd62120..206d3f66 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -24,14 +24,6 @@ jobs: run: make dev-pipeline working-directory: . - - name: Cache Trivy DB - uses: actions/cache@v4 - with: - path: ~/.cache/trivy - key: ${{ runner.os }}-trivy-db-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-trivy-db- - - name: Install Trivy run: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ @@ -57,25 +49,25 @@ jobs: udx-worker/udx-worker:latest | tee trivy.log | grep -v 'INFO' scan_exit_code=$? - # Check for CRITICAL vulnerabilities + # Check if CRITICAL vulnerabilities were detected if grep -E "Total: [1-9]" trivy.log; then - echo "CRITICAL vulnerabilities detected!" + echo "CRITICAL vulnerabilities detected! Exiting." exit 1 fi - # Check if Trivy exited with an error + # Handle a successful scan (no critical vulnerabilities found) if [ $scan_exit_code -eq 0 ]; then echo "No CRITICAL vulnerabilities found." success=true break else - echo "Trivy scan failed, retrying in 2 minutes..." + echo "Trivy scan encountered an error, retrying in 2 minutes..." sleep 120 attempt=$((attempt+1)) fi done - # If all retries fail, exit with an error + # Exit if all retries fail without a successful scan if [ "$success" = false ]; then echo "Failed to complete Trivy scan after $max_retries attempts." exit 1 @@ -83,7 +75,6 @@ jobs: - name: Trivy SBOM Generation run: | - # Suppress verbose notices and informational messages export TRIVY_DISABLE_VEX_NOTICE=true trivy image --format spdx-json --output sbom.json udx-worker/udx-worker:latest 2>/dev/null @@ -91,7 +82,6 @@ jobs: echo "| Package Name | Version |" echo "|-------------------|-----------|" - # Use jq to extract name and versionInfo, excluding packages with null versions, and pipe to column for formatting jq -r '.packages[] | select(.versionInfo != null) | "\(.name) | \(.versionInfo)"' sbom.json | sort | uniq | head -n 20 | column -t -s '|' - name: Upload SBOM Artifact From 4212dbc7b896fd77e5c4b5f134bdc951f45a7071 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 16:07:48 +0200 Subject: [PATCH 10/65] test release --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9fb049a6..c7f45305 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: Release on: push: - branches: - - "latest" + # branches: + # - "latest" jobs: docker-release: From 3ee2e681556e1ef027034c50879bdb2377c26167 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 16:19:00 +0200 Subject: [PATCH 11/65] improved build trigger --- .github/workflows/build-and-test.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 206d3f66..37d1c2e8 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,9 +1,16 @@ ---- name: Build and Test Docker Image on: push: branches-ignore: latest + paths: + - 'Dockerfile' + - 'bin/**' + - 'lib/**' + - 'src/**' + - 'etc/**' + - 'Makefile' + - 'Makefile.variables' jobs: build: From 3097ae9f8f2a7295b3466d6ac5c7a755345561ca Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 16:21:18 +0200 Subject: [PATCH 12/65] fix --- .github/workflows/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7f45305..d1aadaf8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,6 +33,9 @@ jobs: with: versionSpec: "5.12.0" + - name: Clear GitVersion Cache + run: rm -rf .git/gitversion_cache + - name: Determine Version id: gitversion uses: gittools/actions/gitversion/execute@v3.0.0 From 3543c33bbea2cabfcffbcac1ef3933457bbf0dc8 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 17:56:47 +0200 Subject: [PATCH 13/65] fix release workflow --- .github/workflows/release.yml | 13 ++++++------- ci/git-version.yml | 3 --- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1aadaf8..4d71e2bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,9 +11,7 @@ jobs: permissions: id-token: write contents: write - outputs: - semVer: ${{ steps.gitversion.outputs.semVer }} - changelog: ${{ steps.changelog.outputs.changelog }} + steps: - name: Checkout code uses: actions/checkout@v4 @@ -25,13 +23,10 @@ jobs: with: driver: docker-container - - name: Clear GitVersion Cache - run: rm -rf .git/gitversion_cache - - name: Install GitVersion uses: gittools/actions/gitversion/setup@v3.0.0 with: - versionSpec: "5.12.0" + versionSpec: "5.x" - name: Clear GitVersion Cache run: rm -rf .git/gitversion_cache @@ -107,6 +102,10 @@ jobs: - name: Log out from Docker Hub run: docker logout + outputs: + semVer: ${{ steps.gitversion.outputs.semVer }} + changelog: ${{ steps.changelog.outputs.changelog }} + github-release: runs-on: ubuntu-latest needs: docker-release diff --git a/ci/git-version.yml b/ci/git-version.yml index b5419f77..8343b942 100644 --- a/ci/git-version.yml +++ b/ci/git-version.yml @@ -1,16 +1,13 @@ ---- mode: Mainline branches: latest: regex: ^latest$ - tag: '' increment: Minor source-branches: [] other: regex: ^(?!latest).*$ - tag: 'beta' increment: Patch source-branches: [] tracks-release-branches: false From c639320735479aa0f469e13e7c803df1c50dc186 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 18:23:08 +0200 Subject: [PATCH 14/65] downgrade to gittools/actions/gitversion@v2 --- .github/workflows/release.yml | 6 +++--- ci/git-version.yml | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d71e2bd..b5ebb662 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,16 +24,16 @@ jobs: driver: docker-container - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v3.0.0 + uses: gittools/actions/gitversion/setup@v2 with: - versionSpec: "5.x" + versionSpec: "5.12.0" - name: Clear GitVersion Cache run: rm -rf .git/gitversion_cache - name: Determine Version id: gitversion - uses: gittools/actions/gitversion/execute@v3.0.0 + uses: gittools/actions/gitversion/execute@v2 with: useConfigFile: true configFilePath: ci/git-version.yml diff --git a/ci/git-version.yml b/ci/git-version.yml index 8343b942..b5419f77 100644 --- a/ci/git-version.yml +++ b/ci/git-version.yml @@ -1,13 +1,16 @@ +--- mode: Mainline branches: latest: regex: ^latest$ + tag: '' increment: Minor source-branches: [] other: regex: ^(?!latest).*$ + tag: 'beta' increment: Patch source-branches: [] tracks-release-branches: false From d882125465c474cdd4b4e9055195cb4541089000 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 18:28:26 +0200 Subject: [PATCH 15/65] fix git-version config --- .github/workflows/release.yml | 6 +++--- ci/git-version.yml | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5ebb662..60034ac0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,16 +24,16 @@ jobs: driver: docker-container - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v2 + uses: gittools/actions/gitversion/setup@v3.0.0 with: - versionSpec: "5.12.0" + versionSpec: "6.0.0" - name: Clear GitVersion Cache run: rm -rf .git/gitversion_cache - name: Determine Version id: gitversion - uses: gittools/actions/gitversion/execute@v2 + uses: gittools/actions/gitversion/execute@v3.0.0 with: useConfigFile: true configFilePath: ci/git-version.yml diff --git a/ci/git-version.yml b/ci/git-version.yml index b5419f77..f0c01219 100644 --- a/ci/git-version.yml +++ b/ci/git-version.yml @@ -1,20 +1,17 @@ --- -mode: Mainline +mode: ContinuousDeployment branches: latest: regex: ^latest$ - tag: '' increment: Minor source-branches: [] other: regex: ^(?!latest).*$ - tag: 'beta' increment: Patch source-branches: [] tracks-release-branches: false is-release-branch: false - prevent-increment-of-merged-branch-version: false assembly-versioning-scheme: MajorMinorPatch From d75aeae7fd8a1f88b496bed57b2f3c27b395f85a Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 18:35:17 +0200 Subject: [PATCH 16/65] upgrade vim package version --- .github/workflows/release.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 60034ac0..bfa6fc94 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,7 @@ jobs: versionSpec: "6.0.0" - name: Clear GitVersion Cache - run: rm -rf .git/gitversion_cache + run: rm -rf .git/gitversion_cache - name: Determine Version id: gitversion diff --git a/Dockerfile b/Dockerfile index 5fc7db0d..7953c3e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && \ zip=3.0-13build1 \ unzip=6.0-28ubuntu4 \ nano=7.2-2build1 \ - vim=2:9.1.0016-1ubuntu7.3 && \ + vim=2:9.1.0016-1ubuntu7.4 && \ ln -fs /usr/share/zoneinfo/$TZ /etc/localtime && \ dpkg-reconfigure --frontend noninteractive tzdata && \ apt-get clean && \ From 5f251865eec9cc261b7e43e07956e8111ba3ed83 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 20:49:38 +0200 Subject: [PATCH 17/65] added trivy installing step to release workflow --- .github/workflows/release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfa6fc94..dae8ea25 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,6 +66,11 @@ jobs: usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} usabilitydynamics/udx-worker:latest + - name: Install Trivy + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ + sudo sh -s -- -b /usr/local/bin + - name: Generate SBOM id: generate-sbom run: | From 1cd4e6dac36bef135cf53c0decddf3e9510cfc67 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 20:49:38 +0200 Subject: [PATCH 18/65] added trivy installing step to release workflow --- .github/workflows/release.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfa6fc94..0ceefcca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,6 +66,18 @@ jobs: usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} usabilitydynamics/udx-worker:latest + - name: Retrieve Image Digest + id: digest + run: | + IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }}) + echo "IMAGE_DIGEST=$IMAGE_DIGEST" >> $GITHUB_ENV + echo "::set-output name=IMAGE_DIGEST::$IMAGE_DIGEST" + + - name: Install Trivy + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ + sudo sh -s -- -b /usr/local/bin + - name: Generate SBOM id: generate-sbom run: | @@ -85,19 +97,19 @@ jobs: env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - cosign sign \ + cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + "${{ steps.digest.outputs.IMAGE_DIGEST }}" - name: Sign SBOM with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - cosign attest \ + cosign attest -y \ --key env://COSIGN_PRIVATE_KEY \ --type sbom \ --predicate sbom.json \ - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + "${{ steps.digest.outputs.IMAGE_DIGEST }}" - name: Log out from Docker Hub run: docker logout From a46b0bda9c0661409398a35a6a3b195005d7a92c Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 14 Nov 2024 23:44:20 +0200 Subject: [PATCH 19/65] fix digest dependency for attestation steps --- .github/workflows/release.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0ceefcca..0730be34 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,13 +66,6 @@ jobs: usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} usabilitydynamics/udx-worker:latest - - name: Retrieve Image Digest - id: digest - run: | - IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }}) - echo "IMAGE_DIGEST=$IMAGE_DIGEST" >> $GITHUB_ENV - echo "::set-output name=IMAGE_DIGEST::$IMAGE_DIGEST" - - name: Install Trivy run: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ @@ -97,19 +90,19 @@ jobs: env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - cosign sign -y \ + cosign sign \ --key env://COSIGN_PRIVATE_KEY \ - "${{ steps.digest.outputs.IMAGE_DIGEST }}" + usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} - name: Sign SBOM with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - cosign attest -y \ + cosign attest \ --key env://COSIGN_PRIVATE_KEY \ --type sbom \ --predicate sbom.json \ - "${{ steps.digest.outputs.IMAGE_DIGEST }}" + usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} - name: Log out from Docker Hub run: docker logout @@ -117,6 +110,7 @@ jobs: outputs: semVer: ${{ steps.gitversion.outputs.semVer }} changelog: ${{ steps.changelog.outputs.changelog }} + digest: ${{ steps.build-push.outputs.digest }} github-release: runs-on: ubuntu-latest From e3c98a04b7b05e762809af3c2917f6dddc53d0b9 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 15 Nov 2024 00:47:45 +0200 Subject: [PATCH 20/65] fix --- .github/workflows/release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0730be34..39920049 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,6 +71,10 @@ jobs: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ sudo sh -s -- -b /usr/local/bin + - name: Pull Docker Image for SBOM Generation + run: | + docker pull usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + - name: Generate SBOM id: generate-sbom run: | From e21d6730d5925cf85c679089a112a3f50b953bf2 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 15 Nov 2024 01:43:03 +0200 Subject: [PATCH 21/65] debug release --- .github/workflows/release.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39920049..0b5ec36e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,12 +74,22 @@ jobs: - name: Pull Docker Image for SBOM Generation run: | docker pull usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + docker images # List images to verify the image is present locally + + - name: Verify Image Availability + run: | + if docker image inspect usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} > /dev/null 2>&1; then + echo "Image is available locally for SBOM generation." + else + echo "Image is not available locally. Exiting." + exit 1 + fi - name: Generate SBOM id: generate-sbom run: | export TRIVY_DISABLE_VEX_NOTICE=true - trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} 2>/dev/null + trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} || exit 1 - name: Upload SBOM Artifact uses: actions/upload-artifact@v4 From d62d8c172b67e6ae43ccb28b2ce284c1fc80e121 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 15 Nov 2024 09:10:10 +0200 Subject: [PATCH 22/65] SBOM generation retry logic --- .github/workflows/release.yml | 43 +++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0b5ec36e..c7a45b75 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,20 +76,43 @@ jobs: docker pull usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} docker images # List images to verify the image is present locally - - name: Verify Image Availability - run: | - if docker image inspect usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} > /dev/null 2>&1; then - echo "Image is available locally for SBOM generation." - else - echo "Image is not available locally. Exiting." - exit 1 - fi - - name: Generate SBOM id: generate-sbom run: | export TRIVY_DISABLE_VEX_NOTICE=true - trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} || exit 1 + + # Disable exit on error for the retry logic + set +e + + # Retry logic for Trivy SBOM generation + max_retries=5 + attempt=1 + success=false + + while [ $attempt -le $max_retries ]; do + echo "Generating SBOM, attempt $attempt..." + + # Run Trivy SBOM generation and capture the exit status + trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} 2>/dev/null + sbom_exit_code=$? + + # Check if SBOM generation was successful + if [ $sbom_exit_code -eq 0 ]; then + echo "SBOM generation successful." + success=true + break + else + echo "SBOM generation encountered an error, retrying in 2 minutes..." + sleep 120 + attempt=$((attempt+1)) + fi + done + + # Exit if all retries fail without a successful SBOM generation + if [ "$success" = false ]; then + echo "Failed to complete SBOM generation after $max_retries attempts." + exit 1 + fi - name: Upload SBOM Artifact uses: actions/upload-artifact@v4 From 2e1ecb74de2286069b44d9754c91873340bc7f19 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 15 Nov 2024 12:18:25 +0200 Subject: [PATCH 23/65] release debugging --- .github/workflows/release.yml | 119 +++++++++++++++++----------------- 1 file changed, 58 insertions(+), 61 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7a45b75..d2191779 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,17 +54,18 @@ jobs: username: ${{ vars.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - name: Multi-arch build and push to Docker Hub - id: build-push - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: | - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - usabilitydynamics/udx-worker:latest + # Temporarily comment out the build and push step to save time during testing + # - name: Multi-arch build and push to Docker Hub + # id: build-push + # uses: docker/build-push-action@v6 + # with: + # context: . + # file: ./Dockerfile + # platforms: linux/amd64,linux/arm64 + # push: true + # tags: | + # usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + # usabilitydynamics/udx-worker:latest - name: Install Trivy run: | @@ -127,9 +128,9 @@ jobs: env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - cosign sign \ + cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ - usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - name: Sign SBOM with Cosign env: @@ -137,56 +138,52 @@ jobs: run: | cosign attest \ --key env://COSIGN_PRIVATE_KEY \ - --type sbom \ --predicate sbom.json \ - usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} + --type https://spdx.dev/spdx-specification-2-2-pdf \ + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - name: Log out from Docker Hub run: docker logout - outputs: - semVer: ${{ steps.gitversion.outputs.semVer }} - changelog: ${{ steps.changelog.outputs.changelog }} - digest: ${{ steps.build-push.outputs.digest }} - - github-release: - runs-on: ubuntu-latest - needs: docker-release - permissions: - contents: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure git for pushing - run: | - git config --global user.email "worker@udx.io" - git config --global user.name "UDX Worker" - - - name: Create GitHub Tag - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - run: | - git tag ${{ needs.docker-release.outputs.semVer }} - git push origin ${{ needs.docker-release.outputs.semVer }} - - - name: Download SBOM Artifact - uses: actions/download-artifact@v4 - with: - name: sbom - - - name: Create GitHub release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ needs.docker-release.outputs.semVer }} - body: | - Release version ${{ needs.docker-release.outputs.semVer }}. - [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). - ${{ needs.docker-release.outputs.changelog }} - draft: false - prerelease: false - files: sbom.json - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + # Temporarily comment out GitHub release job to avoid duplicate releases + # github-release: + # runs-on: ubuntu-latest + # needs: docker-release + # permissions: + # contents: write + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + + # - name: Configure git for pushing + # run: | + # git config --global user.email "worker@udx.io" + # git config --global user.name "UDX Worker" + + # - name: Create GitHub Tag + # env: + # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + # run: | + # git tag ${{ needs.docker-release.outputs.semVer }} + # git push origin ${{ needs.docker-release.outputs.semVer }} + + # - name: Download SBOM Artifact + # uses: actions/download-artifact@v4 + # with: + # name: sbom + + # - name: Create GitHub release + # uses: softprops/action-gh-release@v2 + # with: + # tag_name: ${{ needs.docker-release.outputs.semVer }} + # body: | + # Release version ${{ needs.docker-release.outputs.semVer }}. + # [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). + # ${{ needs.docker-release.outputs.changelog }} + # draft: false + # prerelease: false + # files: sbom.json + # env: + # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} From d174e12a7729b011cd932ee08f8280f8649903f0 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 15 Nov 2024 13:31:34 +0200 Subject: [PATCH 24/65] improved trivy retry --- .github/workflows/release.yml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d2191779..e5cac2c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -77,24 +77,23 @@ jobs: docker pull usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} docker images # List images to verify the image is present locally - - name: Generate SBOM + - name: Generate SBOM with Retry Logic id: generate-sbom run: | export TRIVY_DISABLE_VEX_NOTICE=true - # Disable exit on error for the retry logic - set +e + set +e # Disable exit on error for the retry logic # Retry logic for Trivy SBOM generation - max_retries=5 + max_retries=10 attempt=1 success=false while [ $attempt -le $max_retries ]; do echo "Generating SBOM, attempt $attempt..." - # Run Trivy SBOM generation and capture the exit status - trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} 2>/dev/null + # Run Trivy SBOM generation and capture the output and exit status + output=$(trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} 2>&1) sbom_exit_code=$? # Check if SBOM generation was successful @@ -103,8 +102,18 @@ jobs: success=true break else - echo "SBOM generation encountered an error, retrying in 2 minutes..." - sleep 120 + echo "SBOM generation encountered an error." + + # Extract and remove decimals from retry-after value + retry_after=$(echo "$output" | grep -oP 'retry-after: \K[0-9]+') + + # Default sleep time if retry-after is not found + if [ -z "$retry_after" ]; then + retry_after=120 # Default to 2 minutes if retry-after is missing + fi + + echo "Retrying in ${retry_after} seconds..." + sleep "$retry_after" attempt=$((attempt+1)) fi done @@ -140,7 +149,7 @@ jobs: --key env://COSIGN_PRIVATE_KEY \ --predicate sbom.json \ --type https://spdx.dev/spdx-specification-2-2-pdf \ - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - name: Log out from Docker Hub run: docker logout From 69e645e7c7201786fd3b77fd13c75aea7f29b933 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 15 Nov 2024 13:47:52 +0200 Subject: [PATCH 25/65] fix cosign steps --- .github/workflows/release.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5cac2c8..4ce6291b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,10 +73,18 @@ jobs: sudo sh -s -- -b /usr/local/bin - name: Pull Docker Image for SBOM Generation + id: pull-image run: | docker pull usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} docker images # List images to verify the image is present locally + - name: Get Image Digest + id: get-digest + run: | + IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }}) + echo "IMAGE_DIGEST=${IMAGE_DIGEST}" >> $GITHUB_ENV + echo "::set-output name=image_digest::${IMAGE_DIGEST}" + - name: Generate SBOM with Retry Logic id: generate-sbom run: | @@ -139,7 +147,7 @@ jobs: run: | cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + "$IMAGE_DIGEST" - name: Sign SBOM with Cosign env: @@ -149,7 +157,8 @@ jobs: --key env://COSIGN_PRIVATE_KEY \ --predicate sbom.json \ --type https://spdx.dev/spdx-specification-2-2-pdf \ - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + "$IMAGE_DIGEST" + - name: Log out from Docker Hub run: docker logout From a1c721422cde5b4160e109177f1bbcddde8aca06 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 15 Nov 2024 15:47:20 +0200 Subject: [PATCH 26/65] fix --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4ce6291b..369fc928 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -153,7 +153,7 @@ jobs: env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - cosign attest \ + cosign attest -y \ --key env://COSIGN_PRIVATE_KEY \ --predicate sbom.json \ --type https://spdx.dev/spdx-specification-2-2-pdf \ From ecc20e106e67871ceee5a08c17d16707ace87ce9 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 15 Nov 2024 17:57:17 +0200 Subject: [PATCH 27/65] pre-release --- .github/workflows/release.yml | 131 +++++++++++++++------------------- 1 file changed, 56 insertions(+), 75 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 369fc928..52730c10 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,12 +41,9 @@ jobs: - name: Generate changelog id: changelog run: | - git log $(git describe --tags --abbrev=0)..HEAD -- . \ - --pretty=format:"- %s" > changelog.txt + git log $(git describe --tags --abbrev=0)..HEAD -- . --pretty=format:"- %s" > changelog.txt CHANGELOG=$(cat changelog.txt | jq -sRr @uri) - echo "changelog<> $GITHUB_ENV - echo "$CHANGELOG" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV + echo "CHANGELOG=$CHANGELOG" >> $GITHUB_ENV - name: Log in to Docker Hub uses: docker/login-action@v3 @@ -54,37 +51,23 @@ jobs: username: ${{ vars.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - # Temporarily comment out the build and push step to save time during testing - # - name: Multi-arch build and push to Docker Hub - # id: build-push - # uses: docker/build-push-action@v6 - # with: - # context: . - # file: ./Dockerfile - # platforms: linux/amd64,linux/arm64 - # push: true - # tags: | - # usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - # usabilitydynamics/udx-worker:latest + - name: Multi-arch build and push to Docker Hub + id: build-push + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + usabilitydynamics/udx-worker:latest - name: Install Trivy run: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ sudo sh -s -- -b /usr/local/bin - - name: Pull Docker Image for SBOM Generation - id: pull-image - run: | - docker pull usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - docker images # List images to verify the image is present locally - - - name: Get Image Digest - id: get-digest - run: | - IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }}) - echo "IMAGE_DIGEST=${IMAGE_DIGEST}" >> $GITHUB_ENV - echo "::set-output name=image_digest::${IMAGE_DIGEST}" - - name: Generate SBOM with Retry Logic id: generate-sbom run: | @@ -147,7 +130,7 @@ jobs: run: | cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ - "$IMAGE_DIGEST" + usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} - name: Sign SBOM with Cosign env: @@ -157,51 +140,49 @@ jobs: --key env://COSIGN_PRIVATE_KEY \ --predicate sbom.json \ --type https://spdx.dev/spdx-specification-2-2-pdf \ - "$IMAGE_DIGEST" - + usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} - name: Log out from Docker Hub run: docker logout - # Temporarily comment out GitHub release job to avoid duplicate releases - # github-release: - # runs-on: ubuntu-latest - # needs: docker-release - # permissions: - # contents: write - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - - # - name: Configure git for pushing - # run: | - # git config --global user.email "worker@udx.io" - # git config --global user.name "UDX Worker" - - # - name: Create GitHub Tag - # env: - # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - # run: | - # git tag ${{ needs.docker-release.outputs.semVer }} - # git push origin ${{ needs.docker-release.outputs.semVer }} - - # - name: Download SBOM Artifact - # uses: actions/download-artifact@v4 - # with: - # name: sbom - - # - name: Create GitHub release - # uses: softprops/action-gh-release@v2 - # with: - # tag_name: ${{ needs.docker-release.outputs.semVer }} - # body: | - # Release version ${{ needs.docker-release.outputs.semVer }}. - # [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). - # ${{ needs.docker-release.outputs.changelog }} - # draft: false - # prerelease: false - # files: sbom.json - # env: - # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + github-release: + runs-on: ubuntu-latest + needs: docker-release + permissions: + contents: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure git for pushing + run: | + git config --global user.email "worker@udx.io" + git config --global user.name "UDX Worker" + + - name: Create GitHub Tag + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + run: | + git tag ${{ needs.docker-release.outputs.semVer }} + git push origin ${{ needs.docker-release.outputs.semVer }} + + - name: Download SBOM Artifact + uses: actions/download-artifact@v4 + with: + name: sbom + + - name: Create GitHub release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.docker-release.outputs.semVer }} + body: | + Release version ${{ needs.docker-release.outputs.semVer }}. + [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). + ${{ needs.docker-release.outputs.changelog }} + draft: false + prerelease: false + files: sbom.json + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} From ef6399288bc4a04febfd63924386e94f05b22281 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Wed, 20 Nov 2024 17:46:32 +0200 Subject: [PATCH 28/65] fixes --- Dockerfile | 10 +++-- lib/environment.sh | 35 ++++++++--------- lib/worker_config.sh | 94 +++++++++++++++++++++++++++++++++----------- 3 files changed, 92 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7953c3e3..0ae19272 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update && \ apt-get install -y --no-install-recommends \ tzdata=2024a-3ubuntu1.1 \ - curl=8.5.0-2ubuntu10.4 \ + curl=8.5.0-2ubuntu10.5 \ bash=5.2.21-2ubuntu4 \ apt-utils=2.7.14build2 \ gettext=0.21-14ubuntu2 \ @@ -94,16 +94,18 @@ RUN mkdir -p /home/${USER}/.gnupg && \ mkdir -p /home/${USER}/etc /home/${USER}/.cd/configs && \ chown -R ${USER}:${USER} /home/${USER} +# Copy built-in worker.yml to the container +COPY ./src/configs/worker.yml /usr/src/app/src/configs/worker.yml + # Copy the bin, etc, and lib directories COPY ./etc/home /home/${USER}/etc -COPY ./src/configs /home/${USER}/.cd/configs COPY ./lib /usr/local/lib COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh COPY ./bin/test.sh /usr/local/bin/test.sh -# Set executable permissions and ownership for scripts +# Set executable permissions and ownership for scripts and configs RUN chmod +x /usr/local/lib/* /usr/local/bin/entrypoint.sh /usr/local/bin/test.sh && \ - chown -R ${USER}:${USER} /usr/local/lib /home/${USER}/etc /home/${USER}/.cd/configs + chown -R ${USER}:${USER} /usr/local/lib /usr/src/app/src/configs /home/${USER}/etc /home/${USER}/.cd/configs # Switch to non-root user USER ${USER} diff --git a/lib/environment.sh b/lib/environment.sh index b2f258e7..7f131292 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -1,7 +1,6 @@ #!/bin/bash # Include necessary modules -# shellcheck source=/dev/null source /usr/local/lib/utils.sh source /usr/local/lib/auth.sh source /usr/local/lib/secrets.sh @@ -10,48 +9,46 @@ source /usr/local/lib/worker_config.sh # Main function to coordinate environment setup configure_environment() { + log_info "Starting environment configuration..." + # Load and resolve the worker configuration local resolved_config - resolved_config=$(load_and_resolve_worker_config) - if [ -z "$resolved_config" ]; then - log_error "Failed to resolve worker configuration." + resolved_config=$(load_and_parse_config) + if [[ -z "$resolved_config" ]]; then + log_error "Configuration loading failed. Exiting..." return 1 fi - # Verify the config file exists at the expected path - local config_path - config_path=$(get_worker_config_path) - if [[ ! -f "$config_path" ]]; then - log_error "Configuration file not found at: $config_path" - return 1 - fi - log_info "Config file found: $config_path" + log_info "Worker configuration loaded successfully." - # Extract actors section and authenticate + # Extract and authenticate actors local actors - actors=$(get_worker_section "$resolved_config" "config.actors") + actors=$(get_config_section "$resolved_config" "actors") if [[ $? -eq 0 && -n "$actors" ]]; then + log_info "Authenticating actors from configuration..." if ! authenticate_actors "$actors"; then log_error "Failed to authenticate actors." return 1 fi else - log_info "No actors defined or required for authentication." + log_info "No actors defined in the configuration." fi - # Extract secrets section and fetch secrets if available + # Extract and fetch secrets local secrets - secrets=$(get_worker_section "$resolved_config" "config.secrets") + secrets=$(get_config_section "$resolved_config" "secrets") if [[ $? -eq 0 && -n "$secrets" ]]; then + log_info "Fetching secrets from configuration..." if ! fetch_secrets "$secrets"; then log_error "Failed to fetch secrets." return 1 fi else - log_info "No secrets found or required in the configuration." + log_info "No secrets defined in the configuration." fi - # Clean up actors and sensitive environment variables + # Perform cleanup + log_info "Cleaning up sensitive data..." if ! cleanup_actors; then log_error "Failed to clean up actors." return 1 diff --git a/lib/worker_config.sh b/lib/worker_config.sh index c4aeaa88..21574c2c 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -9,37 +9,56 @@ log_error() { echo "[ERROR] $1" } -# Function to get the path to the worker.yml configuration file -get_worker_config_path() { - local config_path="/home/${USER}/.cd/configs/worker.yml" - +# Paths for configurations +BUILT_IN_CONFIG="/usr/src/app/src/configs/worker.yml" +USER_CONFIG="/home/${USER}/.cd/configs/worker.yml" +MERGED_CONFIG="/home/${USER}/.cd/configs/merged_worker.yml" + +# Function to ensure configuration file exists +ensure_config_exists() { + local config_path="$1" if [[ ! -f "$config_path" ]]; then log_error "Configuration file not found: $config_path" return 1 fi - - echo "$config_path" } -# Function to load the worker configuration from YAML and convert it to JSON -load_and_resolve_worker_config() { - local config_path - config_path=$(get_worker_config_path) +# Function to merge the built-in and user-provided configuration files +merge_worker_configs() { + log_info "Merging worker configurations..." - # Check if the config_path retrieval was successful - if [[ -z "$config_path" ]]; then - return 1 + # Ensure the built-in config exists + ensure_config_exists "$BUILT_IN_CONFIG" || return 1 + + if [[ -f "$USER_CONFIG" ]]; then + log_info "User-provided configuration detected. Merging with the built-in configuration." + + # Merge user config with built-in config, prioritizing user config + if ! yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' "$BUILT_IN_CONFIG" "$USER_CONFIG" > "$MERGED_CONFIG"; then + log_error "Failed to merge configurations. yq returned an error." + return 1 + fi + else + log_info "No user-provided configuration detected. Using the built-in configuration." + cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG" fi - # Convert the YAML configuration to JSON using yq + log_info "Configuration merged successfully: $MERGED_CONFIG" +} + +# Function to load the merged configuration and convert it to JSON +load_and_parse_config() { + merge_worker_configs || return 1 + + # Convert the merged YAML configuration to JSON using yq local json_output - if ! json_output=$(yq eval -o=json "$config_path" 2>/dev/null); then - log_error "Failed to parse YAML from $config_path. yq returned an error." + if ! json_output=$(yq eval -o=json "$MERGED_CONFIG" 2>/dev/null); then + log_error "Failed to parse merged YAML from $MERGED_CONFIG. yq returned an error." return 1 fi if [[ -z "$json_output" ]]; then - log_error "YAML parsed to an empty JSON output." + log_error "Merged YAML parsed to an empty JSON output." return 1 fi @@ -47,7 +66,7 @@ load_and_resolve_worker_config() { } # Function to extract a specific section from the JSON configuration -get_worker_section() { +get_config_section() { local config_json="$1" local section="$2" @@ -58,18 +77,45 @@ get_worker_section() { # Attempt to extract the section and handle missing/null cases local extracted_section - extracted_section=$(echo "$config_json" | jq -r ".${section} // empty") + extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null) + + if [[ $? -ne 0 ]]; then + log_error "Failed to parse section '${section}' from configuration." + return 1 + fi - # Return success if the section is empty or null if [[ -z "$extracted_section" || "$extracted_section" == "null" ]]; then - return 0 # No error, section simply doesn't exist + log_info "Section '${section}' is not defined in the configuration." + return 0 fi echo "$extracted_section" } -# Example usage of the above functions -# You can comment this out if it’s just a library +# Debugging helper: Validate JSON structure +validate_json() { + local json="$1" + if ! echo "$json" | jq empty 2>/dev/null; then + log_error "Invalid JSON structure detected." + return 1 + fi +} + +# Example usage if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - configure_environment + log_info "Loading and resolving worker configuration..." + config_json=$(load_and_parse_config) || exit 1 + validate_json "$config_json" || exit 1 + log_info "Worker configuration loaded successfully." + + # Extract and process sections + actors=$(get_config_section "$config_json" "actors") + if [[ $? -eq 0 && -n "$actors" ]]; then + log_info "Actors loaded: $actors" + fi + + secrets=$(get_config_section "$config_json" "secrets") + if [[ $? -eq 0 && -n "$secrets" ]]; then + log_info "Secrets loaded: $secrets" + fi fi From 4fd12f317a544abccc1598ba117a66c18415c5bd Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Tue, 26 Nov 2024 18:17:59 +0200 Subject: [PATCH 29/65] improvements for worker_config module --- Dockerfile | 12 +++--- Makefile.variables | 2 +- lib/worker_config.sh | 93 +++++++------------------------------------- 3 files changed, 21 insertions(+), 86 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0ae19272..21a152f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -89,13 +89,11 @@ RUN groupadd -g ${GID} ${USER} && \ WORKDIR /home/${USER} # Create necessary directories and set permissions for GPG and other files -RUN mkdir -p /home/${USER}/.gnupg && \ - chmod 700 /home/${USER}/.gnupg && \ - mkdir -p /home/${USER}/etc /home/${USER}/.cd/configs && \ - chown -R ${USER}:${USER} /home/${USER} +RUN mkdir -p /etc/worker /home/${USER}/.cd/configs && \ + chown -R ${USER}:${USER} /home/${USER} /etc/worker # Copy built-in worker.yml to the container -COPY ./src/configs/worker.yml /usr/src/app/src/configs/worker.yml +COPY ./src/configs/worker.yml /etc/worker/worker.yml # Copy the bin, etc, and lib directories COPY ./etc/home /home/${USER}/etc @@ -104,8 +102,8 @@ COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh COPY ./bin/test.sh /usr/local/bin/test.sh # Set executable permissions and ownership for scripts and configs -RUN chmod +x /usr/local/lib/* /usr/local/bin/entrypoint.sh /usr/local/bin/test.sh && \ - chown -R ${USER}:${USER} /usr/local/lib /usr/src/app/src/configs /home/${USER}/etc /home/${USER}/.cd/configs +RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/test.sh && \ + chown -R ${USER}:${USER} /usr/local/lib /etc/worker /home/${USER}/etc /home/${USER}/.cd/configs # Switch to non-root user USER ${USER} diff --git a/Makefile.variables b/Makefile.variables index 56494988..0e558242 100644 --- a/Makefile.variables +++ b/Makefile.variables @@ -1,5 +1,5 @@ # Variables -IMAGE_NAME ?= udx-worker/udx-worker +IMAGE_NAME ?= usabilitydynamics/udx-worker TAG ?= latest DOCKER_IMAGE := $(IMAGE_NAME):$(TAG) CONTAINER_NAME ?= udx-worker-container diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 21574c2c..e9d4ba26 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -1,20 +1,9 @@ -#!/bin/bash - -# Utility functions for logging -log_info() { - echo "[INFO] $1" -} - -log_error() { - echo "[ERROR] $1" -} - # Paths for configurations -BUILT_IN_CONFIG="/usr/src/app/src/configs/worker.yml" -USER_CONFIG="/home/${USER}/.cd/configs/worker.yml" -MERGED_CONFIG="/home/${USER}/.cd/configs/merged_worker.yml" +BUILT_IN_CONFIG="/etc/worker/worker.yml" +USER_CONFIG="/home/udx/.cd/configs/worker.yml" +MERGED_CONFIG="/home/udx/.cd/configs/merged_worker.yml" -# Function to ensure configuration file exists +# Ensure configuration file exists ensure_config_exists() { local config_path="$1" if [[ ! -f "$config_path" ]]; then @@ -23,34 +12,37 @@ ensure_config_exists() { fi } -# Function to merge the built-in and user-provided configuration files +# Merge built-in and user-provided configurations merge_worker_configs() { log_info "Merging worker configurations..." - # Ensure the built-in config exists + # Ensure built-in config exists ensure_config_exists "$BUILT_IN_CONFIG" || return 1 + # Ensure target directory exists + mkdir -p "$(dirname "$MERGED_CONFIG")" + if [[ -f "$USER_CONFIG" ]]; then - log_info "User-provided configuration detected. Merging with the built-in configuration." + log_info "User configuration detected. Merging with the built-in configuration." - # Merge user config with built-in config, prioritizing user config + # Merge configurations, prioritizing user-provided values if ! yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' "$BUILT_IN_CONFIG" "$USER_CONFIG" > "$MERGED_CONFIG"; then log_error "Failed to merge configurations. yq returned an error." return 1 fi else - log_info "No user-provided configuration detected. Using the built-in configuration." + log_info "No user configuration provided. Using built-in configuration only." cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG" fi - log_info "Configuration merged successfully: $MERGED_CONFIG" + log_info "Merged configuration created successfully at $MERGED_CONFIG" } -# Function to load the merged configuration and convert it to JSON +# Load and parse the merged configuration load_and_parse_config() { merge_worker_configs || return 1 - # Convert the merged YAML configuration to JSON using yq + # Convert merged YAML to JSON local json_output if ! json_output=$(yq eval -o=json "$MERGED_CONFIG" 2>/dev/null); then log_error "Failed to parse merged YAML from $MERGED_CONFIG. yq returned an error." @@ -64,58 +56,3 @@ load_and_parse_config() { echo "$json_output" } - -# Function to extract a specific section from the JSON configuration -get_config_section() { - local config_json="$1" - local section="$2" - - if [[ -z "$config_json" ]]; then - log_error "Empty configuration JSON provided." - return 1 - fi - - # Attempt to extract the section and handle missing/null cases - local extracted_section - extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null) - - if [[ $? -ne 0 ]]; then - log_error "Failed to parse section '${section}' from configuration." - return 1 - fi - - if [[ -z "$extracted_section" || "$extracted_section" == "null" ]]; then - log_info "Section '${section}' is not defined in the configuration." - return 0 - fi - - echo "$extracted_section" -} - -# Debugging helper: Validate JSON structure -validate_json() { - local json="$1" - if ! echo "$json" | jq empty 2>/dev/null; then - log_error "Invalid JSON structure detected." - return 1 - fi -} - -# Example usage -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - log_info "Loading and resolving worker configuration..." - config_json=$(load_and_parse_config) || exit 1 - validate_json "$config_json" || exit 1 - log_info "Worker configuration loaded successfully." - - # Extract and process sections - actors=$(get_config_section "$config_json" "actors") - if [[ $? -eq 0 && -n "$actors" ]]; then - log_info "Actors loaded: $actors" - fi - - secrets=$(get_config_section "$config_json" "secrets") - if [[ $? -eq 0 && -n "$secrets" ]]; then - log_info "Secrets loaded: $secrets" - fi -fi From 0a287296806e32d924abedeab089923991d7e65a Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Tue, 26 Nov 2024 18:24:50 +0200 Subject: [PATCH 30/65] fix shell scripts --- .github/workflows/release.yml | 4 ++-- lib/environment.sh | 32 ++++++++++++++++++++++----- lib/worker_config.sh | 41 +++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52730c10..7a10d021 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: Release on: push: - # branches: - # - "latest" + branches: + - "latest" jobs: docker-release: diff --git a/lib/environment.sh b/lib/environment.sh index 7f131292..f350934d 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -1,11 +1,31 @@ #!/bin/bash -# Include necessary modules -source /usr/local/lib/utils.sh -source /usr/local/lib/auth.sh -source /usr/local/lib/secrets.sh -source /usr/local/lib/cleanup.sh -source /usr/local/lib/worker_config.sh +# Get the directory of this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source a file if it exists +source_if_exists() { + local file_path="$1" + if [[ -f "$file_path" ]]; then + # shellcheck disable=SC1090 + source "$file_path" + else + echo "[ERROR] Missing file: $file_path" >&2 + exit 1 + fi +} + +# Include necessary modules from the same directory +# shellcheck source=./utils.sh +source_if_exists "$SCRIPT_DIR/utils.sh" +# shellcheck source=./auth.sh +source_if_exists "$SCRIPT_DIR/auth.sh" +# shellcheck source=./secrets.sh +source_if_exists "$SCRIPT_DIR/secrets.sh" +# shellcheck source=./cleanup.sh +source_if_exists "$SCRIPT_DIR/cleanup.sh" +# shellcheck source=./worker_config.sh +source_if_exists "$SCRIPT_DIR/worker_config.sh" # Main function to coordinate environment setup configure_environment() { diff --git a/lib/worker_config.sh b/lib/worker_config.sh index e9d4ba26..55ac4f98 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -1,8 +1,28 @@ +#!/bin/bash +set -o nounset +set -o errexit +set -o pipefail + # Paths for configurations BUILT_IN_CONFIG="/etc/worker/worker.yml" USER_CONFIG="/home/udx/.cd/configs/worker.yml" MERGED_CONFIG="/home/udx/.cd/configs/merged_worker.yml" +# Utility functions for logging +log_info() { + echo "[INFO] $1" +} + +log_error() { + echo "[ERROR] $1" >&2 +} + +# Ensure `yq` is available +if ! command -v yq >/dev/null 2>&1; then + log_error "yq is not installed. Please ensure it is available in the PATH." + exit 1 +fi + # Ensure configuration file exists ensure_config_exists() { local config_path="$1" @@ -56,3 +76,24 @@ load_and_parse_config() { echo "$json_output" } + +# Debugging helper: Validate JSON structure +validate_json() { + local json="$1" + if ! echo "$json" | jq empty 2>/dev/null; then + log_error "Invalid JSON structure detected." + return 1 + fi +} + +# Example usage (when run as a standalone script) +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + log_info "Loading and resolving worker configuration..." + config_json=$(load_and_parse_config) || exit 1 + validate_json "$config_json" || exit 1 + log_info "Worker configuration loaded successfully." + + # Extract and process additional sections if needed + # actors=$(echo "$config_json" | jq -r ".actors // empty") + # secrets=$(echo "$config_json" | jq -r ".secrets // empty") +fi From d469ad8f15f566b36a59a28fab442a61a983d753 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Tue, 26 Nov 2024 18:49:05 +0200 Subject: [PATCH 31/65] fix image ref for build workflow --- .github/workflows/build-and-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 37d1c2e8..3929ddd0 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -4,6 +4,7 @@ on: push: branches-ignore: latest paths: + - '.github/workflows/build-and-test.yml' - 'Dockerfile' - 'bin/**' - 'lib/**' @@ -53,7 +54,7 @@ jobs: # Run the Trivy scan and capture the exit status trivy image --severity CRITICAL --exit-code 1 --quiet \ - udx-worker/udx-worker:latest | tee trivy.log | grep -v 'INFO' + usabilitydynamics/udx-worker:latest | tee trivy.log | grep -v 'INFO' scan_exit_code=$? # Check if CRITICAL vulnerabilities were detected @@ -83,7 +84,7 @@ jobs: - name: Trivy SBOM Generation run: | export TRIVY_DISABLE_VEX_NOTICE=true - trivy image --format spdx-json --output sbom.json udx-worker/udx-worker:latest 2>/dev/null + trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:latest 2>/dev/null echo "SBOM Top Packages Summary:" echo "| Package Name | Version |" From b05183e896846924faa03bbc212c7ef343ffaea4 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Wed, 27 Nov 2024 12:32:11 +0200 Subject: [PATCH 32/65] Dockerfile improvements --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 21a152f1..22a78823 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,13 +85,13 @@ RUN curl -Lso /usr/local/bin/bw "https://vault.bitwarden.com/download/?app=cli&p RUN groupadd -g ${GID} ${USER} && \ useradd -l -m -u ${UID} -g ${GID} -s /bin/bash ${USER} +# Prepare directories for the user and worker configuration +RUN mkdir -p /etc/worker /home/${USER}/.cd/bin /home/${USER}/.cd/configs && \ + chown -R ${UID}:${GID} /etc/worker /home/${USER}/.cd + # Switch to the user directory WORKDIR /home/${USER} -# Create necessary directories and set permissions for GPG and other files -RUN mkdir -p /etc/worker /home/${USER}/.cd/configs && \ - chown -R ${USER}:${USER} /home/${USER} /etc/worker - # Copy built-in worker.yml to the container COPY ./src/configs/worker.yml /etc/worker/worker.yml @@ -101,9 +101,9 @@ COPY ./lib /usr/local/lib COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh COPY ./bin/test.sh /usr/local/bin/test.sh -# Set executable permissions and ownership for scripts and configs +# Set permissions during build RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/test.sh && \ - chown -R ${USER}:${USER} /usr/local/lib /etc/worker /home/${USER}/etc /home/${USER}/.cd/configs + chown -R ${UID}:${GID} /usr/local/lib /etc/worker /home/${USER}/etc /home/${USER}/.cd # Switch to non-root user USER ${USER} From 3c4eb1c4e5228e1d6b10f1efbedb60899dd73b3d Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Wed, 27 Nov 2024 12:33:54 +0200 Subject: [PATCH 33/65] test release --- .github/workflows/release.yml | 126 +++++++++++++++++----------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a10d021..072edb48 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: Release on: push: - branches: - - "latest" + # branches: + # - "latest" jobs: docker-release: @@ -121,68 +121,68 @@ jobs: name: sbom path: sbom.json - - name: Install Cosign - uses: sigstore/cosign-installer@v3.7.0 - - - name: Sign Docker image with Cosign - env: - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - run: | - cosign sign -y \ - --key env://COSIGN_PRIVATE_KEY \ - usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} - - - name: Sign SBOM with Cosign - env: - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - run: | - cosign attest -y \ - --key env://COSIGN_PRIVATE_KEY \ - --predicate sbom.json \ - --type https://spdx.dev/spdx-specification-2-2-pdf \ - usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} + # - name: Install Cosign + # uses: sigstore/cosign-installer@v3.7.0 + + # - name: Sign Docker image with Cosign + # env: + # COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + # run: | + # cosign sign -y \ + # --key env://COSIGN_PRIVATE_KEY \ + # usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} + + # - name: Sign SBOM with Cosign + # env: + # COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + # run: | + # cosign attest -y \ + # --key env://COSIGN_PRIVATE_KEY \ + # --predicate sbom.json \ + # --type https://spdx.dev/spdx-specification-2-2-pdf \ + # usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} - name: Log out from Docker Hub run: docker logout - github-release: - runs-on: ubuntu-latest - needs: docker-release - permissions: - contents: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure git for pushing - run: | - git config --global user.email "worker@udx.io" - git config --global user.name "UDX Worker" - - - name: Create GitHub Tag - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - run: | - git tag ${{ needs.docker-release.outputs.semVer }} - git push origin ${{ needs.docker-release.outputs.semVer }} - - - name: Download SBOM Artifact - uses: actions/download-artifact@v4 - with: - name: sbom - - - name: Create GitHub release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ needs.docker-release.outputs.semVer }} - body: | - Release version ${{ needs.docker-release.outputs.semVer }}. - [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). - ${{ needs.docker-release.outputs.changelog }} - draft: false - prerelease: false - files: sbom.json - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + # github-release: + # runs-on: ubuntu-latest + # needs: docker-release + # permissions: + # contents: write + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + + # - name: Configure git for pushing + # run: | + # git config --global user.email "worker@udx.io" + # git config --global user.name "UDX Worker" + + # - name: Create GitHub Tag + # env: + # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + # run: | + # git tag ${{ needs.docker-release.outputs.semVer }} + # git push origin ${{ needs.docker-release.outputs.semVer }} + + # - name: Download SBOM Artifact + # uses: actions/download-artifact@v4 + # with: + # name: sbom + + # - name: Create GitHub release + # uses: softprops/action-gh-release@v2 + # with: + # tag_name: ${{ needs.docker-release.outputs.semVer }} + # body: | + # Release version ${{ needs.docker-release.outputs.semVer }}. + # [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). + # ${{ needs.docker-release.outputs.changelog }} + # draft: false + # prerelease: false + # files: sbom.json + # env: + # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} From 9bb2e553203b41e7b568b994282b00a033151536 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 12:17:53 +0200 Subject: [PATCH 34/65] worker config logic fixes --- .yamllint | 4 ++++ bin/test.sh | 54 ++---------------------------------------- lib/worker_config.sh | 47 ++++++++++++++++++++---------------- src/configs/worker.yml | 12 +++++----- 4 files changed, 38 insertions(+), 79 deletions(-) diff --git a/.yamllint b/.yamllint index 9f34b2e7..0e07d104 100644 --- a/.yamllint +++ b/.yamllint @@ -10,3 +10,7 @@ rules: comments-indentation: level: warning # Level should be either "error" or "warning" + + quotes: + level: warning + prefer: double \ No newline at end of file diff --git a/bin/test.sh b/bin/test.sh index 6b2500da..19c63b46 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -1,58 +1,8 @@ #!/bin/bash -echo "Starting validation tests..." - -# Dynamically redact sensitive environment variables for output -redact_sensitive_vars() { - local var_name="$1" - local value="$2" - local SENSITIVE_PATTERN="PASSWORD|SECRET|KEY|TOKEN" - if echo "$var_name" | grep -Eq "$SENSITIVE_PATTERN"; then - echo "$var_name=********" - else - echo "$var_name=$value" - fi -} - -# Extract required environment variables from the configuration -WORKER_CONFIG="/home/$USER/.cd/configs/worker.yml" -REQUIRED_VARS=$(yq e '.config.env | to_entries | .[].key' "$WORKER_CONFIG") -echo "Required environment variables: $REQUIRED_VARS" -# Ensure all required environment variables are set -for var in $REQUIRED_VARS; do - value=$(printenv "$var") - if [ -z "$value" ]; then - echo "Error: Environment variable $var is not set." - exit 1 - else - echo "Environment variable $var is set." - fi -done - -# Verify secrets are fetched (assuming secrets are set as environment variables) -SECRETS=$(yq e '.config.workerSecrets | to_entries | .[].key' "$WORKER_CONFIG") -for secret in $SECRETS; do - value=$(printenv "$secret") - if [ -z "$value" ]; then - echo "Error: Secret $secret is not resolved correctly." - exit 1 - else - echo "Secret $secret is resolved correctly." - fi -done +echo "Starting validation tests..." -# Confirm that sensitive actor variables are not set -ACTOR_VARS=$(yq e '.config.workerActors[] | to_entries | .[].value' "$WORKER_CONFIG" | grep -E "PASSWORD|SECRET|KEY|TOKEN") -for var in $ACTOR_VARS; do - var_name="${var//*\{/\{}" - var_name="${var_name//\}*/\}}" - if [ -n "$(printenv "$var_name")" ]; then - echo "Error: Sensitive variable $var_name is still set after cleanup." - exit 1 - else - echo "Sensitive variable $var_name is not set as expected." - fi -done +##@TODO: Add validation tests echo "All validation tests passed successfully." diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 55ac4f98..eef46647 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -1,7 +1,4 @@ #!/bin/bash -set -o nounset -set -o errexit -set -o pipefail # Paths for configurations BUILT_IN_CONFIG="/etc/worker/worker.yml" @@ -10,7 +7,7 @@ MERGED_CONFIG="/home/udx/.cd/configs/merged_worker.yml" # Utility functions for logging log_info() { - echo "[INFO] $1" + echo "[INFO] $1" >&2 } log_error() { @@ -62,19 +59,39 @@ merge_worker_configs() { load_and_parse_config() { merge_worker_configs || return 1 - # Convert merged YAML to JSON + # Suppress logs when parsing YAML into JSON local json_output if ! json_output=$(yq eval -o=json "$MERGED_CONFIG" 2>/dev/null); then log_error "Failed to parse merged YAML from $MERGED_CONFIG. yq returned an error." return 1 fi - if [[ -z "$json_output" ]]; then - log_error "Merged YAML parsed to an empty JSON output." + # Ensure output is valid JSON + validate_json "$json_output" || return 1 + + echo "$json_output" +} + +# Function to extract a specific section from the JSON configuration +get_config_section() { + local config_json="$1" + local section="$2" + + if [[ -z "$config_json" ]]; then + log_error "Empty configuration JSON provided." return 1 fi - echo "$json_output" + # Attempt to extract the section and handle missing/null cases + local extracted_section + extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null) + + if [[ $? -ne 0 ]]; then + log_error "Failed to parse section '${section}' from configuration." + return 1 + fi + + echo "$extracted_section" } # Debugging helper: Validate JSON structure @@ -84,16 +101,4 @@ validate_json() { log_error "Invalid JSON structure detected." return 1 fi -} - -# Example usage (when run as a standalone script) -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - log_info "Loading and resolving worker configuration..." - config_json=$(load_and_parse_config) || exit 1 - validate_json "$config_json" || exit 1 - log_info "Worker configuration loaded successfully." - - # Extract and process additional sections if needed - # actors=$(echo "$config_json" | jq -r ".actors // empty") - # secrets=$(echo "$config_json" | jq -r ".secrets // empty") -fi +} \ No newline at end of file diff --git a/src/configs/worker.yml b/src/configs/worker.yml index 7faa866d..f2700e65 100644 --- a/src/configs/worker.yml +++ b/src/configs/worker.yml @@ -2,15 +2,15 @@ kind: workerConfig version: udx.io/worker-v1/config config: - variables: - DOCKER_IMAGE_NAME: 'udx-worker' + env: + DOCKER_IMAGE_NAME: "udx-worker" actors: - type: gcp - creds: '${GCP_CREDS}' + creds: "${GCP_CREDS}" - type: azure - creds: '${AZURE_CREDS}' + creds: "${AZURE_CREDS}" - type: aws - creds: '${AWS_CREDS}' + creds: "${AWS_CREDS}" - type: bitwarden - creds: '${BITWARDEN_CREDS}' + creds: "${BITWARDEN_CREDS}" From 7cd25ed89b7bc41eb5467d85824fd5591f6c9916 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 12:18:03 +0200 Subject: [PATCH 35/65] test release --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 072edb48..ae920b43 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,7 +57,8 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm64 + # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: | usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} From 0a9e572d088763ee05afee89a18f4a8fb4989a87 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 12:25:56 +0200 Subject: [PATCH 36/65] vim package upgrade --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 22a78823..3639faac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && \ zip=3.0-13build1 \ unzip=6.0-28ubuntu4 \ nano=7.2-2build1 \ - vim=2:9.1.0016-1ubuntu7.4 && \ + vim2:9.1.0016-1ubuntu7.5 && \ ln -fs /usr/share/zoneinfo/$TZ /etc/localtime && \ dpkg-reconfigure --frontend noninteractive tzdata && \ apt-get clean && \ From fd5afc77b04a3817b8befa4cd0764bdace47a098 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 12:28:32 +0200 Subject: [PATCH 37/65] disable test release --- .github/workflows/release.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ae920b43..018a1052 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: Release on: push: - # branches: - # - "latest" + branches: + - "latest" jobs: docker-release: @@ -57,8 +57,7 @@ jobs: with: context: . file: ./Dockerfile - # platforms: linux/amd64,linux/arm64 - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true tags: | usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} From 4fbcd07a3d704fd1cabcb7762037836de94d0f88 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 12:28:48 +0200 Subject: [PATCH 38/65] code scanning fixes --- .yamllint | 6 +----- lib/worker_config.sh | 5 +---- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.yamllint b/.yamllint index 0e07d104..b9759552 100644 --- a/.yamllint +++ b/.yamllint @@ -9,8 +9,4 @@ rules: level: warning # Correct the level to "error" or "warning" instead of any invalid value comments-indentation: - level: warning # Level should be either "error" or "warning" - - quotes: - level: warning - prefer: double \ No newline at end of file + level: warning # Level should be either "error" or "warning" \ No newline at end of file diff --git a/lib/worker_config.sh b/lib/worker_config.sh index eef46647..c03401eb 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -83,10 +83,7 @@ get_config_section() { fi # Attempt to extract the section and handle missing/null cases - local extracted_section - extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null) - - if [[ $? -ne 0 ]]; then + if ! extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null); then log_error "Failed to parse section '${section}' from configuration." return 1 fi From 51b1ccf8e85afdb841c1df9e11d7cb20a7d552d2 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 12:29:56 +0200 Subject: [PATCH 39/65] tweak --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3639faac..9c4231b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && \ zip=3.0-13build1 \ unzip=6.0-28ubuntu4 \ nano=7.2-2build1 \ - vim2:9.1.0016-1ubuntu7.5 && \ + vim=2:9.1.0016-1ubuntu7.5 && \ ln -fs /usr/share/zoneinfo/$TZ /etc/localtime && \ dpkg-reconfigure --frontend noninteractive tzdata && \ apt-get clean && \ From 8d3b2a62f0ed962541fc452a3964f0811a981361 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 12:30:39 +0200 Subject: [PATCH 40/65] test release --- .github/workflows/release.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 018a1052..ae920b43 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: Release on: push: - branches: - - "latest" + # branches: + # - "latest" jobs: docker-release: @@ -57,7 +57,8 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm64 + # platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: | usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} From 8539c617d8e6408f80145070575f59ab173ad1f8 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 12:53:00 +0200 Subject: [PATCH 41/65] fix permissions for merged config --- Dockerfile | 4 +++- lib/worker_config.sh | 32 ++++++++++---------------------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9c4231b5..350c3073 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,7 +87,9 @@ RUN groupadd -g ${GID} ${USER} && \ # Prepare directories for the user and worker configuration RUN mkdir -p /etc/worker /home/${USER}/.cd/bin /home/${USER}/.cd/configs && \ - chown -R ${UID}:${GID} /etc/worker /home/${USER}/.cd + touch /home/${USER}/.cd/configs/merged_worker.yml && \ + chown -R ${UID}:${GID} /etc/worker /home/${USER}/.cd && \ + chmod 600 /home/${USER}/.cd/configs/merged_worker.yml # Switch to the user directory WORKDIR /home/${USER} diff --git a/lib/worker_config.sh b/lib/worker_config.sh index c03401eb..4567bbb0 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -23,8 +23,8 @@ fi # Ensure configuration file exists ensure_config_exists() { local config_path="$1" - if [[ ! -f "$config_path" ]]; then - log_error "Configuration file not found: $config_path" + if [[ ! -s "$config_path" ]]; then + log_error "Configuration file not found or empty: $config_path" return 1 fi } @@ -36,39 +36,36 @@ merge_worker_configs() { # Ensure built-in config exists ensure_config_exists "$BUILT_IN_CONFIG" || return 1 - # Ensure target directory exists - mkdir -p "$(dirname "$MERGED_CONFIG")" - + # If a user-provided configuration exists, merge it if [[ -f "$USER_CONFIG" ]]; then log_info "User configuration detected. Merging with the built-in configuration." - # Merge configurations, prioritizing user-provided values if ! yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' "$BUILT_IN_CONFIG" "$USER_CONFIG" > "$MERGED_CONFIG"; then log_error "Failed to merge configurations. yq returned an error." return 1 fi else log_info "No user configuration provided. Using built-in configuration only." - cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG" - fi - log_info "Merged configuration created successfully at $MERGED_CONFIG" + # Copy the built-in configuration to the merged configuration + if ! cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG"; then + log_error "Failed to copy built-in configuration to merged configuration." + return 1 + fi + fi } # Load and parse the merged configuration load_and_parse_config() { merge_worker_configs || return 1 - # Suppress logs when parsing YAML into JSON + # Parse the merged configuration into JSON local json_output if ! json_output=$(yq eval -o=json "$MERGED_CONFIG" 2>/dev/null); then log_error "Failed to parse merged YAML from $MERGED_CONFIG. yq returned an error." return 1 fi - # Ensure output is valid JSON - validate_json "$json_output" || return 1 - echo "$json_output" } @@ -90,12 +87,3 @@ get_config_section() { echo "$extracted_section" } - -# Debugging helper: Validate JSON structure -validate_json() { - local json="$1" - if ! echo "$json" | jq empty 2>/dev/null; then - log_error "Invalid JSON structure detected." - return 1 - fi -} \ No newline at end of file From 3b1c066cf0985e10ea437b08868c8f0f8e4fefa3 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 17:33:49 +0200 Subject: [PATCH 42/65] added worker vars->env logic --- lib/environment.sh | 7 +++++++ lib/worker_config.sh | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/lib/environment.sh b/lib/environment.sh index f350934d..c1ca2aa2 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -41,6 +41,13 @@ configure_environment() { log_info "Worker configuration loaded successfully." + # Export variables from the configuration + log_info "Exporting variables from configuration to environment..." + if ! export_variables_from_config "$resolved_config"; then + log_error "Failed to export variables." + return 1 + fi + # Extract and authenticate actors local actors actors=$(get_config_section "$resolved_config" "actors") diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 4567bbb0..90db44ee 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -69,6 +69,30 @@ load_and_parse_config() { echo "$json_output" } +# Export variables from the configuration +export_variables_from_config() { + local config_json="$1" + + log_info "Exporting variables from configuration..." + + # Extract the `variables` section + local variables + variables=$(echo "$config_json" | jq -r '.config.variables // empty') + if [[ -z "$variables" || "$variables" == "null" ]]; then + log_info "No variables found in the configuration." + return 0 + fi + + # Export each variable + echo "$variables" | jq -r 'to_entries[] | "\(.key)=\(.value)"' | while IFS= read -r line; do + # Use `eval` to safely split the key=value pair + local key="${line%%=*}" + local value="${line#*=}" + export "$key=$value" + log_info "Exported: $key=$value" + done +} + # Function to extract a specific section from the JSON configuration get_config_section() { local config_json="$1" From cf96edec4f5b131ee9cd8cafe5ddfbb244add1e6 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 17:42:04 +0200 Subject: [PATCH 43/65] fix config.env ref --- lib/worker_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 90db44ee..0e9afdfe 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -77,7 +77,7 @@ export_variables_from_config() { # Extract the `variables` section local variables - variables=$(echo "$config_json" | jq -r '.config.variables // empty') + variables=$(echo "$config_json" | jq -r '.config.env // empty') if [[ -z "$variables" || "$variables" == "null" ]]; then log_info "No variables found in the configuration." return 0 From 03c397fb4c201a721d199555371d031bdc877761 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 18:56:50 +0200 Subject: [PATCH 44/65] fix worker_config module --- lib/worker_config.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 0e9afdfe..bd827b80 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -83,14 +83,10 @@ export_variables_from_config() { return 0 fi - # Export each variable - echo "$variables" | jq -r 'to_entries[] | "\(.key)=\(.value)"' | while IFS= read -r line; do - # Use `eval` to safely split the key=value pair - local key="${line%%=*}" - local value="${line#*=}" - export "$key=$value" - log_info "Exported: $key=$value" - done + # Iterate over variables and export them into the main shell + while IFS="=" read -r key value; do + eval "export $key=\"$value\"" + done < <(echo "$variables" | jq -r 'to_entries[] | "\(.key)=\(.value)"') } # Function to extract a specific section from the JSON configuration @@ -104,10 +100,13 @@ get_config_section() { fi # Attempt to extract the section and handle missing/null cases - if ! extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null); then + local extracted_section + extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null) + + if [[ $? -ne 0 ]]; then log_error "Failed to parse section '${section}' from configuration." return 1 fi echo "$extracted_section" -} +} \ No newline at end of file From 008cb0512b480282c37bfec209577bf891c84c6f Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 19:21:37 +0200 Subject: [PATCH 45/65] test release --- .github/workflows/release.yml | 40 +++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ae920b43..61fe616a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -122,26 +122,26 @@ jobs: name: sbom path: sbom.json - # - name: Install Cosign - # uses: sigstore/cosign-installer@v3.7.0 - - # - name: Sign Docker image with Cosign - # env: - # COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - # run: | - # cosign sign -y \ - # --key env://COSIGN_PRIVATE_KEY \ - # usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} - - # - name: Sign SBOM with Cosign - # env: - # COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - # run: | - # cosign attest -y \ - # --key env://COSIGN_PRIVATE_KEY \ - # --predicate sbom.json \ - # --type https://spdx.dev/spdx-specification-2-2-pdf \ - # usabilitydynamics/udx-worker@${{ steps.build-push.outputs.digest }} + - name: Install Cosign + uses: sigstore/cosign-installer@v3.7.0 + + - name: Sign Docker image with Cosign + env: + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + run: | + cosign sign -y \ + --key env://COSIGN_PRIVATE_KEY \ + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + + - name: Sign SBOM with Cosign + env: + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + run: | + cosign attest -y \ + --key env://COSIGN_PRIVATE_KEY \ + --predicate sbom.json \ + --type https://spdx.dev/spdx-specification-2-2-pdf \ + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - name: Log out from Docker Hub run: docker logout From 18615b785a0c6e88d2263d129a9f946285bf3dc1 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 19:23:11 +0200 Subject: [PATCH 46/65] shell fix --- lib/worker_config.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/worker_config.sh b/lib/worker_config.sh index bd827b80..1c4dc08f 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -101,9 +101,7 @@ get_config_section() { # Attempt to extract the section and handle missing/null cases local extracted_section - extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null) - - if [[ $? -ne 0 ]]; then + if ! extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null); then log_error "Failed to parse section '${section}' from configuration." return 1 fi From f2c612cb77ff3036bc0b659d92141284244ff4fc Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 19:38:10 +0200 Subject: [PATCH 47/65] sign latest tag when release --- .github/workflows/release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 61fe616a..07e8bd6d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -132,6 +132,9 @@ jobs: cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + cosign sign -y \ + --key env://COSIGN_PRIVATE_KEY \ + usabilitydynamics/udx-worker:latest - name: Sign SBOM with Cosign env: @@ -142,6 +145,11 @@ jobs: --predicate sbom.json \ --type https://spdx.dev/spdx-specification-2-2-pdf \ usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + cosign attest -y \ + --key env://COSIGN_PRIVATE_KEY \ + --predicate sbom.json \ + --type https://spdx.dev/spdx-specification-2-2-pdf \ + usabilitydynamics/udx-worker:latest - name: Log out from Docker Hub run: docker logout From 04aa7b88ef7a2cf875509332d9da57375a047a97 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 28 Nov 2024 20:15:34 +0200 Subject: [PATCH 48/65] test release --- .github/workflows/release.yml | 87 +++++++++++++++-------------------- 1 file changed, 38 insertions(+), 49 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 07e8bd6d..7769bbaf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,26 +38,50 @@ jobs: useConfigFile: true configFilePath: ci/git-version.yml - - name: Generate changelog - id: changelog - run: | - git log $(git describe --tags --abbrev=0)..HEAD -- . --pretty=format:"- %s" > changelog.txt - CHANGELOG=$(cat changelog.txt | jq -sRr @uri) - echo "CHANGELOG=$CHANGELOG" >> $GITHUB_ENV + - name: Build Multi-Arch Docker Image + id: build-image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64 + push: false # Do not push yet + load: true # Ensure the built image is loaded into the local Docker environment + tags: | + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + usabilitydynamics/udx-worker:latest - name: Log in to Docker Hub uses: docker/login-action@v3 with: - username: ${{ vars.DOCKER_USERNAME }} + username: "usabilitydynamics" password: ${{ secrets.DOCKER_TOKEN }} - - name: Multi-arch build and push to Docker Hub - id: build-push + - name: Install Cosign + uses: sigstore/cosign-installer@v3.7.0 + + - name: Extract Image Digest + id: extract-digest + run: | + # Extract the digest of the built image + IMAGE_DIGEST=$(docker inspect usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} | grep -o 'sha256:[a-f0-9]\{64\}' | head -n 1) + echo "IMAGE_DIGEST=$IMAGE_DIGEST" >> $GITHUB_ENV + shell: bash + + - name: Sign Docker Image with Cosign + env: + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + IMAGE_DIGEST: ${{ env.IMAGE_DIGEST }} + run: | + # Correctly format the reference with @sha256 for Cosign + cosign sign -y \ + --key env://COSIGN_PRIVATE_KEY \ + usabilitydynamics/udx-worker@${IMAGE_DIGEST} + + - name: Push Signed Docker Images uses: docker/build-push-action@v6 with: context: . - file: ./Dockerfile - # platforms: linux/amd64,linux/arm64 platforms: linux/amd64 push: true tags: | @@ -73,46 +97,25 @@ jobs: id: generate-sbom run: | export TRIVY_DISABLE_VEX_NOTICE=true - - set +e # Disable exit on error for the retry logic - - # Retry logic for Trivy SBOM generation + set +e max_retries=10 attempt=1 success=false - while [ $attempt -le $max_retries ]; do echo "Generating SBOM, attempt $attempt..." - - # Run Trivy SBOM generation and capture the output and exit status output=$(trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} 2>&1) sbom_exit_code=$? - - # Check if SBOM generation was successful if [ $sbom_exit_code -eq 0 ]; then echo "SBOM generation successful." success=true break else - echo "SBOM generation encountered an error." - - # Extract and remove decimals from retry-after value - retry_after=$(echo "$output" | grep -oP 'retry-after: \K[0-9]+') - - # Default sleep time if retry-after is not found - if [ -z "$retry_after" ]; then - retry_after=120 # Default to 2 minutes if retry-after is missing - fi - - echo "Retrying in ${retry_after} seconds..." - sleep "$retry_after" + echo "Retrying in 120 seconds..." + sleep 120 attempt=$((attempt+1)) fi done - - # Exit if all retries fail without a successful SBOM generation if [ "$success" = false ]; then - echo "Failed to complete SBOM generation after $max_retries attempts." exit 1 fi @@ -122,20 +125,6 @@ jobs: name: sbom path: sbom.json - - name: Install Cosign - uses: sigstore/cosign-installer@v3.7.0 - - - name: Sign Docker image with Cosign - env: - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - run: | - cosign sign -y \ - --key env://COSIGN_PRIVATE_KEY \ - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - cosign sign -y \ - --key env://COSIGN_PRIVATE_KEY \ - usabilitydynamics/udx-worker:latest - - name: Sign SBOM with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} From 0deba0c062c3c4caa320802815347ced2e9675b8 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 16:57:56 +0200 Subject: [PATCH 49/65] test release --- .github/workflows/release.yml | 41 +++++++++-------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7769bbaf..6430d40d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,35 +38,24 @@ jobs: useConfigFile: true configFilePath: ci/git-version.yml - - name: Build Multi-Arch Docker Image - id: build-image + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: "usabilitydynamics" + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Push Signed Docker Images uses: docker/build-push-action@v6 with: context: . - file: ./Dockerfile platforms: linux/amd64 - push: false # Do not push yet - load: true # Ensure the built image is loaded into the local Docker environment + push: true tags: | usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} usabilitydynamics/udx-worker:latest - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: "usabilitydynamics" - password: ${{ secrets.DOCKER_TOKEN }} - - name: Install Cosign - uses: sigstore/cosign-installer@v3.7.0 - - - name: Extract Image Digest - id: extract-digest - run: | - # Extract the digest of the built image - IMAGE_DIGEST=$(docker inspect usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} | grep -o 'sha256:[a-f0-9]\{64\}' | head -n 1) - echo "IMAGE_DIGEST=$IMAGE_DIGEST" >> $GITHUB_ENV - shell: bash + uses: sigstore/cosign-installer@v3.7.0 - name: Sign Docker Image with Cosign env: @@ -76,17 +65,7 @@ jobs: # Correctly format the reference with @sha256 for Cosign cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ - usabilitydynamics/udx-worker@${IMAGE_DIGEST} - - - name: Push Signed Docker Images - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64 - push: true - tags: | - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - usabilitydynamics/udx-worker:latest + usabilitydynamics/udx-worker - name: Install Trivy run: | From 6419debedd2975ccd79be5ffe74540de57b5f18d Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 17:12:30 +0200 Subject: [PATCH 50/65] test release --- .github/workflows/release.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6430d40d..a613a06d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,8 @@ jobs: username: "usabilitydynamics" password: ${{ secrets.DOCKER_TOKEN }} - - name: Push Signed Docker Images + - name: Push Docker Image + id: docker_push uses: docker/build-push-action@v6 with: context: . @@ -54,18 +55,22 @@ jobs: usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} usabilitydynamics/udx-worker:latest - - name: Install Cosign - uses: sigstore/cosign-installer@v3.7.0 + - name: Extract Docker Hub Image Digest + run: | + echo "IMAGE_DIGEST=${{ steps.docker_push.outputs.digest }}" >> $GITHUB_ENV - name: Sign Docker Image with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - IMAGE_DIGEST: ${{ env.IMAGE_DIGEST }} run: | - # Correctly format the reference with @sha256 for Cosign cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ - usabilitydynamics/udx-worker + usabilitydynamics/udx-worker@${IMAGE_DIGEST} + + - name: Verify Cosign Signature + run: | + cosign verify \ + usabilitydynamics/udx-worker@${IMAGE_DIGEST} - name: Install Trivy run: | From 61eb0ed148c8058f47667a9b8e02044dfe66f177 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 17:16:04 +0200 Subject: [PATCH 51/65] test release --- .github/workflows/release.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a613a06d..be0e976d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,6 +59,9 @@ jobs: run: | echo "IMAGE_DIGEST=${{ steps.docker_push.outputs.digest }}" >> $GITHUB_ENV + - name: Install Cosign + uses: sigstore/cosign-installer@v3.7.0 + - name: Sign Docker Image with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} @@ -70,7 +73,7 @@ jobs: - name: Verify Cosign Signature run: | cosign verify \ - usabilitydynamics/udx-worker@${IMAGE_DIGEST} + usabilitydynamics/udx-worker@${IMAGE_DIGEST} - name: Install Trivy run: | From cd4496061f2bfce09ba9a7ef12de1a7ea213f646 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 17:23:38 +0200 Subject: [PATCH 52/65] test release --- .github/workflows/release.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be0e976d..dae9c1b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,15 +65,20 @@ jobs: - name: Sign Docker Image with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + IMAGE_DIGEST: ${{ env.IMAGE_DIGEST }} run: | + # Ensure signing targets the digest, not the tag cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ - usabilitydynamics/udx-worker@${IMAGE_DIGEST} + "usabilitydynamics/udx-worker@${IMAGE_DIGEST}" - name: Verify Cosign Signature + env: + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - cosign verify \ - usabilitydynamics/udx-worker@${IMAGE_DIGEST} + cosign verify -y \ + --key env://COSIGN_PRIVATE_KEY \ + "usabilitydynamics/udx-worker@${IMAGE_DIGEST}" - name: Install Trivy run: | From 18f66a3fd9223c9515fc39f039144c9cd4f769e8 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 17:35:19 +0200 Subject: [PATCH 53/65] test release --- .github/workflows/release.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dae9c1b7..6d8a4429 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,20 +44,20 @@ jobs: username: "usabilitydynamics" password: ${{ secrets.DOCKER_TOKEN }} - - name: Push Docker Image - id: docker_push - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64 - push: true - tags: | - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - usabilitydynamics/udx-worker:latest - - - name: Extract Docker Hub Image Digest + - name: Build Docker Image + run: | + docker buildx build \ + --platform linux/amd64 \ + --tag usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} \ + --tag usabilitydynamics/udx-worker:latest \ + --push \ + . + + - name: Retrieve Image Digest from Docker Hub + id: retrieve_digest run: | - echo "IMAGE_DIGEST=${{ steps.docker_push.outputs.digest }}" >> $GITHUB_ENV + DIGEST=$(docker manifest inspect usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} | jq -r '.config.digest') + echo "IMAGE_DIGEST=usabilitydynamics/udx-worker@${DIGEST}" >> $GITHUB_ENV - name: Install Cosign uses: sigstore/cosign-installer@v3.7.0 From 4703fdf74f74d8f213b2b4749d2c8a7c3f4aa89e Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 17:45:43 +0200 Subject: [PATCH 54/65] test release --- .github/workflows/release.yml | 44 ++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d8a4429..90f5e15a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,20 +44,29 @@ jobs: username: "usabilitydynamics" password: ${{ secrets.DOCKER_TOKEN }} - - name: Build Docker Image - run: | - docker buildx build \ - --platform linux/amd64 \ - --tag usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} \ - --tag usabilitydynamics/udx-worker:latest \ - --push \ - . + - name: Build and Push Docker Image + id: docker_push + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + push: true + tags: | + usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + usabilitydynamics/udx-worker:latest - name: Retrieve Image Digest from Docker Hub id: retrieve_digest + env: + DOCKER_USERNAME: "usabilitydynamics" + DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }} run: | - DIGEST=$(docker manifest inspect usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} | jq -r '.config.digest') + DIGEST=$(curl -sSL -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + -u "${DOCKER_USERNAME}:${DOCKER_PASSWORD}" \ + "https://registry-1.docker.io/v2/usabilitydynamics/udx-worker/manifests/${{ steps.gitversion.outputs.semVer }}" \ + -I | grep -i "Docker-Content-Digest" | awk '{print $2}' | tr -d '\r') echo "IMAGE_DIGEST=usabilitydynamics/udx-worker@${DIGEST}" >> $GITHUB_ENV + echo "Image Digest: ${DIGEST}" - name: Install Cosign uses: sigstore/cosign-installer@v3.7.0 @@ -65,20 +74,18 @@ jobs: - name: Sign Docker Image with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - IMAGE_DIGEST: ${{ env.IMAGE_DIGEST }} run: | - # Ensure signing targets the digest, not the tag cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ - "usabilitydynamics/udx-worker@${IMAGE_DIGEST}" + "${IMAGE_DIGEST}" - name: Verify Cosign Signature env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - cosign verify -y \ + cosign verify \ --key env://COSIGN_PRIVATE_KEY \ - "usabilitydynamics/udx-worker@${IMAGE_DIGEST}" + "${IMAGE_DIGEST}" - name: Install Trivy run: | @@ -121,16 +128,11 @@ jobs: env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | - cosign attest -y \ + cosign attest \ --key env://COSIGN_PRIVATE_KEY \ --predicate sbom.json \ --type https://spdx.dev/spdx-specification-2-2-pdf \ - usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} - cosign attest -y \ - --key env://COSIGN_PRIVATE_KEY \ - --predicate sbom.json \ - --type https://spdx.dev/spdx-specification-2-2-pdf \ - usabilitydynamics/udx-worker:latest + "${IMAGE_DIGEST}" - name: Log out from Docker Hub run: docker logout From 61ecf95c874e34329a2cbff2da909de7b546c44f Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 17:54:43 +0200 Subject: [PATCH 55/65] test release --- .github/workflows/release.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 90f5e15a..6e7c0113 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,10 +61,25 @@ jobs: DOCKER_USERNAME: "usabilitydynamics" DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }} run: | - DIGEST=$(curl -sSL -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + echo "Fetching digest for tag: ${{ steps.gitversion.outputs.semVer }}" + + # Query the Docker Registry API + RESPONSE_HEADERS=$(curl -sI -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ -u "${DOCKER_USERNAME}:${DOCKER_PASSWORD}" \ - "https://registry-1.docker.io/v2/usabilitydynamics/udx-worker/manifests/${{ steps.gitversion.outputs.semVer }}" \ - -I | grep -i "Docker-Content-Digest" | awk '{print $2}' | tr -d '\r') + "https://registry-1.docker.io/v2/usabilitydynamics/udx-worker/manifests/${{ steps.gitversion.outputs.semVer }}") + + # Debug headers + echo "Response headers:" + echo "${RESPONSE_HEADERS}" + + # Extract the Docker-Content-Digest + DIGEST=$(echo "${RESPONSE_HEADERS}" | grep -i "Docker-Content-Digest" | awk '{print $2}' | tr -d '\r') + + if [ -z "${DIGEST}" ]; then + echo "Failed to retrieve digest. Check if the tag exists or if authentication is correct." + exit 1 + fi + echo "IMAGE_DIGEST=usabilitydynamics/udx-worker@${DIGEST}" >> $GITHUB_ENV echo "Image Digest: ${DIGEST}" From 41a43b3651edf9990984279ca1404f1dc36359b8 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 17:59:59 +0200 Subject: [PATCH 56/65] test release --- .github/workflows/release.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e7c0113..6603a464 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,13 +59,25 @@ jobs: id: retrieve_digest env: DOCKER_USERNAME: "usabilitydynamics" - DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }} + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} run: | - echo "Fetching digest for tag: ${{ steps.gitversion.outputs.semVer }}" + echo "Fetching OAuth2 token for Docker Hub using token as password" + + # Request an OAuth2 access token + TOKEN=$(curl -s -X GET "https://auth.docker.io/token?service=registry.docker.io&scope=repository:usabilitydynamics/udx-worker:pull" \ + -u "${DOCKER_USERNAME}:${DOCKER_TOKEN}" | jq -r .token) + + if [ -z "${TOKEN}" ]; then + echo "Failed to retrieve OAuth2 token. Check Docker credentials." + exit 1 + fi + + echo "OAuth2 token retrieved successfully." - # Query the Docker Registry API - RESPONSE_HEADERS=$(curl -sI -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - -u "${DOCKER_USERNAME}:${DOCKER_PASSWORD}" \ + # Fetch the digest using the token + echo "Fetching digest for tag: ${{ steps.gitversion.outputs.semVer }}" + RESPONSE_HEADERS=$(curl -sI -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ "https://registry-1.docker.io/v2/usabilitydynamics/udx-worker/manifests/${{ steps.gitversion.outputs.semVer }}") # Debug headers @@ -83,6 +95,7 @@ jobs: echo "IMAGE_DIGEST=usabilitydynamics/udx-worker@${DIGEST}" >> $GITHUB_ENV echo "Image Digest: ${DIGEST}" + - name: Install Cosign uses: sigstore/cosign-installer@v3.7.0 From 6120d98be96b55e73b482ec91fd95c81b85bff51 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 18:12:51 +0200 Subject: [PATCH 57/65] test release --- .github/workflows/release.yml | 58 ++++++++--------------------------- 1 file changed, 13 insertions(+), 45 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6603a464..99015c97 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,46 +55,12 @@ jobs: usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} usabilitydynamics/udx-worker:latest - - name: Retrieve Image Digest from Docker Hub - id: retrieve_digest - env: - DOCKER_USERNAME: "usabilitydynamics" - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + - name: Extract Full Image Reference run: | - echo "Fetching OAuth2 token for Docker Hub using token as password" - - # Request an OAuth2 access token - TOKEN=$(curl -s -X GET "https://auth.docker.io/token?service=registry.docker.io&scope=repository:usabilitydynamics/udx-worker:pull" \ - -u "${DOCKER_USERNAME}:${DOCKER_TOKEN}" | jq -r .token) - - if [ -z "${TOKEN}" ]; then - echo "Failed to retrieve OAuth2 token. Check Docker credentials." - exit 1 - fi - - echo "OAuth2 token retrieved successfully." - - # Fetch the digest using the token - echo "Fetching digest for tag: ${{ steps.gitversion.outputs.semVer }}" - RESPONSE_HEADERS=$(curl -sI -H "Authorization: Bearer ${TOKEN}" \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - "https://registry-1.docker.io/v2/usabilitydynamics/udx-worker/manifests/${{ steps.gitversion.outputs.semVer }}") - - # Debug headers - echo "Response headers:" - echo "${RESPONSE_HEADERS}" - - # Extract the Docker-Content-Digest - DIGEST=$(echo "${RESPONSE_HEADERS}" | grep -i "Docker-Content-Digest" | awk '{print $2}' | tr -d '\r') - - if [ -z "${DIGEST}" ]; then - echo "Failed to retrieve digest. Check if the tag exists or if authentication is correct." - exit 1 - fi - - echo "IMAGE_DIGEST=usabilitydynamics/udx-worker@${DIGEST}" >> $GITHUB_ENV - echo "Image Digest: ${DIGEST}" - + IMAGE_DIGEST=${{ steps.docker_push.outputs.digest }} + FULL_REFERENCE="usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }}@${IMAGE_DIGEST}" + echo "FULL_REFERENCE=${FULL_REFERENCE}" >> $GITHUB_ENV + echo "Full Image Reference: ${FULL_REFERENCE}" - name: Install Cosign uses: sigstore/cosign-installer@v3.7.0 @@ -102,18 +68,19 @@ jobs: - name: Sign Docker Image with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + FULL_REFERENCE: ${{ env.FULL_REFERENCE }} run: | - cosign sign -y \ + cosign sign \ --key env://COSIGN_PRIVATE_KEY \ - "${IMAGE_DIGEST}" + "${FULL_REFERENCE}" - name: Verify Cosign Signature env: - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + FULL_REFERENCE: ${{ env.FULL_REFERENCE }} run: | cosign verify \ - --key env://COSIGN_PRIVATE_KEY \ - "${IMAGE_DIGEST}" + --key ci/pub.keys/cosign.pub \ + "${FULL_REFERENCE}" - name: Install Trivy run: | @@ -155,12 +122,13 @@ jobs: - name: Sign SBOM with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + FULL_REFERENCE: ${{ env.FULL_REFERENCE }} run: | cosign attest \ --key env://COSIGN_PRIVATE_KEY \ --predicate sbom.json \ --type https://spdx.dev/spdx-specification-2-2-pdf \ - "${IMAGE_DIGEST}" + "${FULL_REFERENCE}" - name: Log out from Docker Hub run: docker logout From bd9fb02b10114ad662cb827a7065e4c2f9af59e7 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 18:21:15 +0200 Subject: [PATCH 58/65] test release --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99015c97..d102e52e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,7 +70,7 @@ jobs: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} FULL_REFERENCE: ${{ env.FULL_REFERENCE }} run: | - cosign sign \ + cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ "${FULL_REFERENCE}" @@ -124,7 +124,7 @@ jobs: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} FULL_REFERENCE: ${{ env.FULL_REFERENCE }} run: | - cosign attest \ + cosign attest -y \ --key env://COSIGN_PRIVATE_KEY \ --predicate sbom.json \ --type https://spdx.dev/spdx-specification-2-2-pdf \ From 8be3aa5961076cdb960f91d49016504ef6eb4ebf Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 18:36:27 +0200 Subject: [PATCH 59/65] test release --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d102e52e..11886246 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,7 +70,8 @@ jobs: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} FULL_REFERENCE: ${{ env.FULL_REFERENCE }} run: | - cosign sign -y \ + cosign sign \ + --force \ --key env://COSIGN_PRIVATE_KEY \ "${FULL_REFERENCE}" From a50a7e7c2543bdb609f197624218b9daa6a4bc70 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 18:51:00 +0200 Subject: [PATCH 60/65] test release --- .github/workflows/release.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 11886246..6b10ed86 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,10 +57,13 @@ jobs: - name: Extract Full Image Reference run: | - IMAGE_DIGEST=${{ steps.docker_push.outputs.digest }} - FULL_REFERENCE="usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }}@${IMAGE_DIGEST}" - echo "FULL_REFERENCE=${FULL_REFERENCE}" >> $GITHUB_ENV - echo "Full Image Reference: ${FULL_REFERENCE}" + # Use the digest provided by docker/build-push-action + DIGEST=${{ steps.docker_push.outputs.digest }} + IMAGE_REFERENCE="usabilitydynamics/udx-worker@${DIGEST}" + + # Export the simplified reference + echo "IMAGE_REFERENCE=${IMAGE_REFERENCE}" >> $GITHUB_ENV + echo "Image Reference for Signing: ${IMAGE_REFERENCE}" - name: Install Cosign uses: sigstore/cosign-installer@v3.7.0 @@ -70,8 +73,7 @@ jobs: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} FULL_REFERENCE: ${{ env.FULL_REFERENCE }} run: | - cosign sign \ - --force \ + cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ "${FULL_REFERENCE}" From ef21e70e6e7d0617d40d66626dd8473bdd4d0fab Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 18:58:41 +0200 Subject: [PATCH 61/65] test release --- .github/workflows/release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b10ed86..06480f03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,7 +60,7 @@ jobs: # Use the digest provided by docker/build-push-action DIGEST=${{ steps.docker_push.outputs.digest }} IMAGE_REFERENCE="usabilitydynamics/udx-worker@${DIGEST}" - + # Export the simplified reference echo "IMAGE_REFERENCE=${IMAGE_REFERENCE}" >> $GITHUB_ENV echo "Image Reference for Signing: ${IMAGE_REFERENCE}" @@ -71,19 +71,19 @@ jobs: - name: Sign Docker Image with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - FULL_REFERENCE: ${{ env.FULL_REFERENCE }} + IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} run: | cosign sign -y \ --key env://COSIGN_PRIVATE_KEY \ - "${FULL_REFERENCE}" + "${IMAGE_REFERENCE}" - name: Verify Cosign Signature env: - FULL_REFERENCE: ${{ env.FULL_REFERENCE }} + IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} run: | cosign verify \ --key ci/pub.keys/cosign.pub \ - "${FULL_REFERENCE}" + "${IMAGE_REFERENCE}" - name: Install Trivy run: | @@ -125,13 +125,13 @@ jobs: - name: Sign SBOM with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - FULL_REFERENCE: ${{ env.FULL_REFERENCE }} + IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} run: | cosign attest -y \ --key env://COSIGN_PRIVATE_KEY \ --predicate sbom.json \ --type https://spdx.dev/spdx-specification-2-2-pdf \ - "${FULL_REFERENCE}" + "${IMAGE_REFERENCE}" - name: Log out from Docker Hub run: docker logout From e43d94ef9b1c016a0d23d13d67db7339dfb1a02c Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 19:18:39 +0200 Subject: [PATCH 62/65] test release --- .github/workflows/release.yml | 65 +++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 06480f03..d270f08f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,22 +68,22 @@ jobs: - name: Install Cosign uses: sigstore/cosign-installer@v3.7.0 - - name: Sign Docker Image with Cosign - env: - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} - run: | - cosign sign -y \ - --key env://COSIGN_PRIVATE_KEY \ - "${IMAGE_REFERENCE}" - - - name: Verify Cosign Signature - env: - IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} - run: | - cosign verify \ - --key ci/pub.keys/cosign.pub \ - "${IMAGE_REFERENCE}" + # - name: Sign Docker Image with Cosign + # env: + # COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + # IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} + # run: | + # cosign sign -y \ + # --key env://COSIGN_PRIVATE_KEY \ + # "${IMAGE_REFERENCE}" + + # - name: Verify Cosign Signature + # env: + # IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} + # run: | + # cosign verify \ + # --key ci/pub.keys/cosign.pub \ + # "${IMAGE_REFERENCE}" - name: Install Trivy run: | @@ -122,15 +122,42 @@ jobs: name: sbom path: sbom.json - - name: Sign SBOM with Cosign + # - name: Sign SBOM with Cosign + # env: + # COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + # IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} + # run: | + # cosign attest -y \ + # --key env://COSIGN_PRIVATE_KEY \ + # --predicate sbom.json \ + # --type https://spdx.dev/spdx-specification-2-2-pdf \ + # "${IMAGE_REFERENCE}" + + - name: Generate Provenance + run: | + echo '{ + "buildType": "https://mobyproject.org/buildkit@v1", + "builder": { + "id": "https://github.com/usabilitydynamics/udx-worker/actions/runs/${{ github.run_id }}" + }, + "invocation": { + "parameters": { + "context": ".", + "dockerfile": "./Dockerfile" + } + } + }' > provenance.json + echo "Provenance file created: provenance.json" + + - name: Sign Provenance with Cosign env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} run: | cosign attest -y \ --key env://COSIGN_PRIVATE_KEY \ - --predicate sbom.json \ - --type https://spdx.dev/spdx-specification-2-2-pdf \ + --predicate provenance.json \ + --type https://in-toto.io/Statement/v0.1 \ "${IMAGE_REFERENCE}" - name: Log out from Docker Hub From 289456572c2069dda0c5b4ba26f520543afe718c Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 5 Dec 2024 19:40:56 +0200 Subject: [PATCH 63/65] test release --- .github/workflows/release.yml | 60 ++--------------------------------- 1 file changed, 2 insertions(+), 58 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d270f08f..6b0debf9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,6 +51,8 @@ jobs: context: . platforms: linux/amd64 push: true + sbom: true + provenance: true tags: | usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} usabilitydynamics/udx-worker:latest @@ -65,26 +67,6 @@ jobs: echo "IMAGE_REFERENCE=${IMAGE_REFERENCE}" >> $GITHUB_ENV echo "Image Reference for Signing: ${IMAGE_REFERENCE}" - - name: Install Cosign - uses: sigstore/cosign-installer@v3.7.0 - - # - name: Sign Docker Image with Cosign - # env: - # COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - # IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} - # run: | - # cosign sign -y \ - # --key env://COSIGN_PRIVATE_KEY \ - # "${IMAGE_REFERENCE}" - - # - name: Verify Cosign Signature - # env: - # IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} - # run: | - # cosign verify \ - # --key ci/pub.keys/cosign.pub \ - # "${IMAGE_REFERENCE}" - - name: Install Trivy run: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ @@ -122,44 +104,6 @@ jobs: name: sbom path: sbom.json - # - name: Sign SBOM with Cosign - # env: - # COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - # IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} - # run: | - # cosign attest -y \ - # --key env://COSIGN_PRIVATE_KEY \ - # --predicate sbom.json \ - # --type https://spdx.dev/spdx-specification-2-2-pdf \ - # "${IMAGE_REFERENCE}" - - - name: Generate Provenance - run: | - echo '{ - "buildType": "https://mobyproject.org/buildkit@v1", - "builder": { - "id": "https://github.com/usabilitydynamics/udx-worker/actions/runs/${{ github.run_id }}" - }, - "invocation": { - "parameters": { - "context": ".", - "dockerfile": "./Dockerfile" - } - } - }' > provenance.json - echo "Provenance file created: provenance.json" - - - name: Sign Provenance with Cosign - env: - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} - run: | - cosign attest -y \ - --key env://COSIGN_PRIVATE_KEY \ - --predicate provenance.json \ - --type https://in-toto.io/Statement/v0.1 \ - "${IMAGE_REFERENCE}" - - name: Log out from Docker Hub run: docker logout From 4c3de12730728c7fbcfd78928257c6fe325ba973 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 6 Dec 2024 13:25:55 +0200 Subject: [PATCH 64/65] release workflow cleanup/improvements --- .github/workflows/release.yml | 109 +++++++++++++++++----------------- 1 file changed, 54 insertions(+), 55 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b0debf9..ced6cd55 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: Release on: push: - # branches: - # - "latest" + branches: + - "latest" jobs: docker-release: @@ -12,6 +12,10 @@ jobs: id-token: write contents: write + outputs: + semVer: ${{ steps.gitversion.outputs.semVer }} + changelog: ${{ steps.changelog.outputs.changelog }} + steps: - name: Checkout code uses: actions/checkout@v4 @@ -38,6 +42,18 @@ jobs: useConfigFile: true configFilePath: ci/git-version.yml + - name: Generate Changelog + id: changelog + run: | + git log $(git describe --tags --abbrev=0)..HEAD -- . \ + --pretty=format:"- %s" > changelog.txt + CHANGELOG=$(cat changelog.txt | jq -sRr @uri) + echo "changelog<> $GITHUB_ENV + echo "$CHANGELOG" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Log in to Docker Hub uses: docker/login-action@v3 with: @@ -52,21 +68,11 @@ jobs: platforms: linux/amd64 push: true sbom: true - provenance: true + provenance: true tags: | usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} usabilitydynamics/udx-worker:latest - - name: Extract Full Image Reference - run: | - # Use the digest provided by docker/build-push-action - DIGEST=${{ steps.docker_push.outputs.digest }} - IMAGE_REFERENCE="usabilitydynamics/udx-worker@${DIGEST}" - - # Export the simplified reference - echo "IMAGE_REFERENCE=${IMAGE_REFERENCE}" >> $GITHUB_ENV - echo "Image Reference for Signing: ${IMAGE_REFERENCE}" - - name: Install Trivy run: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ @@ -76,7 +82,6 @@ jobs: id: generate-sbom run: | export TRIVY_DISABLE_VEX_NOTICE=true - set +e max_retries=10 attempt=1 success=false @@ -107,44 +112,38 @@ jobs: - name: Log out from Docker Hub run: docker logout - # github-release: - # runs-on: ubuntu-latest - # needs: docker-release - # permissions: - # contents: write - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - - # - name: Configure git for pushing - # run: | - # git config --global user.email "worker@udx.io" - # git config --global user.name "UDX Worker" - - # - name: Create GitHub Tag - # env: - # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - # run: | - # git tag ${{ needs.docker-release.outputs.semVer }} - # git push origin ${{ needs.docker-release.outputs.semVer }} - - # - name: Download SBOM Artifact - # uses: actions/download-artifact@v4 - # with: - # name: sbom - - # - name: Create GitHub release - # uses: softprops/action-gh-release@v2 - # with: - # tag_name: ${{ needs.docker-release.outputs.semVer }} - # body: | - # Release version ${{ needs.docker-release.outputs.semVer }}. - # [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). - # ${{ needs.docker-release.outputs.changelog }} - # draft: false - # prerelease: false - # files: sbom.json - # env: - # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + github-release: + runs-on: ubuntu-latest + needs: docker-release + permissions: + contents: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure Git for Pushing + run: | + git config --global user.email "worker@udx.io" + git config --global user.name "UDX Worker" + + - name: Download SBOM Artifact + uses: actions/download-artifact@v4 + with: + name: sbom + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.docker-release.outputs.semVer }} + body: | + Release version ${{ needs.docker-release.outputs.semVer }}. + [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). + ${{ needs.docker-release.outputs.changelog }} + draft: false + prerelease: false + files: sbom.json + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} From 1951dfec11957f1006a8b2ceeecef1bb77e709fa Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 6 Dec 2024 13:42:38 +0200 Subject: [PATCH 65/65] enable multi-platform release --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ced6cd55..35bcaa8d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/amd64, linux/arm64 push: true sbom: true provenance: true