new fork - new version. a lot of changes.

.dockerignore

@@ -0,0 +1,2 @@
+*.deb
+.deb-created

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# it's not a bug that we aren't using all of hacking, ignore:
+# E241: multiple spaces after ','
+# E402: module level import not at top of file
+# E501: line too long (82 > 79 characters)
+# W601:	.has_key() is deprecated, use 'in'
+ignore = E241, E402, E501, W601

.gitignore

@@ -1,3 +1,5 @@
 __pycache__
 .envrc
-build
+build
+*.deb
+.deb-created

Dockerfile

@@ -1,44 +1,61 @@
-# Build exporter
-FROM python:3.9.16-bullseye AS exporter-builder
-
-WORKDIR /usr/src/
-
-# Install exporter requirements and build
-COPY requirements.txt /usr/src/
+ARG BASE_DIST
+
+##############################################################################
+#=======================| Wal-g exporter Builder Image |=====================#
+##############################################################################
+FROM python:$BASE_DIST AS exporter-builder
+ARG DEBIAN_FRONTEND=noninteractive
+COPY . /exporter
+WORKDIR /exporter
 RUN pip3 install -r requirements.txt
-ADD exporter.py /usr/src/
-RUN pyinstaller --name exporter \
-    --onefile exporter.py && \
-    mv dist/exporter wal-g-prometheus-exporter
-
-# Install wget and download wal-g
-ARG TARGETARCH
-RUN apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/* && \
-    if [ "${TARGETARCH}" = "arm64" ]; then \
-    export WALG_ARCH="aarch64"; \
-    else \
-    export WALG_ARCH="${TARGETARCH}"; \
-    fi && \
-    wget -O /wal-g-pg-ubuntu-20.04.tar.gz https://github.com/wal-g/wal-g/releases/download/v2.0.1/wal-g-pg-ubuntu-20.04-${WALG_ARCH}.tar.gz
-
-# Build final image
-FROM debian:11.6-slim
-COPY --from=exporter-builder /usr/src/wal-g-prometheus-exporter /usr/bin/
-COPY --from=exporter-builder /wal-g-pg-ubuntu-20.04.tar.gz /usr/bin/
-
-RUN apt-get update && \
-    apt-get install -y ca-certificates daemontools && \
-    apt-get upgrade -y -q && \
-    apt-get dist-upgrade -y -q && \
-    apt-get -y -q autoclean && \
-    apt-get -y -q autoremove
-
-RUN cd /usr/bin/ && \
-    tar -zxvf wal-g-pg-ubuntu-20.04.tar.gz && \
-    rm wal-g-pg-ubuntu-20.04.tar.gz && \
-    (mv wal-g-pg-ubuntu20.04-* wal-g || mv wal-g-pg-ubuntu-20.04-* wal-g)
-
-COPY scripts/entrypoint.sh /usr/bin/
-RUN chmod +x /usr/bin/entrypoint.sh
-
-ENTRYPOINT ["/usr/bin/entrypoint.sh"]
+RUN pyinstaller --name wal-g-exporter --onefile src/exporter.py
+
+##############################################################################
+#===========================| Wal-g Builder Image |==========================#
+##############################################################################
+FROM golang:$BASE_DIST AS walg-builder
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt update 
+RUN apt install -y \
+    ca-certificates \
+    cmake \
+    curl \
+    git \
+    gzip \
+    libbrotli-dev \
+    libsodium-dev \
+    make
+ARG WALG_RELEASE
+RUN git clone --depth 1 --branch $WALG_RELEASE https://github.com/wal-g/wal-g.git 
+WORKDIR /go/wal-g
+RUN go get ./...
+RUN make deps
+RUN make pg_install
+
+##############################################################################
+#==============================| Final Image |===============================#
+##############################################################################
+FROM debian:$BASE_DIST as final
+ENTRYPOINT ["/usr/bin/wal-g-exporter"]
+COPY --chmod=755 --from=walg-builder /wal-g /usr/bin/
+COPY --chmod=755 --from=exporter-builder /exporter/dist/wal-g-exporter /usr/bin/
+COPY Dockerfile /
+COPY README.md /
+
+##############################################################################
+#===========================| Deb Builder Image |============================#
+##############################################################################
+FROM ghcr.io/shipilovds/deb-builder as deb
+WORKDIR /build
+COPY --chmod=755 --from=walg-builder /wal-g deb/wal-g/
+COPY --chmod=755 --from=exporter-builder /exporter/dist/wal-g-exporter deb/wal-g-exporter/
+COPY deb deb
+ARG WALG_RELEASE
+ARG WALG_EXPORTER_RELEASE
+WORKDIR /build/deb/wal-g
+RUN sed -i -e "s|^Version:.*|Version: $WALG_RELEASE|g" -e "s|^Version: v|Version: |g" debian/control
+RUN dpkg-buildpackage -b
+WORKDIR /build/deb/wal-g-exporter
+RUN sed -i "s|^Version:.*|Version: $WALG_EXPORTER_RELEASE|g" debian/control
+RUN dpkg-buildpackage -b
+WORKDIR /build/deb

Makefile

@@ -0,0 +1,54 @@
+.PHONY: all docker docker-build docker-tag docker-push deb deb-docker deb-clean clean
+.EXPORT_ALL_VARIABLES:
+
+REVISION              ?= 1.0
+REGISTRY_USER         ?= shipilovds
+REGISTRY_PASSWORD     ?= CHANGE_ME
+REGISTRY_ADDR         ?= ghcr.io/$(REGISTRY_USER)
+BASE_DIST             ?= bookworm
+WALG_RELEASE          ?= v2.0.1
+WALG_RELEASE_FIXED    := $(shell echo $(WALG_RELEASE) | sed "s/v//")
+WALG_EXPORTER_RELEASE ?= 1.0.0
+WALG_EXPORTER_IMAGE   ?= $(REGISTRY_ADDR)/wal-g-exporter
+
+all:
+	@echo "There is no 'all' target here."
+	@echo "Select target:"
+	@echo "  - docker"
+	@echo "  - deb"
+	@echo "  - clean"
+
+docker: docker-build docker-tag docker-push
+
+docker-build:
+	docker compose build wal-g-exporter
+
+docker-tag:
+	docker tag $(WALG_EXPORTER_IMAGE):latest $(WALG_EXPORTER_IMAGE):$(REVISION)
+
+docker-push:
+	docker push $(WALG_EXPORTER_IMAGE):latest
+	docker push $(WALG_EXPORTER_IMAGE):$(REVISION)
+
+deb: deb-docker wal-g_$(WALG_RELEASE_FIXED)-1_amd64.deb wal-g-exporter_$(WALG_EXPORTER_RELEASE)-1_amd64.deb
+	@touch .deb-created
+
+deb-docker:
+	docker compose build deb
+
+wal-g_$(WALG_RELEASE_FIXED)-1_amd64.deb:
+	$(eval _CONTANER_ID := $(shell docker create deb))
+	docker cp $(_CONTANER_ID):/build/deb/wal-g_$(WALG_RELEASE_FIXED)-1_amd64.deb .
+	docker rm $(_CONTANER_ID)
+
+wal-g-exporter_$(WALG_EXPORTER_RELEASE)-1_amd64.deb:
+	$(eval _CONTANER_ID := $(shell docker create deb))
+	docker cp $(_CONTANER_ID):/build/deb/wal-g-exporter_$(WALG_EXPORTER_RELEASE)-1_amd64.deb .
+	docker rm $(_CONTANER_ID)
+
+deb-clean:
+	rm -f wal-g_$(WALG_RELEASE_FIXED)-1_amd64.deb
+	rm -f wal-g-exporter_$(WALG_EXPORTER_RELEASE)-1_amd64.deb
+	rm -f .deb-created
+
+clean: deb-clean

README.md

@@ -1,125 +1,64 @@
 # wal-g-exporter
----
 
-wal-g-exporter is a a Prometheus exporter for gathering WAL-G backup metrics for Postgres
-databases. It plays nice with the [Zalando Spilo](https://github.com/zalando/spilo)
-container especially and is configured by default to run within a Kubernetes environment
-as a sidecar container of Spilo (see [Features and limitations](#features-and-limitations)).
+wal-g-exporter is a Prometheus exporter for gathering WAL-G backup metrics for the PostgreSQL database.
 
 ## Features and limitations
----
 
-As mentioned above, wal-g-exporter plays well along Spilo. This includes also Patroni
-cluster configurations with Spilo. So wal-g-exporter is aware when running on a primary
-instance or not. It will not collect and export metrics for standby / follower instances
-and will get aware of a role change while running. This ensures, that you can always
-collect metrics from all exporters in the Postgres cluster with no unwanted cumulation
-of metrics between the exporters.
+#### Primary/Secondary Node Discovery
 
-For a list of all exported metrics and a description on then see the [metrics](docs/metrics.md)
-overview.
+wal-g-exporter is aware of whether it is running on a primary instance or not. It will not collect and export metrics
+for standby/follower instances and will become aware of a role change while running. This ensures that you
+can always collect metrics from all exporters in the PostgreSQL cluster with no unwanted accumulation of
+metrics between the exporters.
 
-wal-g-exporter relies on having all needed WAL-G environment variables within an envdir under
-`/run/etc/wal-e.d/env`. This is the default behavior of the Spilo image when you set the WAL-G
-environment variables (e.g. `AWS_ACCESS_KEY_ID`, `AWS_ENDPOINT`...). The entrypoint of the image
+#### http/oneshot Modes
+
+wal-g-exporter has two operating modes (`EXPORTER_OPS_MODE` environment variable):
+
+- `http` - run an HTTP server on `EXPORTER_HTTP_PORT` to make metrics accessible, like most exporters. *Default*
+- `oneshot` - write metrics to `EXPORTER_METRICS_FILE`. The service creates a metrics file and then shuts down.
+In Docker, you can use a volume to save the file on disk. It can then be taken by [node-exporter](https://github.com/prometheus/node_exporter)
+to serve your exporter metrics.
+
+> Open [this link](https://github.com/prometheus/node_exporter#textfile-collector) to read more about the textfile collector.
 
 ## Configuration
----
 
 The following environment variables can be used to configure wal-g-exporter.
 
-| Variable name         | description                                                                    | default value |
-| --------------------- | ------------------------------------------------------------------------------ | ------------- |
-| HTTP_PORT             | Port on which the http process of wal-g-exporter will run on to expose metrics | `9351`        |
-| PGHOST                | Hostname or IP of the Postgres instance to monitor metrics on                  | `localhost`   |
-| PGPORT                | Port of the Postgres instance to monitor metrics on                            | `5432`        |
-| PGUSER                | Username with which to connect to the Postgres instance                        | `postgres`    |
-| PGDATABASE            | Database name of the Postgres instance to connect to                           | `postgres`    |
-| PGPASSWORD            | Password of the above configured user                                          |               |
-| PGSSLMODE             | SSL mode of the Postgres connection                                            | `require`     |
-| WAL_G_SCRAPE_INTERVAL | Scrape interval of the exporter                                                | `60`          |
-
-## Running as a sidecar with Spilo
----
-
-Here you find an example sidecar configuration for wal-g-exporter to run along within a Spilo pod.
-The most of the configuration is straightforward with one thing to mention. To make the envdir
-`/run/etc/wal-e.d/env` shared between the Spilo container and wal-g-exporter, you need to mount
-the volume `walg` (as here named in this example) also to the Spilo container. Spilo will take
-care of the content in this directory.
-
-```yaml
-...
-      - env:
-        - name: POSTGRES_USER
-          value: postgres
-        - name: POSTGRES_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              key: password
-              name: postgres.postgres-backup-exporter.credentials.postgresql.acid.zalan.do
-        - name: PGHOST
-          value: 127.0.0.1
-        - name: PGPORT
-          value: "5432"
-        - name: PGPASSWORD
-          valueFrom:
-            secretKeyRef:
-              key: password
-              name: postgres.postgres-backup-exporter.credentials.postgresql.acid.zalan.do
-        - name: PGUSER
-          valueFrom:
-            secretKeyRef:
-              key: username
-              name: postgres.postgres-backup-exporter.credentials.postgresql.acid.zalan.do
-        image: ghcr.io/thedatabaseme/wal-g-prometheus-exporter:latest
-        imagePullPolicy: IfNotPresent
-        name: backup-exporter
-        volumeMounts:
-        - mountPath: /home/postgres/pgdata
-          name: pgdata
-        - mountPath: /run/etc
-          name: walg
-...
-      volumes:
-      - emptyDir:
-          medium: Memory
-        name: dshm
-      - emptyDir: {}
-        name: walg
-...
-```
-
-An example log output of wal-g-exporter looks like this:
-
-```
-2023-05-06 10:12:49,413 - Is NOT in recovery mode? True
-2023-05-06 10:12:49,413 - Connected to primary database
-2023-05-06 10:12:49,413 - Evaluating wal-g backups...
-2023-05-06 10:12:49,413 - Updating basebackup metrics...
-2023-05-06 10:12:49,439 - 4 basebackups found (first: 2023-05-06T09:57:42.19257Z, last: 2023-05-06T10:10:01.951179Z)
-2023-05-06 10:12:49,440 - Last basebackup duration: 1.3355910778045654
-2023-05-06 10:12:49,440 - Finished updating basebackup metrics...
-2023-05-06 10:12:49,440 - Updating WAL archive metrics...
-2023-05-06 10:12:49,478 - WAL integrity status is: OK
-2023-05-06 10:12:49,478 - Found 7 WAL archives in 1 timelines, 0 WAL archives missing
-2023-05-06 10:12:49,479 - Finished updating WAL archive metrics...
-2023-05-06 10:12:49,479 - Updating S3 disk usage...
-2023-05-06 10:12:49,508 - S3 diskusage in bytes: 31420173
-2023-05-06 10:12:49,508 - Finished updating S3 metrics...
-2023-05-06 10:12:49,508 - All metrics collected. Waiting for next update cycle...
-```
-
-## Grafana dashboard
----
-
-An example dashboard can be found under `grafana/dashboard.json`. Here is an example how it looks like:
-![alt text](assets/grafana_dashboard.png "Grafana dashboard")
-
-# Disclaimer
----
-
-This project has its roots in the [camptocamp/wal-g-prometheus-exporter](https://github.com/camptocamp/wal-g-prometheus-exporter)
-project. I took both the idea and quite some amount of code from there. Since this project hasn't been
-contributed in quite some time now and I had a different strategy on how to write code, I decided to
-start over and take the code as a basis. So, many kudos to camptocamp.
+| Variable Name         | Default Value          | Description                                                        |
+|-----------------------|------------------------|--------------------------------------------------------------------|
+| PGHOST                | localhost              | PostgreSQL host (set to '/var/run/postgresql/' to use unix socket) |
+| PGPORT                | 5432                   | PostgreSQL port                                                    |
+| PGUSER                | postgres               | PostgreSQL user                                                    |
+| POSTGRES_DB           | postgres               | PostgreSQL database name                                           |
+| POSTGRES_PASSWORD     |                        | PostgreSQL password (no default, must be set in env)               |
+| PGSSLMODE             | allow                  | PostgreSQL SSL mode                                                |
+| WAL_G_SCRAPE_INTERVAL | 60                     | Interval for scraping WAL-G metrics (for 'http' mode).             |
+| EXPORTER_OPS_MODE     | http                   | Operation mode for the exporter ('http' or 'oneshot').             |
+| EXPORTER_HTTP_PORT    | 9351                   | Port for HTTP service.                                             |
+| EXPORTER_UNIT_NAME    | wal-g                  | Unit name for the exporter. Becomes a label on every metric.       |
+| EXPORTER_METRICS_FILE | /prometheus/wal-g.prom | Path to the metrics file for Node exporter textfile collector.     |
+
+## Metrics
+
+| Metric name                           | Labels                     | Description                                                                                    |
+| ------------------------------------- | ---------------------------| ---------------------------------------------------------------------------------------------- |
+| walg_basebackup_count                 |                            | Number of basebackups stored on S3                                                             |
+| walg_oldest_basebackup                |                            | Timestamp of the oldest basebackup                                                             |
+| walg_newest_basebackup                |                            | Timestamp of the newest basebackup                                                             |
+| walg_last_basebackup_duration         |                            | Duration in seconds of the last basebackup                                                     |
+| walg_last_basebackup_throughput_bytes |                            | Throughput in bytes of the last basebackup                                                     |
+| walg_wal_archive_count                |                            | Number of WAL archives stored on S3                                                            |
+| walg_wal_archive_missing_count        |                            | Amount of missing WAL archives, will only be > 0 when `walg_wal_integrity_status` is `FAILURE` |
+| walg_wal_integrity_status             | `OK`, `WARNING`, `FAILURE` | Can be `1` or `0`, while `1` means that the integrity_status is true                           |
+| walg_last_upload                      | `basebackup`, `wal`        | Timestamp of the last upload to S3 of the respective label / file type                         |
+| walg_s3_diskusage                     |                            | Disk usage on S3 in byte for all backup / archive objects related to this Postgres instance    |
+
+> Every metric receives a label `unit` that can help you identify your instance if you use multiple on the host at the same time.
+>> The `unit` label takes it's data from the `EXPORTER_UNIT_NAME` environment variable!
+
+# Credits
+
+- [thedatabaseme](https://github.com/thedatabaseme) - Fork source author
+- [camptocamp](https://github.com/camptocamp) - Original project authors

deb/Dockerfile

@@ -0,0 +1,11 @@
+FROM debian:bookworm as deb-builder
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        devscripts \
+        debhelper \
+        dh-make \
+        fakeroot \
+        lintian
+COPY Dockerfile /

deb/wal-g-exporter/debian/changelog

@@ -0,0 +1,5 @@
+wal-g-exporter (1.0.0-1) stable; urgency=low
+
+  * Initial release.
+
+ -- Denis Shipilov <[email protected]> Wed, 28 Feb 2024 10:00:00 +0000

deb/wal-g-exporter/debian/compat

@@ -0,0 +1 @@
+12

deb/wal-g-exporter/debian/control

@@ -0,0 +1,10 @@
+Source: wal-g-exporter
+Section: utils
+Priority: optional
+Maintainer: Denis Shipilov <[email protected]>
+Build-Depends: debhelper (>= 12)
+
+Package: wal-g-exporter
+Architecture: amd64
+Description: wal-g-exporter is a Prometheus exporter for gathering WAL-G backup metrics for the PostgreSQL database.
+Version: REPLACE_ME

deb/wal-g-exporter/debian/install

@@ -0,0 +1 @@
+wal-g-exporter usr/bin/

deb/wal-g-exporter/debian/rules

@@ -0,0 +1,4 @@
+#!/usr/bin/make -f
+
+%:
+	dh $@