Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 9217d68

Browse files
bastjanbastjan
committedNov 12, 2024·
Rewrite dumper in Go
Differences: - All APIs are fully qualified in both the options (`--must-exist=certificates.cert-manager.io`, `--ignore=deployment.apps`) and the output files (`objects-Certificate.cert-manager.io.json`). This makes it possible to distinguish between objects with the same kind but different groups. See #47. - Resources without a list endpoint are ignored and do not cause and error or need to be explicitly ignored. - Ignore and must-exist options are now command line flags instead of files in `/usr/local/share`. Fixes #47 and #42.
1 parent 616829c commit 9217d68

35 files changed

+1463
-817
lines changed
 

‎.dockerignore

-4
This file was deleted.

‎.editorconfig

-26
This file was deleted.

‎.github/ISSUE_TEMPLATE/01_bug_report.md

-20
This file was deleted.

‎.github/ISSUE_TEMPLATE/02_feature_request.md

-18
This file was deleted.

‎.github/ISSUE_TEMPLATE/bug_report.yml

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: 🐛 Bug report
2+
description: Create a report to help us improve 🎉
3+
labels:
4+
- bug
5+
6+
body:
7+
- type: textarea
8+
id: description
9+
attributes:
10+
label: Description
11+
description: A clear and concise description of what the bug is.
12+
validations:
13+
required: true
14+
- type: textarea
15+
id: context
16+
attributes:
17+
label: Additional Context
18+
description: Add any other context about the problem here.
19+
validations:
20+
required: false
21+
- type: textarea
22+
id: logs
23+
attributes:
24+
label: Logs
25+
description: If applicable, add logs to help explain the bug.
26+
render: shell
27+
validations:
28+
required: false
29+
- type: textarea
30+
id: expected_behavior
31+
attributes:
32+
label: Expected Behavior
33+
description: A clear and concise description of what you expected to happen.
34+
validations:
35+
required: true
36+
- type: textarea
37+
id: reproduction_steps
38+
attributes:
39+
label: Steps To Reproduce
40+
description: Describe steps to reproduce the behavior
41+
validations:
42+
required: false
43+
- type: textarea
44+
id: version
45+
attributes:
46+
label: Versions
47+
placeholder: v1.2.3 [, Kubernetes 1.21]
48+
validations:
49+
required: true

‎.github/ISSUE_TEMPLATE/config.yml

-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1 @@
11
blank_issues_enabled: false
2-
contact_links:
3-
- name: ❓ Help and Support RocketChat Channel
4-
url: https://community.appuio.ch
5-
about: Please ask and answer questions here. 🏥

‎.github/ISSUE_TEMPLATE/feature_request.yml

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: 🚀 Feature request
2+
description: Suggest an idea for this project 💡
3+
labels:
4+
- enhancement
5+
6+
body:
7+
- type: textarea
8+
id: summary
9+
attributes:
10+
label: Summary
11+
value: |
12+
**As** role name\
13+
**I want** a feature or functionality\
14+
**So that** I get certain business value
15+
description: This user story helps us to quickly understand what this idea is about.
16+
validations:
17+
required: true
18+
- type: textarea
19+
id: context
20+
attributes:
21+
label: Context
22+
description: Add more information here. You are completely free regarding form and length.
23+
validations:
24+
required: true
25+
- type: textarea
26+
id: out_of_scope
27+
attributes:
28+
label: Out of Scope
29+
description: List aspects that are explicitly not part of this feature
30+
placeholder: |
31+
- ...
32+
- ...
33+
- ...
34+
validations:
35+
required: false
36+
- type: textarea
37+
id: links
38+
attributes:
39+
label: Further links
40+
description: URLs of relevant Git repositories, PRs, Issues, etc.
41+
placeholder: |
42+
- #567
43+
- https://kubernetes.io/docs/reference/
44+
validations:
45+
required: false
46+
- type: textarea
47+
id: acceptance_criteria
48+
attributes:
49+
label: Acceptance Criteria
50+
description: If you already have ideas what the detailed requirements are, please list them below in given-when-then expressions.
51+
placeholder: |
52+
- Given a precondition, when an action happens, then expect a result
53+
54+
```gherkin
55+
Given a precondition
56+
When an action happens
57+
Then expect a result
58+
```
59+
validations:
60+
required: false
61+
- type: textarea
62+
id: implementation_idea
63+
attributes:
64+
label: Implementation Ideas
65+
description: If applicable, shortly list possible implementation ideas
66+
validations:
67+
required: false

‎.github/PULL_REQUEST_TEMPLATE.md

+6-10
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,19 @@
1-
<!--
2-
Thank you for your pull request. Please provide a description above and
3-
review the checklist below.
1+
## Summary
42

5-
Contributors guide: ./CONTRIBUTING.md
6-
-->
3+
* Short summary of what's included in the PR
4+
* Give special note to breaking changes
75

86
## Checklist
9-
<!--
10-
Remove items that do not apply. For completed items, change [ ] to [x].
11-
-->
127

13-
- [ ] Keep pull requests small so they can be easily reviewed.
14-
- [ ] Update the documentation.
158
- [ ] Categorize the PR by setting a good title and adding one of the labels:
169
`bug`, `enhancement`, `documentation`, `change`, `breaking`, `dependency`
1710
as they show up in the changelog
11+
- [ ] Update tests.
1812
- [ ] Link this PR to related issues.
1913

2014
<!--
15+
Remove items that do not apply. For completed items, change [ ] to [x].
16+
2117
NOTE: these things are not required to open a PR and can be done afterwards,
2218
while the PR is open.
2319
-->

‎.github/changelog-configuration.json

+40-28
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,42 @@
11
{
2-
"pr_template": "- ${{TITLE}} (#${{NUMBER}})",
3-
"categories": [
4-
{
5-
"title": "## 🚀 Features",
6-
"labels": ["enhancement", "feature"]
7-
},
8-
{
9-
"title": "## 🛠️ Minor Changes",
10-
"labels": ["change"]
11-
},
12-
{
13-
"title": "## 🔎 Breaking Changes",
14-
"labels": ["breaking"]
15-
},
16-
{
17-
"title": "## 🐛 Fixes",
18-
"labels": ["bug", "fix"]
19-
},
20-
{
21-
"title": "## 📄 Documentation",
22-
"labels": ["documentation"]
23-
},
24-
{
25-
"title": "## 🔗 Dependency Updates",
26-
"labels": ["dependency"]
27-
}
28-
],
29-
"template": "${{CATEGORIZED_COUNT}} changes since ${{FROM_TAG}}\n\n${{CHANGELOG}}"
2+
"pr_template": "- ${{TITLE}} (#${{NUMBER}})",
3+
"categories": [
4+
{
5+
"title": "## 🚀 Features",
6+
"labels": [
7+
"enhancement"
8+
]
9+
},
10+
{
11+
"title": "## 🛠️ Minor Changes",
12+
"labels": [
13+
"change"
14+
]
15+
},
16+
{
17+
"title": "## 🔎 Breaking Changes",
18+
"labels": [
19+
"breaking"
20+
]
21+
},
22+
{
23+
"title": "## 🐛 Fixes",
24+
"labels": [
25+
"bug"
26+
]
27+
},
28+
{
29+
"title": "## 📄 Documentation",
30+
"labels": [
31+
"documentation"
32+
]
33+
},
34+
{
35+
"title": "## 🔗 Dependency Updates",
36+
"labels": [
37+
"dependency"
38+
]
39+
}
40+
],
41+
"template": "${{CATEGORIZED_COUNT}} changes since ${{FROM_TAG}}\n\n${{CHANGELOG}}"
3042
}

‎.github/workflows/build.yml

+17-90
Original file line numberDiff line numberDiff line change
@@ -1,105 +1,32 @@
1-
name: Docker image build
1+
name: Build
22

33
on:
4-
schedule:
5-
- cron: '0 7 * * *' # everyday at 07:00
6-
push:
7-
branches:
8-
- 'master'
9-
tags:
10-
- 'v*.*.*'
114
pull_request:
125
branches:
136
- master
7+
push:
8+
branches:
9+
- master
1410

1511
jobs:
16-
docker:
12+
go:
1713
runs-on: ubuntu-latest
1814
steps:
19-
- name: Checkout
20-
uses: actions/checkout@v4
21-
with:
22-
fetch-depth: 0
23-
24-
- name: Prepare
25-
id: prep
26-
run: |
27-
DOCKER_IMAGE=projectsyn/k8s-object-dumper
28-
VERSION=noop
29-
if [ "${{ github.event_name }}" = "schedule" ]; then
30-
VERSION=nightly
31-
elif [[ $GITHUB_REF == refs/tags/* ]]; then
32-
VERSION=${GITHUB_REF#refs/tags/}
33-
elif [[ $GITHUB_REF == refs/heads/* ]]; then
34-
VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
35-
if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then
36-
VERSION=edge
37-
fi
38-
elif [[ $GITHUB_REF == refs/pull/* ]]; then
39-
VERSION=pr-${{ github.event.number }}
40-
fi
41-
TAGS="${DOCKER_IMAGE}:${VERSION}"
42-
if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
43-
MINOR=${VERSION%.*}
44-
MAJOR=${MINOR%.*}
45-
TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR},${DOCKER_IMAGE}:${MAJOR},${DOCKER_IMAGE}:latest"
46-
elif [ "${{ github.event_name }}" = "push" ]; then
47-
TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}"
48-
fi
49-
echo "version=${VERSION}" >> ${GITHUB_ENV}
50-
echo "tags=${TAGS}" >> ${GITHUB_ENV}
51-
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> ${GITHUB_ENV}
52-
53-
- name: Set up QEMU
54-
uses: docker/setup-qemu-action@v3
15+
- uses: actions/checkout@v4
5516

56-
- name: Set up Docker Buildx
57-
uses: docker/setup-buildx-action@v3
17+
- name: Determine Go version from go.mod
18+
run: echo "GO_VERSION=$(grep "go 1." go.mod | cut -d " " -f 2)" >> $GITHUB_ENV
5819

59-
- name: Login to DockerHub
60-
if: github.event_name != 'pull_request'
61-
uses: docker/login-action@v3
20+
- uses: actions/setup-go@v5
6221
with:
63-
username: ${{ secrets.DOCKERHUB_USERNAME }}
64-
password: ${{ secrets.DOCKERHUB_TOKEN }}
22+
go-version: ${{ env.GO_VERSION }}
6523

66-
- name: Build and push
67-
id: docker_build
68-
uses: docker/build-push-action@v6
24+
- uses: actions/cache@v4
6925
with:
70-
context: .
71-
file: ./Dockerfile
72-
platforms: linux/amd64
73-
push: ${{ github.event_name != 'pull_request' }}
74-
tags: ${{ env.tags }}
75-
labels: |
76-
org.opencontainers.image.title=${{ github.event.repository.name }}
77-
org.opencontainers.image.description=${{ github.event.repository.description }}
78-
org.opencontainers.image.url=${{ github.event.repository.html_url }}
79-
org.opencontainers.image.source=${{ github.event.repository.clone_url }}
80-
org.opencontainers.image.version=${{ env.version }}
81-
org.opencontainers.image.created=${{ env.created }}
82-
org.opencontainers.image.revision=${{ github.sha }}
83-
org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }}
26+
path: ~/go/pkg/mod
27+
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
28+
restore-keys: |
29+
${{ runner.os }}-go-
8430
85-
- name: Build changelog from PRs with labels
86-
if: startsWith(github.ref, 'refs/tags/v')
87-
id: build_changelog
88-
uses: mikepenz/release-changelog-builder-action@v5
89-
with:
90-
configuration: ".github/changelog-configuration.json"
91-
# PreReleases still get a changelog, but the next full release gets a diff since the last full release,
92-
# combining possible changelogs of all previous PreReleases in between.
93-
# PreReleases show a partial changelog since last PreRelease.
94-
ignorePreReleases: "${{ !contains(github.ref, '-rc') }}"
95-
env:
96-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
97-
- name: Create Release
98-
if: startsWith(github.ref, 'refs/tags/v')
99-
uses: actions/create-release@v1
100-
with:
101-
tag_name: ${{ github.ref }}
102-
release_name: ${{ github.ref }}
103-
body: ${{steps.build_changelog.outputs.changelog}}
104-
env:
105-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
31+
- name: Run build
32+
run: make build

‎.github/workflows/lint.yml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Lint
2+
3+
on:
4+
pull_request: {}
5+
6+
jobs:
7+
lint:
8+
runs-on: ubuntu-latest
9+
steps:
10+
- uses: actions/checkout@v4
11+
12+
- name: Determine Go version from go.mod
13+
run: echo "GO_VERSION=$(grep "go 1." go.mod | cut -d " " -f 2)" >> $GITHUB_ENV
14+
15+
- uses: actions/setup-go@v5
16+
with:
17+
go-version: ${{ env.GO_VERSION }}
18+
19+
- uses: actions/cache@v4
20+
with:
21+
path: ~/go/pkg/mod
22+
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
23+
restore-keys: |
24+
${{ runner.os }}-go-
25+
26+
- name: Run linters
27+
run: make lint

‎.github/workflows/release.yml

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: Release
2+
3+
on:
4+
push:
5+
tags:
6+
- "*"
7+
8+
jobs:
9+
goreleaser:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v4
13+
with:
14+
fetch-depth: 0
15+
16+
- name: Determine Go version from go.mod
17+
run: echo "GO_VERSION=$(grep "go 1." go.mod | cut -d " " -f 2)" >> $GITHUB_ENV
18+
19+
- uses: actions/setup-go@v5
20+
with:
21+
go-version: ${{ env.GO_VERSION }}
22+
23+
- name: Set up QEMU
24+
uses: docker/setup-qemu-action@v3
25+
26+
- name: Set up Docker Buildx
27+
uses: docker/setup-buildx-action@v3
28+
29+
- uses: actions/cache@v4
30+
with:
31+
path: ~/go/pkg/mod
32+
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
33+
restore-keys: |
34+
${{ runner.os }}-go-
35+
36+
- name: Login to ghcr.io
37+
uses: docker/login-action@v3
38+
with:
39+
registry: ghcr.io
40+
username: ${{ github.repository_owner }}
41+
password: ${{ secrets.GITHUB_TOKEN }}
42+
43+
- name: Build changelog from PRs with labels
44+
id: build_changelog
45+
uses: mikepenz/release-changelog-builder-action@v5
46+
with:
47+
configuration: ".github/changelog-configuration.json"
48+
# PreReleases still get a changelog, but the next full release gets a diff since the last full release,
49+
# combining possible changelogs of all previous PreReleases in between.
50+
# PreReleases show a partial changelog since last PreRelease.
51+
ignorePreReleases: "${{ !contains(github.ref, '-rc') }}"
52+
outputFile: .github/release-notes.md
53+
env:
54+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
55+
56+
- name: Publish releases
57+
uses: goreleaser/goreleaser-action@v6
58+
with:
59+
args: release --release-notes .github/release-notes.md
60+
env:
61+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
62+
REGISTRY: ghcr.io
63+
IMAGE_NAME: ${{ github.repository }}

‎.github/workflows/test.yml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Test
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- master
7+
push:
8+
branches:
9+
- master
10+
11+
jobs:
12+
test:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v4
16+
17+
- name: Determine Go version from go.mod
18+
run: echo "GO_VERSION=$(grep "go 1." go.mod | cut -d " " -f 2)" >> $GITHUB_ENV
19+
20+
- uses: actions/setup-go@v5
21+
with:
22+
go-version: ${{ env.GO_VERSION }}
23+
24+
- uses: actions/cache@v4
25+
with:
26+
path: ~/go/pkg/mod
27+
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
28+
restore-keys: |
29+
${{ runner.os }}-go-
30+
31+
- name: Run tests
32+
run: make test

‎.gitignore

+13-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
1-
tmp/
2-
log/
3-
redhat/rpms
1+
# Goreleaser
2+
dist/
3+
.github/release-notes.md
44

5-
# Data directory used for local testing
6-
data/
5+
# Tools
6+
bin/
7+
8+
# Build
9+
k8s-object-dumper
10+
*.out
11+
12+
# Docs
13+
.cache/
14+
.public/

‎.goreleaser.yml

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# This is an example goreleaser.yaml file with some sane defaults.
2+
# Make sure to check the documentation at http://goreleaser.com
3+
builds:
4+
- env:
5+
- CGO_ENABLED=0 # this is needed otherwise the Docker image build is faulty
6+
goarch:
7+
- amd64
8+
- arm64
9+
goos:
10+
- linux
11+
goarm:
12+
- 8
13+
14+
archives:
15+
- format: binary
16+
name_template: "{{ .Binary }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
17+
18+
checksum:
19+
name_template: "checksums.txt"
20+
21+
snapshot:
22+
name_template: "{{ incpatch .Version }}-snapshot"
23+
24+
dockers:
25+
- goarch: amd64
26+
use: buildx
27+
build_flag_templates:
28+
- "--platform=linux/amd64"
29+
image_templates:
30+
- "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-amd64"
31+
32+
- goarch: arm64
33+
use: buildx
34+
build_flag_templates:
35+
- "--platform=linux/arm64/v8"
36+
image_templates:
37+
- "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-arm64"
38+
39+
docker_manifests:
40+
## ghcr.io
41+
# For prereleases, updating `latest` does not make sense.
42+
# Only the image for the exact version should be pushed.
43+
- name_template: "{{ if not .Prerelease }}{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:latest{{ end }}"
44+
image_templates:
45+
- "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-amd64"
46+
- "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-arm64"
47+
48+
- name_template: "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}"
49+
image_templates:
50+
- "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-amd64"
51+
- "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-arm64"
52+
53+
release:
54+
prerelease: auto

‎.yamllint.yml

-7
This file was deleted.

‎CODE_OF_CONDUCT.md

-4
This file was deleted.

‎CONTRIBUTING.md

-2
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,3 @@
22

33
This code repository is part of Project Syn and the contribution guide at
44
https://syn.tools/syn/contribution_guide.html does apply.
5-
6-
Submit Pull Requests at https://github.com/projectsyn/component-k8s-object-dumper/pulls.

‎Dockerfile

+10-40
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,13 @@
1-
FROM docker.io/debian:12.7-slim as base
1+
FROM docker.io/library/alpine:3.20 as runtime
22

3-
RUN apt-get update \
4-
&& apt-get install -y --no-install-recommends \
5-
bash \
6-
jq \
7-
less \
8-
moreutils \
9-
procps \
10-
&& rm -rf /var/lib/apt/lists/*
3+
RUN \
4+
apk add --update --no-cache \
5+
bash \
6+
curl \
7+
ca-certificates \
8+
tzdata
119

12-
FROM base as downloader
10+
ENTRYPOINT ["k8s-object-dumper"]
11+
COPY k8s-object-dumper /usr/bin/
1312

14-
RUN apt-get update \
15-
&& apt-get install -y --no-install-recommends \
16-
ca-certificates \
17-
curl \
18-
&& rm -rf /var/lib/apt/lists/*
19-
20-
ARG K8S_VERSION=v1.18.20
21-
22-
RUN curl -sLo /tmp/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" \
23-
&& chmod +x /tmp/kubectl
24-
25-
RUN curl -sLo /tmp/krossa.tar.gz https://github.com/appuio/krossa/releases/download/v0.0.4/krossa_0.0.4_linux_amd64.tar.gz \
26-
&& mkdir /tmp/krossa \
27-
&& tar -xzf /tmp/krossa.tar.gz --directory /tmp/krossa \
28-
&& ls /tmp/krossa
29-
30-
FROM base
31-
32-
RUN mkdir /data \
33-
&& chown 1001:0 /data
34-
35-
COPY --from=downloader /tmp/kubectl /usr/local/bin
36-
COPY --from=downloader /tmp/krossa/krossa /usr/local/bin
37-
COPY dump-objects /usr/local/bin
38-
COPY must-exist /usr/local/share/k8s-object-dumper/
39-
COPY known-to-fail /usr/local/share/k8s-object-dumper/
40-
41-
USER 1001
42-
ENTRYPOINT ["/usr/local/bin/dump-objects"]
43-
CMD ["-d", "/data"]
13+
USER 65536:0

‎Makefile

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Set Shell to bash, otherwise some targets fail with dash/zsh etc.
2+
SHELL := /bin/bash
3+
4+
# Disable built-in rules
5+
MAKEFLAGS += --no-builtin-rules
6+
MAKEFLAGS += --no-builtin-variables
7+
.SUFFIXES:
8+
.SECONDARY:
9+
.DEFAULT_GOAL := help
10+
11+
# General variables
12+
include Makefile.vars.mk
13+
14+
.PHONY: help
15+
help: ## Show this help
16+
@grep -E -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
17+
18+
.PHONY: build
19+
build: build-bin build-docker ## All-in-one build
20+
21+
.PHONY: build-bin
22+
build-bin: export CGO_ENABLED = 0
23+
build-bin: fmt vet ## Build binary
24+
@go build -o $(BIN_FILENAME)
25+
26+
.PHONY: build-docker
27+
build-docker: build-bin ## Build docker image
28+
$(DOCKER_CMD) build -t $(CONTAINER_IMG) .
29+
30+
.PHONY: run
31+
run:
32+
go run .
33+
34+
.PHONY: test
35+
test: envtest ## Test with envtest
36+
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race -coverprofile cover.out -covermode atomic ./...
37+
38+
.PHONY: fmt
39+
fmt: ## Run 'go fmt' against code
40+
go fmt ./...
41+
42+
.PHONY: vet
43+
vet: ## Run 'go vet' against code
44+
go vet ./...
45+
46+
.PHONY: lint
47+
lint: fmt vet generate ## All-in-one linting
48+
@echo 'Check for uncommitted changes ...'
49+
git diff --exit-code
50+
51+
.PHONY: generate
52+
generate: ## Generate additional code and artifacts
53+
@go generate ./...
54+
55+
.PHONY: clean
56+
clean: ## Cleans local build artifacts
57+
rm -rf dist .cache
58+
59+
LOCALBIN ?= $(shell pwd)/bin
60+
$(LOCALBIN):
61+
mkdir -p $(LOCALBIN)
62+
63+
.PHONY: envtest
64+
envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
65+
$(ENVTEST): $(LOCALBIN)
66+
test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest

‎Makefile.vars.mk

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
## These are some common variables for Make
2+
3+
PROJECT_ROOT_DIR = .
4+
PROJECT_NAME ?= k8s-object-dumper
5+
PROJECT_OWNER ?= projectsyn
6+
7+
## BUILD:go
8+
BIN_FILENAME ?= $(PROJECT_NAME)
9+
10+
## BUILD:docker
11+
DOCKER_CMD ?= docker
12+
13+
IMG_TAG ?= latest
14+
# Image URL to use all building/pushing image targets
15+
CONTAINER_IMG ?= local.dev/$(PROJECT_OWNER)/$(PROJECT_NAME):$(IMG_TAG)
16+
17+
LOCALBIN ?= $(shell pwd)/bin
18+
ENVTEST ?= $(LOCALBIN)/setup-envtest
19+
ENVTEST_K8S_VERSION = 1.28.3

‎README.md

+45-67
Original file line numberDiff line numberDiff line change
@@ -1,92 +1,70 @@
11
# K8s Object Dumper
22

3-
K8s Object Dumper allows to collect all objects from Kubernetes and write them into files.
4-
It is written to be used as a pre backup command for [K8up](https://k8up.io).
3+
Discover and dump all listable objects from a Kubernetes cluster into JSON files.
4+
Written to be used as a pre backup command for [K8up](https://k8up.io).
55

6-
This repository is part of Project Syn.
7-
For documentation on Project Syn, see https://syn.tools.
8-
9-
K8s Object Dumper consists of a shell script.
10-
This script uses the Kubernetes API to list each and every API and Kind known to the targeted cluster.
11-
It then dumps all those kinds to Json files (one file per kind).
12-
For easier restore, those dumped objects then get split up by namespace and kind.
6+
## Usage
137

14-
The resulting structure looks like the following:
8+
The project uses controller-runtime's configuration discovery to find the Kubernetes API server.
159

16-
```
17-
├─ objects-<kind>.json
18-
├─ …
19-
└─ split/
20-
├─ <namespace>/
21-
| ├─ __all__.json
22-
| ├─ <kind>.json
23-
| └─ …
24-
└─ …
25-
```
2610

27-
## Usage
2811

29-
Using Docker
12+
### Dump to STDOUT
3013

3114
```bash
32-
docker run --rm -v "/path/to/kubeconfig:/kubeconfig" -e KUBECONFIG=/kubeconfig -v "${PWD}/data:/data" projectsyn/k8s-object-dumper:latest -d /data > objects.tar.gz
15+
$ k8s-object-dumper
16+
{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"v1", ...}]}
17+
{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"apps/v1", ...}]}
3318
```
3419

35-
Using Kubernetes (with K8up)
36-
37-
See [Commodore Component: cluster-backup](https://github.com/projectsyn/component-cluster-backup).
38-
39-
## Configuration
40-
41-
The `dump-objects` scripts reads configuration from two files.
20+
### Dump to a directory
4221

43-
`/usr/local/share/k8s-object-dumper/must-exists` contains a list of types that must exist within the list of discovered types.
44-
This is a safeguard helping to detect failure of the discover mechanism.
45-
Types must be all lower case and plural.
46-
One type per line.
22+
```bash
23+
$ k8s-object-dumper -d dir
24+
```
4725

48-
Example:
26+
Will result in the following directory structure:
4927

5028
```
51-
configmaps
52-
daemonsets
53-
deployments
54-
endpoints
55-
ingresses
56-
jobs
57-
namespaces
58-
nodes
59-
persistentvolumeclaims
60-
persistentvolumes
61-
replicasets
62-
roles
63-
secrets
64-
serviceaccounts
65-
services
66-
statefulsets
29+
└─ dir/
30+
├─ objects-<kind>[.<group>].json
31+
├─ …
32+
└─ split/
33+
├─ <namespace>/
34+
| ├─ __all__.json
35+
| ├─ <kind>[.<group>].json
36+
| └─ …
37+
└─ …
6738
```
6839

69-
Some types can not be exported and the script will return an error for them.
70-
Those errors can be suppressed by placing those types in `/usr/local/share/k8s-object-dumper/known-to-fail`.
71-
Like `must-exist` types are listed line by line but in addition Bash regular expressions can be used.
40+
### Advanced usage
41+
42+
```bash
43+
# Fail if a Pods, Deployments or AlertingRules are not found
44+
$ k8s-object-dumper \
45+
-must-exist=pods \
46+
-must-exist=deployments.apps \
47+
-must-exist=alertingrules.monitoring.openshift.io
48+
# Ignore all Secrets and all cert-manager objects
49+
$ k8s-object-dumper \
50+
-ignore=secrets \
51+
-ignore=.+cert-manager.io
52+
```
7253

54+
## Development
7355

74-
Example:
56+
The project uses [envtest](https://book.kubebuilder.io/reference/envtest) to run tests against a real Kubernetes API server.
7557

58+
```bash
59+
$ make test
7660
```
77-
.+mutators
78-
.+reviews
79-
.+validators
80-
bindings
81-
deploymentconfigrollbacks
82-
imagesignatures
83-
imagestream.+
84-
mutations
85-
useridentitymappings
86-
validations
87-
```
8861

89-
To enable informative logging output for non-error cases, set the `-D` argument.
62+
## Differences to the original `bash` version `< 0.3.0`
63+
64+
- All APIs are fully qualified in both the options (`--must-exist=certificates.cert-manager.io`, `--ignore=deployment.apps`) and the output files (`objects-Certificate.cert-manager.io.json`).
65+
This makes it possible to distinguish between objects with the same kind but different groups. See https://github.com/projectsyn/k8s-object-dumper/issues/47.
66+
- Resources without a list endpoint are ignored, do not cause an error, and don't need to be explicitly ignored.
67+
- Ignore and must-exist options are now command line flags instead of files in `/usr/local/share`.
9068

9169
## Contributing and license
9270

‎dump-objects

-456
This file was deleted.

‎go.mod

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
module github.com/projectsyn/k8s-object-dumper
2+
3+
go 1.23.2
4+
5+
require (
6+
github.com/stretchr/testify v1.9.0
7+
go.uber.org/multierr v1.11.0
8+
k8s.io/api v0.31.0
9+
k8s.io/apimachinery v0.31.0
10+
k8s.io/client-go v0.31.0
11+
sigs.k8s.io/controller-runtime v0.19.0
12+
)
13+
14+
require (
15+
github.com/beorn7/perks v1.0.1 // indirect
16+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
17+
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
18+
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
19+
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
20+
github.com/fsnotify/fsnotify v1.7.0 // indirect
21+
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
22+
github.com/go-logr/logr v1.4.2 // indirect
23+
github.com/go-openapi/jsonpointer v0.19.6 // indirect
24+
github.com/go-openapi/jsonreference v0.20.2 // indirect
25+
github.com/go-openapi/swag v0.22.4 // indirect
26+
github.com/gogo/protobuf v1.3.2 // indirect
27+
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
28+
github.com/golang/protobuf v1.5.4 // indirect
29+
github.com/google/gnostic-models v0.6.8 // indirect
30+
github.com/google/go-cmp v0.6.0 // indirect
31+
github.com/google/gofuzz v1.2.0 // indirect
32+
github.com/google/uuid v1.6.0 // indirect
33+
github.com/imdario/mergo v0.3.6 // indirect
34+
github.com/josharian/intern v1.0.0 // indirect
35+
github.com/json-iterator/go v1.1.12 // indirect
36+
github.com/mailru/easyjson v0.7.7 // indirect
37+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
38+
github.com/modern-go/reflect2 v1.0.2 // indirect
39+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
40+
github.com/pkg/errors v0.9.1 // indirect
41+
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
42+
github.com/prometheus/client_golang v1.19.1 // indirect
43+
github.com/prometheus/client_model v0.6.1 // indirect
44+
github.com/prometheus/common v0.55.0 // indirect
45+
github.com/prometheus/procfs v0.15.1 // indirect
46+
github.com/spf13/pflag v1.0.5 // indirect
47+
github.com/x448/float16 v0.8.4 // indirect
48+
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
49+
golang.org/x/net v0.26.0 // indirect
50+
golang.org/x/oauth2 v0.21.0 // indirect
51+
golang.org/x/sys v0.21.0 // indirect
52+
golang.org/x/term v0.21.0 // indirect
53+
golang.org/x/text v0.16.0 // indirect
54+
golang.org/x/time v0.3.0 // indirect
55+
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
56+
google.golang.org/protobuf v1.34.2 // indirect
57+
gopkg.in/inf.v0 v0.9.1 // indirect
58+
gopkg.in/yaml.v2 v2.4.0 // indirect
59+
gopkg.in/yaml.v3 v3.0.1 // indirect
60+
k8s.io/apiextensions-apiserver v0.31.0 // indirect
61+
k8s.io/klog/v2 v2.130.1 // indirect
62+
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
63+
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
64+
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
65+
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
66+
sigs.k8s.io/yaml v1.4.0 // indirect
67+
)

‎go.sum

+194
Large diffs are not rendered by default.

‎internal/pkg/discovery/discovery.go

+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
package discovery
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"regexp"
8+
"slices"
9+
"strings"
10+
11+
"go.uber.org/multierr"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
14+
"k8s.io/apimachinery/pkg/runtime/schema"
15+
"k8s.io/apimachinery/pkg/util/sets"
16+
"k8s.io/client-go/discovery"
17+
"k8s.io/client-go/dynamic"
18+
"k8s.io/client-go/rest"
19+
)
20+
21+
type DiscoveryOptions struct {
22+
BatchSize int64
23+
LogWriter io.Writer
24+
25+
// MustExistResources is a list of resources that must exist in the cluster.
26+
// This can be used as a sanity check to ensure that the discovery process is working as expected.
27+
// If a resource does not exist, the discovery process will fail.
28+
// If the list is empty, no resources are required to exist.
29+
MustExistResources []string
30+
31+
// IgnoreResources is a list of resources to ignore during discovery.
32+
IgnoreResources []*regexp.Regexp
33+
}
34+
35+
// GetBatchSize returns the set batch size for listing objects or the default.
36+
func (opts DiscoveryOptions) GetBatchSize() int64 {
37+
if opts.BatchSize == 0 {
38+
return 500
39+
}
40+
return opts.BatchSize
41+
}
42+
43+
// GetLogWriter returns the set batch size for listing objects or io.Discard as default.
44+
func (opts DiscoveryOptions) GetLogWriter() io.Writer {
45+
if opts.LogWriter == nil {
46+
return io.Discard
47+
}
48+
return opts.LogWriter
49+
}
50+
51+
// DiscoverObjects discovers all objects in the cluster and calls the provided callback for each list of objects.
52+
// The callback can be called multiple times with the same object kind.
53+
// Objects are unique in general, but the callback should be able to handle duplicates.
54+
// Some API servers do not implement list batching correctly and thus might introduce duplicates.
55+
func DiscoverObjects(ctx context.Context, conf *rest.Config, cb func(*unstructured.UnstructuredList) error, opts DiscoveryOptions) error {
56+
batchSize := opts.GetBatchSize()
57+
logWriter := opts.GetLogWriter()
58+
59+
dc, err := discovery.NewDiscoveryClientForConfig(conf)
60+
if err != nil {
61+
return fmt.Errorf("failed to create discovery client: %w", err)
62+
}
63+
dynClient, err := dynamic.NewForConfig(conf)
64+
if err != nil {
65+
return fmt.Errorf("failed to create dynamic client: %w", err)
66+
}
67+
68+
sprl, err := dc.ServerPreferredResources()
69+
if err != nil {
70+
return fmt.Errorf("failed to get server preferred resources: %w", err)
71+
}
72+
73+
fmt.Fprintln(logWriter, "Discovered resources:")
74+
for _, re := range sprl {
75+
fmt.Fprintln(logWriter, re.GroupVersion)
76+
for _, r := range re.APIResources {
77+
fmt.Fprintln(logWriter, " ", r.Kind)
78+
}
79+
}
80+
81+
if len(opts.MustExistResources) > 0 {
82+
want := sets.New(opts.MustExistResources...)
83+
have := sets.New[string]()
84+
for _, re := range sprl {
85+
for _, r := range re.APIResources {
86+
res := formatGVRForComparison(groupVersionFromString(re.GroupVersion).WithResource(r.Name))
87+
have.Insert(res)
88+
}
89+
}
90+
missing := want.Difference(have)
91+
if missing.Len() > 0 {
92+
return fmt.Errorf("missing resources: %s", sets.List(missing))
93+
}
94+
}
95+
96+
var errors []error
97+
for _, re := range sprl {
98+
for _, r := range re.APIResources {
99+
res := groupVersionFromString(re.GroupVersion).WithResource(r.Name)
100+
if !slices.Contains(r.Verbs, "list") {
101+
fmt.Fprintf(logWriter, "skipping %s: no list verb\n", res)
102+
continue
103+
}
104+
105+
if i := slices.IndexFunc(opts.IgnoreResources, func(re *regexp.Regexp) bool {
106+
return re.MatchString(formatGVRForComparison(res))
107+
}); i > -1 {
108+
fmt.Fprintf(logWriter, "skipping %s: ignored by regex %q\n", res, opts.IgnoreResources[i].String())
109+
continue
110+
}
111+
112+
continueKey := ""
113+
for {
114+
l, err := dynClient.Resource(res).List(ctx, metav1.ListOptions{
115+
Limit: batchSize,
116+
Continue: continueKey,
117+
})
118+
if err != nil {
119+
errors = append(errors, fmt.Errorf("failed to list %s: %w", res, err))
120+
break
121+
}
122+
if err := cb(l); err != nil {
123+
errors = append(errors, fmt.Errorf("failed to dump %s: %w", res, err))
124+
}
125+
if l.GetContinue() == "" {
126+
break
127+
}
128+
continueKey = l.GetContinue()
129+
}
130+
}
131+
}
132+
133+
return multierr.Combine(errors...)
134+
}
135+
136+
func groupVersionFromString(s string) schema.GroupVersion {
137+
parts := strings.Split(s, "/")
138+
if len(parts) == 1 {
139+
return schema.GroupVersion{Version: parts[0]}
140+
}
141+
return schema.GroupVersion{Group: parts[0], Version: parts[1]}
142+
}
143+
144+
func formatGVRForComparison(gvr schema.GroupVersionResource) string {
145+
if gvr.Group == "" {
146+
return gvr.Resource
147+
}
148+
return fmt.Sprintf("%s.%s", gvr.Resource, gvr.Group)
149+
}

‎internal/pkg/discovery/discovery_test.go

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package discovery_test
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"regexp"
7+
"testing"
8+
9+
"github.com/projectsyn/k8s-object-dumper/internal/pkg/discovery"
10+
"github.com/stretchr/testify/require"
11+
corev1 "k8s.io/api/core/v1"
12+
rbacv1 "k8s.io/api/rbac/v1"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
15+
"k8s.io/client-go/rest"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
"sigs.k8s.io/controller-runtime/pkg/envtest"
18+
)
19+
20+
type objKey struct {
21+
apiVersion, kind, name, namespace string
22+
}
23+
24+
func Test_DiscoverObjects(t *testing.T) {
25+
cfg, stop := setupEnvtestEnv(t)
26+
defer stop()
27+
28+
objs := map[objKey]unstructured.Unstructured{}
29+
objTracker := func(obj *unstructured.UnstructuredList) error {
30+
for _, o := range obj.Items {
31+
objs[objKey{apiVersion: o.GetAPIVersion(), kind: o.GetKind(), name: o.GetName(), namespace: o.GetNamespace()}] = o
32+
}
33+
return nil
34+
}
35+
36+
c, err := client.New(cfg, client.Options{})
37+
require.NoError(t, err)
38+
39+
ns := corev1.Namespace{
40+
ObjectMeta: metav1.ObjectMeta{
41+
Name: "test-ns",
42+
},
43+
}
44+
sas := make([]client.Object, 0, 10)
45+
for i := 1; i <= cap(sas); i++ {
46+
sas = append(sas, &corev1.ServiceAccount{
47+
ObjectMeta: metav1.ObjectMeta{
48+
Name: fmt.Sprintf("test-service-account-%d", i),
49+
Namespace: "test-ns",
50+
},
51+
})
52+
}
53+
cr := rbacv1.ClusterRole{
54+
ObjectMeta: metav1.ObjectMeta{
55+
Name: "test-cluster-role",
56+
},
57+
}
58+
r := rbacv1.Role{
59+
ObjectMeta: metav1.ObjectMeta{
60+
Name: "test-role",
61+
Namespace: "test-ns",
62+
},
63+
}
64+
65+
for _, obj := range append([]client.Object{&ns, &cr, &r}, sas...) {
66+
require.NoError(t, c.Create(context.Background(), obj))
67+
}
68+
69+
require.NoError(t, discovery.DiscoverObjects(context.Background(), cfg, objTracker, discovery.DiscoveryOptions{
70+
BatchSize: int64(cap(sas) / 2),
71+
IgnoreResources: []*regexp.Regexp{
72+
regexp.MustCompile(`^roles.rbac.authorization.k8s.io$`),
73+
},
74+
MustExistResources: []string{
75+
"clusterroles.rbac.authorization.k8s.io",
76+
"deployments.apps",
77+
"namespaces",
78+
},
79+
}))
80+
81+
require.Contains(t, objs, objKey{apiVersion: "v1", kind: "Namespace", name: "test-ns", namespace: ""})
82+
require.Contains(t, objs, objKey{apiVersion: "rbac.authorization.k8s.io/v1", kind: "ClusterRole", name: "test-cluster-role", namespace: ""})
83+
for i := 1; i <= cap(sas); i++ {
84+
require.Contains(t, objs, objKey{apiVersion: "v1", kind: "ServiceAccount", name: fmt.Sprintf("test-service-account-%d", i), namespace: "test-ns"})
85+
}
86+
require.NotContains(t, objs, objKey{apiVersion: "rbac.authorization.k8s.io/v1", kind: "Role", name: "test-role", namespace: "test-ns"}, "Roles are ignored by regex")
87+
}
88+
89+
func Test_DiscoverObjects_MustExistResources_NotSatisfied(t *testing.T) {
90+
cfg, stop := setupEnvtestEnv(t)
91+
defer stop()
92+
93+
discard := func(obj *unstructured.UnstructuredList) error {
94+
return nil
95+
}
96+
97+
require.ErrorContains(t, discovery.DiscoverObjects(context.Background(), cfg, discard, discovery.DiscoveryOptions{
98+
MustExistResources: []string{
99+
"fluxcapacitors.spaceship.io",
100+
"namespaces",
101+
},
102+
}), "missing resources: [fluxcapacitors.spaceship.io]")
103+
}
104+
105+
func setupEnvtestEnv(t *testing.T) (cfg *rest.Config, stop func()) {
106+
t.Helper()
107+
108+
testEnv := &envtest.Environment{}
109+
110+
cfg, err := testEnv.Start()
111+
require.NoError(t, err)
112+
113+
return cfg, func() {
114+
require.NoError(t, testEnv.Stop())
115+
}
116+
}

‎internal/pkg/dumper/dir.go

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
package dumper
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"fmt"
7+
"os"
8+
"path/filepath"
9+
10+
"go.uber.org/multierr"
11+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
12+
)
13+
14+
// DirDumper writes objects to a directory.
15+
// Must be initialized with newDirDumper.
16+
// Must be closed after use.
17+
type DirDumper struct {
18+
dir string
19+
20+
openFiles map[string]*os.File
21+
sharedBuf *bytes.Buffer
22+
}
23+
24+
// NewDirDumper creates a new dirDumper that writes objects to the given directory.
25+
// The directory will be created if it does not exist.
26+
// If the directory cannot be created, an error is returned.
27+
func NewDirDumper(dir string) (*DirDumper, error) {
28+
if err := os.MkdirAll(dir, 0755); err != nil {
29+
return nil, fmt.Errorf("failed to create directory %q: %w", dir, err)
30+
}
31+
return &DirDumper{
32+
dir: dir,
33+
openFiles: make(map[string]*os.File),
34+
sharedBuf: new(bytes.Buffer),
35+
}, nil
36+
}
37+
38+
// Close closes the dirDumper and all open files.
39+
// The dirDumper cannot be used after it is closed.
40+
func (d *DirDumper) Close() error {
41+
var errs []error
42+
for _, f := range d.openFiles {
43+
if err := f.Close(); err != nil {
44+
errs = append(errs, err)
45+
}
46+
}
47+
return multierr.Combine(errs...)
48+
}
49+
50+
// Dump writes the objects in the list to the directory.
51+
// The objects are written to the directory in two ways:
52+
// - All objects are written to a file named objects-<kind>.json
53+
// - Objects with a namespace are written to a directory named split/<namespace> with two files:
54+
// - __all__.json contains all objects in the namespace
55+
// - <kind>.json contains all objects of the kind in the namespace
56+
//
57+
// If an object cannot be written, an error is returned.
58+
// This method is not safe for concurrent use.
59+
func (d *DirDumper) Dump(l *unstructured.UnstructuredList) error {
60+
buf := d.sharedBuf
61+
var errs []error
62+
for _, o := range l.Items {
63+
buf.Reset()
64+
if err := json.NewEncoder(buf).Encode(o.Object); err != nil {
65+
errs = append(errs, fmt.Errorf("failed to encode object: %w", err))
66+
continue
67+
}
68+
p := buf.Bytes()
69+
gk := o.GroupVersionKind().GroupKind()
70+
71+
if err := d.writeToFile(fmt.Sprintf("%s/objects-%s.json", d.dir, gk), p); err != nil {
72+
errs = append(errs, err)
73+
}
74+
75+
if o.GetNamespace() == "" {
76+
continue
77+
}
78+
79+
if err := d.writeToFile(fmt.Sprintf("%s/split/%s/__all__.json", d.dir, o.GetNamespace()), p); err != nil {
80+
errs = append(errs, err)
81+
}
82+
if err := d.writeToFile(fmt.Sprintf("%s/split/%s/%s.json", d.dir, o.GetNamespace(), gk), p); err != nil {
83+
errs = append(errs, err)
84+
}
85+
}
86+
return multierr.Combine(errs...)
87+
}
88+
89+
func (d *DirDumper) writeToFile(path string, b []byte) error {
90+
f, err := d.file(path)
91+
if err != nil {
92+
return fmt.Errorf("failed to open file for copying: %w", err)
93+
}
94+
if _, err := f.Write(b); err != nil {
95+
return fmt.Errorf("failed to copy to file: %w", err)
96+
}
97+
return nil
98+
}
99+
100+
func (d *DirDumper) file(path string) (*os.File, error) {
101+
f, ok := d.openFiles[path]
102+
if ok {
103+
return f, nil
104+
}
105+
dir := filepath.Dir(path)
106+
if err := os.MkdirAll(dir, 0755); err != nil {
107+
return nil, fmt.Errorf("failed to create directory %q: %w", dir, err)
108+
}
109+
f, err := os.Create(path)
110+
if err != nil {
111+
return nil, fmt.Errorf("failed to create file %q: %w", path, err)
112+
}
113+
d.openFiles[path] = f
114+
return f, nil
115+
}

‎internal/pkg/dumper/dir_test.go

+165
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
package dumper_test
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"os"
7+
"testing"
8+
9+
"github.com/stretchr/testify/require"
10+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
11+
12+
"github.com/projectsyn/k8s-object-dumper/internal/pkg/dumper"
13+
)
14+
15+
func Test_DirDumper(t *testing.T) {
16+
tdir, err := os.MkdirTemp(".", "test")
17+
require.NoError(t, err)
18+
defer os.RemoveAll(tdir)
19+
20+
subject, err := dumper.NewDirDumper(tdir)
21+
require.NoError(t, err)
22+
23+
uls := []*unstructured.UnstructuredList{
24+
{
25+
Items: []unstructured.Unstructured{
26+
{
27+
Object: map[string]interface{}{
28+
"kind": "Pod",
29+
"apiVersion": "v1",
30+
"metadata": map[string]interface{}{
31+
"name": "test-pod",
32+
"namespace": "test-ns",
33+
},
34+
},
35+
},
36+
{
37+
Object: map[string]interface{}{
38+
"kind": "Pod",
39+
"apiVersion": "v1",
40+
"metadata": map[string]interface{}{
41+
"name": "test-pod",
42+
"namespace": "test-ns-2",
43+
},
44+
},
45+
},
46+
},
47+
},
48+
{
49+
Items: []unstructured.Unstructured{
50+
{
51+
Object: map[string]interface{}{
52+
"kind": "Pod",
53+
"apiVersion": "v1",
54+
"metadata": map[string]interface{}{
55+
"name": "test-pod-2",
56+
"namespace": "test-ns",
57+
},
58+
},
59+
},
60+
},
61+
},
62+
{
63+
Items: []unstructured.Unstructured{
64+
{
65+
Object: map[string]interface{}{
66+
"kind": "Service",
67+
"apiVersion": "v1",
68+
"metadata": map[string]interface{}{
69+
"name": "test-svc",
70+
"namespace": "test-ns",
71+
},
72+
},
73+
},
74+
},
75+
},
76+
{
77+
Items: []unstructured.Unstructured{
78+
{
79+
Object: map[string]interface{}{
80+
"kind": "ClusterRole",
81+
"apiVersion": "rbac.authorization.k8s.io/v1",
82+
"metadata": map[string]interface{}{
83+
"name": "cluster-scoped",
84+
},
85+
},
86+
},
87+
},
88+
},
89+
}
90+
91+
for i, ul := range uls {
92+
require.NoErrorf(t, subject.Dump(ul), "failed to dump list %d", i)
93+
}
94+
defer func() {
95+
require.NoError(t, subject.Close())
96+
}()
97+
98+
require.FileExists(t, tdir+"/objects-Pod.json")
99+
requireFileContains(t, tdir+"/objects-Pod.json", []ExpectedObject{
100+
{Kind: "Pod", Name: "test-pod", Namespace: "test-ns"},
101+
{Kind: "Pod", Name: "test-pod", Namespace: "test-ns-2"},
102+
{Kind: "Pod", Name: "test-pod-2", Namespace: "test-ns"},
103+
})
104+
require.FileExists(t, tdir+"/objects-Service.json")
105+
requireFileContains(t, tdir+"/objects-Service.json", []ExpectedObject{
106+
{Kind: "Service", Name: "test-svc", Namespace: "test-ns"},
107+
})
108+
require.FileExists(t, tdir+"/objects-ClusterRole.rbac.authorization.k8s.io.json")
109+
requireFileContains(t, tdir+"/objects-ClusterRole.rbac.authorization.k8s.io.json", []ExpectedObject{
110+
{Kind: "ClusterRole", Name: "cluster-scoped", Namespace: ""},
111+
})
112+
require.FileExists(t, tdir+"/split/test-ns/__all__.json")
113+
requireFileContains(t, tdir+"/split/test-ns/__all__.json", []ExpectedObject{
114+
{Kind: "Pod", Name: "test-pod", Namespace: "test-ns"},
115+
{Kind: "Pod", Name: "test-pod-2", Namespace: "test-ns"},
116+
{Kind: "Service", Name: "test-svc", Namespace: "test-ns"},
117+
})
118+
require.FileExists(t, tdir+"/split/test-ns/Pod.json")
119+
requireFileContains(t, tdir+"/split/test-ns/Pod.json", []ExpectedObject{
120+
{Kind: "Pod", Name: "test-pod", Namespace: "test-ns"},
121+
{Kind: "Pod", Name: "test-pod-2", Namespace: "test-ns"},
122+
})
123+
require.FileExists(t, tdir+"/split/test-ns/Service.json")
124+
requireFileContains(t, tdir+"/split/test-ns/Service.json", []ExpectedObject{
125+
{Kind: "Service", Name: "test-svc", Namespace: "test-ns"},
126+
})
127+
require.FileExists(t, tdir+"/split/test-ns-2/__all__.json")
128+
requireFileContains(t, tdir+"/split/test-ns-2/__all__.json", []ExpectedObject{
129+
{Kind: "Pod", Name: "test-pod", Namespace: "test-ns-2"},
130+
})
131+
require.FileExists(t, tdir+"/split/test-ns-2/Pod.json")
132+
requireFileContains(t, tdir+"/split/test-ns-2/Pod.json", []ExpectedObject{
133+
{Kind: "Pod", Name: "test-pod", Namespace: "test-ns-2"},
134+
})
135+
}
136+
137+
type ExpectedObject struct {
138+
Kind, Name, Namespace string
139+
}
140+
141+
func requireFileContains(t *testing.T, path string, expected []ExpectedObject) {
142+
t.Helper()
143+
raw, err := os.ReadFile(path)
144+
require.NoError(t, err)
145+
146+
var objs []unstructured.Unstructured
147+
raw = bytes.TrimSuffix(raw, []byte("\n"))
148+
rawObjs := bytes.Split(raw, []byte("\n"))
149+
for _, rawObj := range rawObjs {
150+
var obj unstructured.Unstructured
151+
require.NoError(t, json.Unmarshal(rawObj, &obj))
152+
objs = append(objs, obj)
153+
}
154+
155+
actualObjects := make([]ExpectedObject, 0, len(objs))
156+
for _, obj := range objs {
157+
actualObjects = append(actualObjects, ExpectedObject{
158+
Kind: obj.GetKind(),
159+
Name: obj.GetName(),
160+
Namespace: obj.GetNamespace(),
161+
})
162+
}
163+
164+
require.ElementsMatch(t, expected, actualObjects)
165+
}

‎internal/pkg/dumper/dumper.go

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Dumper provides means to dump a UnstructuredList returned by a dynamic client.
2+
package dumper
3+
4+
import (
5+
"encoding/json"
6+
"io"
7+
8+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
9+
)
10+
11+
// Dumper is an interface for dumping a list of unstructured objects
12+
type DumperFunc func(*unstructured.UnstructuredList) error
13+
14+
// DumpToWriter dumps the list of unstructured objects to the provided writer as JSON
15+
func DumpToWriter(w io.Writer) DumperFunc {
16+
return func(l *unstructured.UnstructuredList) error {
17+
return json.NewEncoder(w).Encode(l)
18+
}
19+
}

‎internal/pkg/dumper/dumper_test.go

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package dumper_test
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"testing"
7+
8+
"github.com/projectsyn/k8s-object-dumper/internal/pkg/dumper"
9+
"github.com/stretchr/testify/require"
10+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
11+
)
12+
13+
func Test_DumpToWriter(t *testing.T) {
14+
var b bytes.Buffer
15+
16+
subject := dumper.DumpToWriter(&b)
17+
18+
require.NoError(t,
19+
subject(&unstructured.UnstructuredList{
20+
Object: map[string]interface{}{
21+
"kind": "List",
22+
},
23+
Items: []unstructured.Unstructured{
24+
{
25+
Object: map[string]interface{}{
26+
"kind": "Pod",
27+
"apiVersion": "v1",
28+
"metadata": map[string]interface{}{
29+
"name": "test-pod",
30+
"namespace": "test-ns",
31+
},
32+
},
33+
},
34+
},
35+
}),
36+
)
37+
38+
var got unstructured.UnstructuredList
39+
require.NoError(t, json.NewDecoder(&b).Decode(&got))
40+
41+
require.Len(t, got.Items, 1)
42+
require.Equal(t, "Pod", got.Items[0].GetKind())
43+
}

‎known-to-fail

-13
This file was deleted.

‎main.go

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"flag"
6+
"fmt"
7+
"os"
8+
"regexp"
9+
10+
ctrl "sigs.k8s.io/controller-runtime"
11+
12+
"github.com/projectsyn/k8s-object-dumper/internal/pkg/discovery"
13+
"github.com/projectsyn/k8s-object-dumper/internal/pkg/dumper"
14+
)
15+
16+
func main() {
17+
var dir string
18+
var batchSize int64
19+
mustExistResources := new(repeatableStringFlag)
20+
ignoreResources := new(repeatableRegexpFlag)
21+
22+
flag.StringVar(&dir, "dir", "", "Directory to dump objects into")
23+
flag.Int64Var(&batchSize, "batch-size", 500, "Batch size for listing objects")
24+
flag.Var(mustExistResources, "must-exist", "Resource that must exist in the cluster. Can be used multiple times.")
25+
flag.Var(ignoreResources, "ignore", "Resource to ignore during discovery. Regexp, anchored by default. Can be used multiple times.")
26+
27+
flag.Parse()
28+
29+
df := dumper.DumpToWriter(os.Stdout)
30+
if dir != "" {
31+
if err := os.MkdirAll(dir, 0755); err != nil {
32+
fmt.Fprintf(os.Stderr, "failed to create directory %s: %v\n", dir, err)
33+
os.Exit(1)
34+
}
35+
d, err := dumper.NewDirDumper(dir)
36+
if err != nil {
37+
fmt.Fprintf(os.Stderr, "failed to create directory dumper: %v\n", err)
38+
os.Exit(1)
39+
}
40+
defer d.Close()
41+
df = d.Dump
42+
}
43+
44+
conf, err := ctrl.GetConfig()
45+
if err != nil {
46+
fmt.Fprintf(os.Stderr, "failed to get Kubernetes config: %v", err)
47+
}
48+
49+
if err := discovery.DiscoverObjects(context.Background(), conf, df, discovery.DiscoveryOptions{
50+
BatchSize: batchSize,
51+
LogWriter: os.Stderr,
52+
MustExistResources: *mustExistResources,
53+
IgnoreResources: *ignoreResources,
54+
}); err != nil {
55+
fmt.Fprintf(os.Stderr, "failed to dump some or all objects: %+v\n", err)
56+
os.Exit(1)
57+
}
58+
}
59+
60+
type repeatableStringFlag []string
61+
62+
func (i *repeatableStringFlag) String() string {
63+
return fmt.Sprintf("%v", *i)
64+
}
65+
66+
func (i *repeatableStringFlag) Set(value string) error {
67+
*i = append(*i, value)
68+
return nil
69+
}
70+
71+
type repeatableRegexpFlag []*regexp.Regexp
72+
73+
func (i *repeatableRegexpFlag) String() string {
74+
return fmt.Sprintf("%v", *i)
75+
}
76+
77+
func (i *repeatableRegexpFlag) Set(value string) error {
78+
value = fmt.Sprintf("^%s$", value)
79+
r, err := regexp.Compile(value)
80+
if err != nil {
81+
return fmt.Errorf("failed to compile regexp %q: %w", value, err)
82+
}
83+
*i = append(*i, r)
84+
return nil
85+
}

‎must-exist

-19
This file was deleted.

‎renovate.json

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
{
2+
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
23
"extends": [
3-
"config:base"
4-
],
5-
"labels": [
6-
"dependency"
4+
"config:recommended"
75
]
86
}

0 commit comments

Comments
 (0)
Please sign in to comment.