From 3ef831e6a2484b5f1d4f69cfc98cfec5d6b3f4e4 Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Mon, 15 May 2023 12:06:44 +0200 Subject: [PATCH 01/62] Add c0ffee to the contributors list (#1414) Signed-off-by: Mauro Morales --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index bf63ea338..f2de084f3 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -11,3 +11,4 @@ These are active contributors who have made multiple contributions to the projec | Jason B. Alonso | [@jbalonso](https://github.com/jbalonso) | | Shawn Wilsher | [@sdwilsh](https://github.com/sdwilsh) | | scuzhanglei | [@scuzhanglei](https://github.com/scuzhanglei) | +| Martin Schuessler | [@c0ffee](https://github.com/c0ffee) | \ No newline at end of file From 38da07311828606818a11ab79cac8c9612451662 Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Tue, 16 May 2023 16:27:22 +0200 Subject: [PATCH 02/62] :robot: Fix CI broken by lack of space for the Ubuntu arm rpi image (#1417) * Expand size for ubuntu arm images Signed-off-by: Mauro Morales * Expand size of active partition for ubuntu arm rpi Signed-off-by: Mauro Morales --------- Signed-off-by: Mauro Morales --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index 5461153dd..cff89d682 100644 --- a/Earthfile +++ b/Earthfile @@ -458,7 +458,7 @@ arm-image: IF [[ "$FLAVOR" =~ ^ubuntu* ]] ENV STATE_SIZE="6900" ENV RECOVERY_SIZE="4600" - ENV DEFAULT_ACTIVE_SIZE="2300" + ENV DEFAULT_ACTIVE_SIZE="2500" ELSE ENV STATE_SIZE="6200" ENV RECOVERY_SIZE="4200" From fdea673d5471f8d1c113bf9d253c602c98fd4e5d Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Wed, 17 May 2023 09:58:30 +0200 Subject: [PATCH 03/62] Remove docs (#1415) * remove docsy as submodule Signed-off-by: Mauro Morales * remove docs Signed-off-by: Mauro Morales --------- Signed-off-by: Mauro Morales --- .github/workflows/image-arm.yaml | 1 - .github/workflows/image.yaml | 2 - .github/workflows/lint.yml | 3 - .github/workflows/schema.yaml | 42 - .gitignore | 13 - .gitmodules | 7 - docs/CNAME | 1 - docs/Makefile | 18 - docs/README.md | 35 - docs/assets/css/sections.css | 0 docs/assets/icons/logo.svg | 21 - docs/assets/scss/_styles_project.scss | 376 ---- docs/assets/scss/_variables_project.scss | 29 - docs/config.toml | 172 -- docs/content/en/blog/Kairos-at-FOSDEM-2023.md | 31 - ...ros-at-the-KCD-Amsterdam-and-Paris-2023.md | 22 - docs/content/en/blog/_index.md | 4 - .../home-lab-with-kairos-and-wireguard.md | 296 --- docs/content/en/blog/kairos-canonical.md | 23 - docs/content/en/blog/media-section.md | 9 - docs/content/en/blog/release-v1.5.md | 41 - docs/content/en/blog/release-v1.6.md | 67 - docs/content/en/blog/release-v2.0.md | 48 - docs/content/en/blog/sena-architecture.md | 50 - .../en/blog/understanding-immutability.md | 141 -- docs/content/en/community/_index.md | 6 - docs/content/en/docs/Advanced/_index.md | 8 - .../content/en/docs/Advanced/after-install.md | 117 -- docs/content/en/docs/Advanced/build.md | 288 --- docs/content/en/docs/Advanced/bundles.md | 151 -- docs/content/en/docs/Advanced/coco.md | 99 - docs/content/en/docs/Advanced/customizing.md | 159 -- docs/content/en/docs/Advanced/livelayering.md | 96 - docs/content/en/docs/Advanced/networking.md | 164 -- .../en/docs/Advanced/partition_encryption.md | 301 ---- docs/content/en/docs/Architecture/_index.md | 7 - .../en/docs/Architecture/cloud-init.md | 73 - .../content/en/docs/Architecture/container.md | 120 -- .../content/en/docs/Architecture/immutable.md | 61 - docs/content/en/docs/Architecture/meta.md | 42 - docs/content/en/docs/Architecture/network.md | 167 -- docs/content/en/docs/Development/_index.md | 6 - .../en/docs/Development/debugging-station.md | 246 --- .../en/docs/Development/development.md | 131 -- docs/content/en/docs/Development/nvidia.md | 269 --- docs/content/en/docs/Examples/_index.md | 17 - docs/content/en/docs/Examples/airgap.md | 122 -- docs/content/en/docs/Examples/bundles.md | 59 - docs/content/en/docs/Examples/core.md | 60 - docs/content/en/docs/Examples/ha.md | 118 -- docs/content/en/docs/Examples/metallb.md | 88 - .../Examples/multi-node-p2p-ha-kubevip.md | 75 - .../en/docs/Examples/multi-node-p2p-ha.md | 61 - .../en/docs/Examples/multi-node-p2p.md | 47 - docs/content/en/docs/Examples/multi-node.md | 64 - docs/content/en/docs/Examples/p2p_e2e.md | 165 -- .../en/docs/Examples/single-node-p2p.md | 54 - docs/content/en/docs/Examples/single-node.md | 49 - .../content/en/docs/Getting started/_index.md | 320 ---- docs/content/en/docs/Installation/_index.md | 8 - .../content/en/docs/Installation/automated.md | 224 --- .../en/docs/Installation/interactive.md | 22 - docs/content/en/docs/Installation/manual.md | 39 - docs/content/en/docs/Installation/netboot.md | 128 -- docs/content/en/docs/Installation/p2p.md | 281 --- docs/content/en/docs/Installation/qrcode.md | 43 - .../content/en/docs/Installation/raspberry.md | 73 - docs/content/en/docs/Installation/takeover.md | 29 - docs/content/en/docs/Installation/webui.md | 20 - docs/content/en/docs/Media/_index.md | 33 - docs/content/en/docs/Reference/_index.md | 6 - .../content/en/docs/Reference/architecture.md | 31 - docs/content/en/docs/Reference/auroraboot.md | 615 ------- .../en/docs/Reference/build-from-scratch.md | 181 -- docs/content/en/docs/Reference/cli.md | 151 -- .../en/docs/Reference/configuration.md | 1101 ------------ docs/content/en/docs/Reference/entangle.md | 402 ----- docs/content/en/docs/Reference/faq.md | 35 - .../content/en/docs/Reference/image_matrix.md | 129 -- .../en/docs/Reference/recovery_mode.md | 54 - docs/content/en/docs/Reference/reset.md | 130 -- .../en/docs/Reference/troubleshooting.md | 44 - docs/content/en/docs/Upgrade/_index.md | 6 - docs/content/en/docs/Upgrade/kubernetes.md | 248 --- docs/content/en/docs/Upgrade/manual.md | 63 - docs/content/en/docs/_index.md | 143 -- docs/content/en/search-index.md | 4 - docs/content/en/search.md | 6 - docs/layouts/404.html | 10 - docs/layouts/index.html | 75 - docs/layouts/shortcodes/armFlavor.html | 1 - docs/layouts/shortcodes/flavor.html | 1 - docs/layouts/shortcodes/githubembed.html | 9 - docs/layouts/shortcodes/k3sVersion.html | 1 - docs/layouts/shortcodes/kairosVersion.html | 1 - docs/layouts/shortcodes/registryURL.html | 1 - docs/netlify.toml | 10 - docs/package-lock.json | 1599 ----------------- docs/package.json | 12 - docs/scripts/build.sh | 26 - docs/scripts/publish.sh | 14 - docs/scripts/serve.sh | 16 - .../favicons/android-chrome-192x192.png | Bin 14851 -> 0 bytes docs/static/favicons/apple-touch-icon.png | Bin 6326 -> 0 bytes docs/static/favicons/browserconfig.xml | 9 - docs/static/favicons/favicon-16x16.png | Bin 1327 -> 0 bytes docs/static/favicons/favicon-32x32.png | Bin 2012 -> 0 bytes docs/static/favicons/favicon.ico | Bin 15086 -> 0 bytes docs/static/favicons/mstile-150x150.png | Bin 7393 -> 0 bytes docs/static/favicons/site.webmanifest | 14 - docs/static/images/kairos-over-wireguard.svg | 16 - .../images/schema-validation-preview.gif | Bin 44517 -> 0 bytes docs/static/index/1.png | Bin 6011 -> 0 bytes docs/static/index/2.png | Bin 5174 -> 0 bytes docs/static/index/3.png | Bin 6275 -> 0 bytes docs/static/index/4.png | Bin 8524 -> 0 bytes docs/static/index/5.png | Bin 4514 -> 0 bytes docs/static/index/6.png | Bin 8555 -> 0 bytes docs/static/index/7.png | Bin 7125 -> 0 bytes docs/static/index/book.png | Bin 968 -> 0 bytes docs/static/index/explore.png | Bin 4476 -> 0 bytes docs/static/index/footer.png | Bin 10330 -> 0 bytes docs/static/index/github.png | Bin 2529 -> 0 bytes docs/static/index/logo.png | Bin 12853 -> 0 bytes docs/static/v1.6.0-rc1/cloud-config.json | 412 ----- docs/static/v1.6.0-rc2/cloud-config.json | 412 ----- docs/static/v1.6.0-rc3/cloud-config.json | 412 ----- docs/static/v1.6.0/cloud-config.json | 412 ----- docs/static/v2.0.0-alpha1/cloud-config.json | 415 ----- docs/static/v2.0.0-alpha2/cloud-config.json | 415 ----- docs/static/v2.0.0-alpha3/cloud-config.json | 415 ----- docs/static/v2.0.0-rc5/cloud-config.json | 415 ----- docs/static/v2.0.0-rc6/cloud-config.json | 415 ----- docs/static/v2.0.0/cloud-config.json | 415 ----- docs/static/v2.0.1/cloud-config.json | 415 ----- docs/themes/docsy | 1 - 136 files changed, 15880 deletions(-) delete mode 100644 .github/workflows/schema.yaml delete mode 100644 docs/CNAME delete mode 100644 docs/Makefile delete mode 100644 docs/README.md delete mode 100644 docs/assets/css/sections.css delete mode 100644 docs/assets/icons/logo.svg delete mode 100644 docs/assets/scss/_styles_project.scss delete mode 100644 docs/assets/scss/_variables_project.scss delete mode 100644 docs/config.toml delete mode 100644 docs/content/en/blog/Kairos-at-FOSDEM-2023.md delete mode 100644 docs/content/en/blog/Kairos-at-the-KCD-Amsterdam-and-Paris-2023.md delete mode 100644 docs/content/en/blog/_index.md delete mode 100644 docs/content/en/blog/home-lab-with-kairos-and-wireguard.md delete mode 100644 docs/content/en/blog/kairos-canonical.md delete mode 100644 docs/content/en/blog/media-section.md delete mode 100644 docs/content/en/blog/release-v1.5.md delete mode 100644 docs/content/en/blog/release-v1.6.md delete mode 100644 docs/content/en/blog/release-v2.0.md delete mode 100644 docs/content/en/blog/sena-architecture.md delete mode 100644 docs/content/en/blog/understanding-immutability.md delete mode 100644 docs/content/en/community/_index.md delete mode 100644 docs/content/en/docs/Advanced/_index.md delete mode 100644 docs/content/en/docs/Advanced/after-install.md delete mode 100644 docs/content/en/docs/Advanced/build.md delete mode 100644 docs/content/en/docs/Advanced/bundles.md delete mode 100644 docs/content/en/docs/Advanced/coco.md delete mode 100644 docs/content/en/docs/Advanced/customizing.md delete mode 100644 docs/content/en/docs/Advanced/livelayering.md delete mode 100644 docs/content/en/docs/Advanced/networking.md delete mode 100644 docs/content/en/docs/Advanced/partition_encryption.md delete mode 100644 docs/content/en/docs/Architecture/_index.md delete mode 100644 docs/content/en/docs/Architecture/cloud-init.md delete mode 100644 docs/content/en/docs/Architecture/container.md delete mode 100644 docs/content/en/docs/Architecture/immutable.md delete mode 100644 docs/content/en/docs/Architecture/meta.md delete mode 100644 docs/content/en/docs/Architecture/network.md delete mode 100644 docs/content/en/docs/Development/_index.md delete mode 100644 docs/content/en/docs/Development/debugging-station.md delete mode 100644 docs/content/en/docs/Development/development.md delete mode 100644 docs/content/en/docs/Development/nvidia.md delete mode 100644 docs/content/en/docs/Examples/_index.md delete mode 100644 docs/content/en/docs/Examples/airgap.md delete mode 100644 docs/content/en/docs/Examples/bundles.md delete mode 100644 docs/content/en/docs/Examples/core.md delete mode 100644 docs/content/en/docs/Examples/ha.md delete mode 100644 docs/content/en/docs/Examples/metallb.md delete mode 100644 docs/content/en/docs/Examples/multi-node-p2p-ha-kubevip.md delete mode 100644 docs/content/en/docs/Examples/multi-node-p2p-ha.md delete mode 100644 docs/content/en/docs/Examples/multi-node-p2p.md delete mode 100644 docs/content/en/docs/Examples/multi-node.md delete mode 100644 docs/content/en/docs/Examples/p2p_e2e.md delete mode 100644 docs/content/en/docs/Examples/single-node-p2p.md delete mode 100644 docs/content/en/docs/Examples/single-node.md delete mode 100644 docs/content/en/docs/Getting started/_index.md delete mode 100644 docs/content/en/docs/Installation/_index.md delete mode 100644 docs/content/en/docs/Installation/automated.md delete mode 100644 docs/content/en/docs/Installation/interactive.md delete mode 100644 docs/content/en/docs/Installation/manual.md delete mode 100644 docs/content/en/docs/Installation/netboot.md delete mode 100644 docs/content/en/docs/Installation/p2p.md delete mode 100644 docs/content/en/docs/Installation/qrcode.md delete mode 100644 docs/content/en/docs/Installation/raspberry.md delete mode 100644 docs/content/en/docs/Installation/takeover.md delete mode 100644 docs/content/en/docs/Installation/webui.md delete mode 100644 docs/content/en/docs/Media/_index.md delete mode 100644 docs/content/en/docs/Reference/_index.md delete mode 100644 docs/content/en/docs/Reference/architecture.md delete mode 100644 docs/content/en/docs/Reference/auroraboot.md delete mode 100644 docs/content/en/docs/Reference/build-from-scratch.md delete mode 100644 docs/content/en/docs/Reference/cli.md delete mode 100644 docs/content/en/docs/Reference/configuration.md delete mode 100644 docs/content/en/docs/Reference/entangle.md delete mode 100644 docs/content/en/docs/Reference/faq.md delete mode 100644 docs/content/en/docs/Reference/image_matrix.md delete mode 100644 docs/content/en/docs/Reference/recovery_mode.md delete mode 100644 docs/content/en/docs/Reference/reset.md delete mode 100644 docs/content/en/docs/Reference/troubleshooting.md delete mode 100644 docs/content/en/docs/Upgrade/_index.md delete mode 100644 docs/content/en/docs/Upgrade/kubernetes.md delete mode 100644 docs/content/en/docs/Upgrade/manual.md delete mode 100755 docs/content/en/docs/_index.md delete mode 100644 docs/content/en/search-index.md delete mode 100644 docs/content/en/search.md delete mode 100644 docs/layouts/404.html delete mode 100644 docs/layouts/index.html delete mode 100644 docs/layouts/shortcodes/armFlavor.html delete mode 100644 docs/layouts/shortcodes/flavor.html delete mode 100644 docs/layouts/shortcodes/githubembed.html delete mode 100644 docs/layouts/shortcodes/k3sVersion.html delete mode 100644 docs/layouts/shortcodes/kairosVersion.html delete mode 100644 docs/layouts/shortcodes/registryURL.html delete mode 100644 docs/netlify.toml delete mode 100644 docs/package-lock.json delete mode 100644 docs/package.json delete mode 100755 docs/scripts/build.sh delete mode 100755 docs/scripts/publish.sh delete mode 100755 docs/scripts/serve.sh delete mode 100644 docs/static/favicons/android-chrome-192x192.png delete mode 100644 docs/static/favicons/apple-touch-icon.png delete mode 100644 docs/static/favicons/browserconfig.xml delete mode 100644 docs/static/favicons/favicon-16x16.png delete mode 100644 docs/static/favicons/favicon-32x32.png delete mode 100644 docs/static/favicons/favicon.ico delete mode 100644 docs/static/favicons/mstile-150x150.png delete mode 100644 docs/static/favicons/site.webmanifest delete mode 100644 docs/static/images/kairos-over-wireguard.svg delete mode 100644 docs/static/images/schema-validation-preview.gif delete mode 100644 docs/static/index/1.png delete mode 100644 docs/static/index/2.png delete mode 100644 docs/static/index/3.png delete mode 100644 docs/static/index/4.png delete mode 100644 docs/static/index/5.png delete mode 100644 docs/static/index/6.png delete mode 100644 docs/static/index/7.png delete mode 100644 docs/static/index/book.png delete mode 100644 docs/static/index/explore.png delete mode 100644 docs/static/index/footer.png delete mode 100644 docs/static/index/github.png delete mode 100644 docs/static/index/logo.png delete mode 100644 docs/static/v1.6.0-rc1/cloud-config.json delete mode 100644 docs/static/v1.6.0-rc2/cloud-config.json delete mode 100644 docs/static/v1.6.0-rc3/cloud-config.json delete mode 100644 docs/static/v1.6.0/cloud-config.json delete mode 100644 docs/static/v2.0.0-alpha1/cloud-config.json delete mode 100644 docs/static/v2.0.0-alpha2/cloud-config.json delete mode 100644 docs/static/v2.0.0-alpha3/cloud-config.json delete mode 100644 docs/static/v2.0.0-rc5/cloud-config.json delete mode 100644 docs/static/v2.0.0-rc6/cloud-config.json delete mode 100644 docs/static/v2.0.0/cloud-config.json delete mode 100644 docs/static/v2.0.1/cloud-config.json delete mode 160000 docs/themes/docsy diff --git a/.github/workflows/image-arm.yaml b/.github/workflows/image-arm.yaml index 4176fffd4..38c659b54 100644 --- a/.github/workflows/image-arm.yaml +++ b/.github/workflows/image-arm.yaml @@ -7,7 +7,6 @@ on: pull_request: paths: - '**' - - '!docs/**' concurrency: group: ci-arm-${{ github.head_ref || github.ref }}-${{ github.repository }} diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index bff502e36..addd7a2b2 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -5,11 +5,9 @@ on: - master paths: - '**' - - '!docs/**' pull_request: paths: - '**' - - '!docs/**' workflow_dispatch: inputs: immucore_dev: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6ed0c6066..0841719fd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,12 +3,9 @@ on: push: branches: - master - paths: - - '!docs/**' pull_request: paths: - '**' - - '!docs/**' env: FORCE_COLOR: 1 jobs: diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml deleted file mode 100644 index f9dd67b88..000000000 --- a/.github/workflows/schema.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: Cloud Config Schema -on: - # To test with push events, it's easy to use a test branch e.g. schematest, you will also need to update the checkout ref and the create PR base - # push: - # branches: - # - schematest - push: - tags: - - v* -jobs: - generate-schema: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: master - fetch-depth: 0 - - name: setup-docker - uses: docker-practice/actions-setup-docker@master - - name: Install earthly - uses: Luet-lab/luet-install-action@v1 - with: - repository: quay.io/kairos/packages - packages: utils/earthly - - name: Generate cloud-config Schema - run: | - earthly +generate-schema - - name: Change ownership - # Chown files generated by earthly since they are owned by root - run: sudo chown -R runner:docker docs/static/* - - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.PAT_TOKEN }} - push-to-fork: ci-robbot/c3os - base: master - branch: cloud-config-schema-updates - commit-message: ':book: Update Schema' - title: ':book: Update Schema' - body: Update latest cloud config schema release - signoff: true diff --git a/.gitignore b/.gitignore index 0e68c0bb6..25f8208d8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,22 +4,9 @@ /cli/cli /kairos /dist -/docs/bin -/docs/.hugo_build.lock /build -docs/node_modules -docs/public -!docs/public/favicon -!docs/public/CNAME -docs/dist -docs/.vscode coverage.out .DS_Store -/docs/bin/ -/docs/public/ -/docs/resources/ -/docs/node_modules/ -/docs/tech-doc-hugo internal/webui/public/cypress/videos/ node_modules/ diff --git a/.gitmodules b/.gitmodules index 89ae3c6a5..e69de29bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +0,0 @@ - -[submodule "themes/docsy"] - path = themes/docsy - url = https://github.com/google/docsy -[submodule "docs/themes/docsy"] - path = docs/themes/docsy - url = https://github.com/google/docsy.git diff --git a/docs/CNAME b/docs/CNAME deleted file mode 100644 index 12507cee5..000000000 --- a/docs/CNAME +++ /dev/null @@ -1 +0,0 @@ -kairos.io diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 71dc4611e..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -export HUGO_VERSION?=0.105.0 -export HUGO_PLATFORM?=Linux-64bit - -export ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) - -.DEFAULT_GOAL := build - -.PHONY: build -build: - scripts/build.sh - -.PHONY: serve -serve: - scripts/serve.sh - -.PHONY: publish -publish: - scripts/publish.sh diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 5099cf0c4..000000000 --- a/docs/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# :book: Kairos documentation - -The Kairos documentation uses [docsy](https://docsy.dev). - -## Prerequisites - -The following software is needed to preview the documentation changes locally. - -* Hugo [v0.105.0+](https://gohugo.io/installation/) -* nodeJs [v16+](https://nodejs.org/en/download/) - -## Test your changes - -After cloning the repo (with submodules), just run `make serve` to test the website locally. - -``` -$> git clone --recurse-submodule https://github.com/kairos-io/kairos -$> cd kairos/docs -$> npm run prepare -$> make serve -``` - -If you have a local copy already checked out, sync the submodules: - -``` -$> git submodule update --init --recursive --depth 1 -``` - -To run the website locally in other platforms, e.g. MacOS: - -``` -$> HUGO_PLATFORM=macOS-64bit make serve -``` - -**Note**: If the `make serve` command does not work for you, try to start hugo directly with the command `hugo server -D`. diff --git a/docs/assets/css/sections.css b/docs/assets/css/sections.css deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/assets/icons/logo.svg b/docs/assets/icons/logo.svg deleted file mode 100644 index 4650398b1..000000000 --- a/docs/assets/icons/logo.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/assets/scss/_styles_project.scss b/docs/assets/scss/_styles_project.scss deleted file mode 100644 index ec13980ab..000000000 --- a/docs/assets/scss/_styles_project.scss +++ /dev/null @@ -1,376 +0,0 @@ -.wrapper:where(.astro-7XAARZHW) { - display: flex; - align-items: center; - justify-content: center; -} -body { - margin: 0; -} -:where(.astro-7XAARZHW) { - font-family: Noto Sans; - box-sizing: border-box; -} -.hero:where(.astro-7XAARZHW) { - position: relative; - overflow: hidden; -} -@media screen and (min-width: 1024px) { - .hero:where(.astro-7XAARZHW) { - height: 45vw; - } -} -.hero:where(.astro-7XAARZHW) { - display: block; - width: 100%; - height: auto; -} -.hero-logo:where(.astro-7XAARZHW) { - margin-bottom: 58px; -} -@media screen and (min-width: 1024px) { - .hero-logo:where(.astro-7XAARZHW) { - margin-right: 260px; - margin-bottom: 0; - } -} -.overlay:where(.astro-7XAARZHW) { - width: 100%; - height: 100%; - min-height: 708px; - display: flex; - flex-direction: row; - align-items: center; - justify-content: center; - padding: 0 24px; - background-color: #03153a; -} -.constraint:where(.astro-7XAARZHW) { - width: 100%; - max-width: 1300px; - display: flex; - flex-direction: row; - align-items: center; - justify-content: center; -} -.constraint:where(.astro-7XAARZHW).vertical { - flex-direction: column; -} -.constraint:where(.astro-7XAARZHW).vertical>.header:where(.astro-7XAARZHW) { - display: flex; - flex-direction: column; - align-items: center; - width: 100%; - max-width: 1200px; - margin-top: 60px; -} -@media screen and (min-width: 1024px) { - .constraint:where(.astro-7XAARZHW).vertical>.header:where(.astro-7XAARZHW) { - flex-direction: row; - margin-top: 0; - } -} -.title:where(.astro-7XAARZHW) { - font-weight: 700; - font-size: 58px; - line-height: 80px; - letter-spacing: .02em; - color: #fff; - margin-top: 0; -} -.subtitle:where(.astro-7XAARZHW) { - font-weight: 500; - font-size: 31px; - line-height: 130%; - letter-spacing: .01em; - color: #fff; -} -.title:where(.astro-7XAARZHW)>span:where(.astro-7XAARZHW) { - color: #e06948; -} -.grid:where(.astro-7XAARZHW) { - display: grid; - grid-gap: 2rem; -} -.sectionTitle:where(.astro-7XAARZHW) { - font-weight: 700; - font-size: var(--f-u8); - margin-top: 4rem; - margin-bottom: 2rem; -} -.roles:where(.astro-7XAARZHW) { - display: flex; - flex-wrap: wrap; - gap: .5em; - font-size: var(--f-d1); -} -.role:where(.astro-7XAARZHW) { - position: relative; - display: inline-block; - white-space: nowrap; - font-weight: 900; - color: var(--t-bg); - background-color: var(--t-fg); - padding: .25em .5em; - z-index: 2; -} -.invert:where(.astro-7XAARZHW) { - position: absolute; - color: var(--t-fg); - display: flex; - align-items: center; - justify-content: center; - width: 100%; - height: 100%; - top: 0; - left: 0; - pointer-events: none; - clip-path: polygon(0% 100%, 100% 100%, 100% 200%, 0% 200%); - transition: clip-path cubic-bezier(.4, 0, .5, 1) .15s; -} -.desc:where(.astro-7XAARZHW) { - font-weight: 400; - font-size: 24px; - line-height: 170%; - color: #fff; - max-width: 748px; - margin: 64px 0; - max-width: 600px; -} -@media screen and (min-width: 1024px) { - .desc:where(.astro-7XAARZHW) { - max-width: 1199px; - font-size: 18px; - } -} -.desc:where(.astro-7XAARZHW) span:where(.astro-7XAARZHW) { - color: #ee5007; - font-weight: 700; -} -.bio:where(.astro-7XAARZHW) { - line-height: 2; - margin-bottom: 2rem; -} -.bio:where(.astro-7XAARZHW)>span:where(.astro-7XAARZHW):first-of-type { - line-height: 1; - margin-bottom: .5em; - display: block; - font-weight: 700; - font-size: var(--f-u4); -} -.orange-line:where(.astro-7XAARZHW) { - height: 16px; - width: 100%; - background: #ee5007; -} -.buttons:where(.astro-7XAARZHW) { - display: flex; - flex-direction: column; - align-items: center; -} -@media screen and (min-width: 420px) { - .buttons:where(.astro-7XAARZHW) { - flex-direction: row; - align-items: flex-start; - } -} -.buttons:where(.astro-7XAARZHW)>a:where(.astro-7XAARZHW) { - display: flex; - flex-direction: row; - justify-content: center; - align-items: center; - padding: 8px 16px; - width: 210px; - height: 50px; - font-weight: 600; - font-size: 22px; - line-height: 100%; - color: #fff; - text-decoration: none; - border-radius: 4px; - margin-top: 53px; -} -@media screen and (min-width: 420px) { - .buttons:where(.astro-7XAARZHW)>a:where(.astro-7XAARZHW) { - margin-right: 40px; - } -} -#quickstart:where(.astro-7XAARZHW) { - background: #ee5007; - box-shadow: 0 0 2px #0003, 0 1px 2px #0000001a, inset 0 -1px #0003; - border-radius: 4px; -} -#download:where(.astro-7XAARZHW) { - border: 1px solid #8facd4; - box-shadow: inset 0 -1px #0003; - filter: drop-shadow(0px 0px 2px rgba(0, 0, 0, .2)) drop-shadow(0px 1px 2px rgba(0, 0, 0, .1)); - color: #8facd4; -} -footer:where(.astro-7XAARZHW) { - background: #e06948; - height: 250px; - display: flex; - flex-direction: row; - justify-content: center; - align-items: center; -} -.sections-title:where(.astro-7XAARZHW) { - font-weight: 600; - font-size: 32px; - line-height: 50px; - color: #0a2239; - margin-top: 123px; - text-align: center; -} -.section-wrapper { - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; -} -.sections { - display: flex; - flex-direction: column; - align-items: center; - max-width: 1200px; - justify-content: center; -} -.sections[aria-current=horizontal] { - flex-direction: column; - width: 100%} -@media screen and (min-width: 1024px) { - .sections[aria-current=horizontal] { - flex-direction: row; - } -} -.main-section { - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - padding: 64px 48px; - box-sizing: border-box; - width: 100%} -.main-section>.sections { - padding-bottom: 0; -} -.main-section>.title { - font-weight: 600; - font-size: 42px; - line-height: 52px; - text-align: center; - color: #161b33; - margin: 43px 0; -} -.secondary-section { - display: flex; - flex-direction: column; - border-bottom: 1px solid rgba(0, 0, 0, .1); - padding: 32px 0; -} -@media screen and (min-width: 1024px) { - .secondary-section { - flex-direction: row; - } -} -.secondary-section img { - margin-top: 80px; - margin-bottom: 40px; -} -.secondary-section:first-child { - padding-top: 0; -} -.secondary-section:last-child { - border-bottom: 0; - padding-bottom: 0; -} -.secondary-section>.title { - min-width: 0px; - width: 100%; - display: flex; - flex-direction: column; - align-items: center; - justify-content: flex-start; - font-family: Noto Sans Mono; - font-weight: 700; - font-size: 24px; - line-height: 38px; - color: #161b33; - text-align: center; - margin-bottom: 32px; -} -@media screen and (min-width: 1024px) { - .secondary-section>.title { - min-width: 480px; - } -} -.sections[aria-current=horizontal] p { - text-align: center; -} -.sections[aria-current=horizontal]>.secondary-section { - flex-direction: column; - align-items: center; - border-bottom: 0; - padding: 0; - width: 100%; - margin: 0; -} -@media screen and (min-width: 1024px) { - .sections[aria-current=horizontal]>.secondary-section { - margin: 0 24px; - } -} -.sections[aria-current=horizontal]>.secondary-section .image-container { - height: 100px; - display: flex; - justify-content: flex-end; -} -.sections[aria-current=horizontal]>.secondary-section .image-container-removeHeight { - display: flex; - justify-content: flex-end; -} -.sections[aria-current=horizontal]>.secondary-section img { - margin: 0; -} -.sections[aria-current=horizontal]>.secondary-section:first-child { - margin-left: 0; -} -.sections[aria-current=horizontal]>.secondary-section:last-child { - margin-right: 0; -} -.sections[aria-current=horizontal]>.secondary-section>.title { - padding: 32px 12px; - flex-grow: 1; - width: 100%; - min-width: auto; - max-width: 340px; -} -.sections[aria-current=horizontal]>.secondary-section>.title>div { - margin: 32px 0; -} -.tertiary-section { - margin-bottom: 24px; - border-bottom: 1px solid rgba(0, 0, 0, .1); - width: 100%} -.tertiary-section:last-child { - border-bottom: 0px; - margin-bottom: 0; -} -.tertiary-section>.title { - font-family: Noto Sans Mono; - font-style: normal; - font-weight: 700; - font-size: 24px; - line-height: 33px; - color: #161b33; - margin-bottom: 16px; -} -.sections p { - font-family: Noto Sans; - font-weight: 400; - font-size: 16px; - line-height: 160%; - letter-spacing: .01em; - color: #344079; - margin-top: 0; - margin-bottom: 24px; -} diff --git a/docs/assets/scss/_variables_project.scss b/docs/assets/scss/_variables_project.scss deleted file mode 100644 index bbc6b7ade..000000000 --- a/docs/assets/scss/_variables_project.scss +++ /dev/null @@ -1,29 +0,0 @@ -/* - -Add styles or override variables from the theme here. - -*/ - -$primary: #ee5007; -$secondary: #e1e5ee; -$enable-gradients: true; -$enable-rounded: false; -$enable-shadows: true; -$google_font_name: "Noto Sans"; -$google_font_family: "Noto+Sans:0,300;0,400;0,500;0,600;0,700;1,300;1,400;1,500;1,600;1,700"; -$td-enable-google-fonts: false; - - -a { - color: #009ffd -} - -a:hover { - color: #03153a -} - -// Disables text "Kairos" in the menu. Logo contains it already -.td-navbar .navbar-brand__name { - display: none; -} - \ No newline at end of file diff --git a/docs/config.toml b/docs/config.toml deleted file mode 100644 index 42d8c4a19..000000000 --- a/docs/config.toml +++ /dev/null @@ -1,172 +0,0 @@ -baseURL = "https://kairos.io" -title = "Kairos" - -enableRobotsTXT = true - -# Hugo allows theme composition (and inheritance). The precedence is from left to right. -theme = ["docsy"] - -# Will give values to .Lastmod etc. -enableGitInfo = true -# Language settings -contentDir = "content/en" -defaultContentLanguage = "en" -defaultContentLanguageInSubdir = false -# Useful when translating. -enableMissingTranslationPlaceholders = true -enableEmoji = true - -disableKinds = ["taxonomy", "taxonomyTerm"] - -# Highlighting config -pygmentsCodeFences = true -pygmentsUseClasses = false -# Use the new Chroma Go highlighter in Hugo. -pygmentsUseClassic = false -#pygmentsOptions = "linenos=table" -# See https://help.farbox.com/pygments.html -pygmentsStyle = "tango" - -# Configure how URLs look like per section. -[permalinks] -blog = "/:section/:year/:month/:day/:slug/" - -## Configuration for BlackFriday markdown parser: https://github.com/russross/blackfriday -[blackfriday] -plainIDAnchors = true -hrefTargetBlank = true -angledQuotes = false -latexDashes = true - -# Image processing configuration. -[imaging] -resampleFilter = "CatmullRom" -quality = 75 -anchor = "smart" - -[services] -[services.googleAnalytics] -# Comment out the next line to disable GA tracking. Also disables the feature described in [params.ui.feedback]. -id = "UA-00000000-0" - -# Language configuration - -[languages] -[languages.en] -title = "Kairos - The immutable Linux meta-distribution for edge Kubernetes" -description = "The immutable edge Kubernetes" -languageName ="English" -# Weight used for sorting. -weight = 1 - -[[menu.main]] - name = "Contribute" - weight = 50 - url = "https://github.com/kairos-io/kairos/contribute" - pre = "" - post = "" -[[menu.main]] - name = "Contribution guidelines" - weight = 50 - url = "https://github.com/kairos-io/kairos/blob/master/CONTRIBUTING.md" - pre = "" - post = "" -[markup] - [markup.goldmark] - [markup.goldmark.renderer] - unsafe = true - [markup.highlight] - # See a complete list of available styles at https://xyproto.github.io/splash/docs/all.html - style = "tango" -# Everything below this are Site Params - -[params] -copyright = "Kairos authors" -privacy_policy = "https://policies.google.com/privacy" - -prism_syntax_highlighting = true - -# First one is picked as the Twitter card image if not set on page. -# images = ["images/project-illustration.png"] - -# Menu title if your navbar has a versions selector to access old versions of your site. -# This menu appears only if you have at least one [params.versions] set. -version_menu = "Releases" - -# Repository configuration (URLs for in-page links to opening issues and suggesting changes) -github_repo = "https://github.com/kairos-io/kairos" -# An optional link to a related project repo. For example, the sibling repository where your product code lives. -github_project_repo = "https://github.com/kairos-io/kairos" - -# Specify a value here if your content directory is not in your repo's root directory -github_subdir = "docs" -github_branch = "master" -# Google Custom Search Engine ID. Remove or comment out to disable search. -#gcs_engine_id = "011737558837375720776:fsdu1nryfng" - -# Enable Algolia DocSearch -algolia_docsearch = false - -# Enable Lunr.js offline search -offlineSearch = true - -[params.softwareVersions] -k3s = "k3sv1.26.3+k3s1" -kairos = "v2.0.0" -flavor = "debian" -armFlavor = "alpine" -registryURL = "quay.io/kairos" -# User interface configuration -[params.ui] -# Enable to show the side bar menu in its compact state. -sidebar_menu_compact = false -# Set to true to disable breadcrumb navigation. -breadcrumb_disable = true -# Set to true to hide the sidebar search box (the top nav search box will still be displayed if search is enabled) -sidebar_search_disable = true -# Set to false if you don't want to display a logo (/assets/icons/logo.svg) in the top nav bar -navbar_logo = true -# Set to true to disable the About link in the site footer -footer_about_disable = false - -# Adds a H2 section titled "Feedback" to the bottom of each doc. The responses are sent to Google Analytics as events. -# This feature depends on [services.googleAnalytics] and will be disabled if "services.googleAnalytics.id" is not set. -# If you want this feature, but occasionally need to remove the "Feedback" section from a single page, -# add "hide_feedback: true" to the page's front matter. -[params.ui.feedback] -enable = true -# The responses that the user sees after clicking "yes" (the page was helpful) or "no" (the page was not helpful). -yes = 'Awesome! Glad to hear it! Please tell us how we can improve.' -no = 'Oh snap! Sorry to hear that. Please tell us how we can improve.' - -[params.links] -[[params.links.developer]] - name = "GitHub" - url = "https://github.com/kairos-io/kairos" - icon = "fab fa-github" - desc = "Development takes place here!" -[[params.links.user]] - name = "Matrix" - url = "https://matrix.to/#/#kairos-io:matrix.org" - icon = "fa fa-message" - desc = "Join us on Matrix!" -[[params.links.user]] - name = "GitHub discussions" - url = "https://github.com/kairos-io/kairos/discussions" - icon = "fa fa-comments" - desc = "Questions?" -[[params.links.user]] - name = "Office hours calendar" - url = "https://calendar.google.com/calendar/embed?src=c_6d65f26502a5a67c9570bb4c16b622e38d609430bce6ce7fc1d8064f2df09c11%40group.calendar.google.com&ctz=Europe%2FRome" - icon = "fa fa-calendar" - desc = "Join us in our Office hours!" -[[params.links.user]] - name = "Slack" - url = "https://join.slack.com/t/spectrocloudcommunity/shared_invite/zt-1k7wsz840-ugSsPKzZCP5gkasJ0kNpqw" - icon = "fab fa-slack" - desc = "Join us on Slack!" -[[params.links.user]] - name = "Newsletter" - url = "https://kairoslinux.substack.com/" - icon = "fa fa-envelope" - desc = "Subscribe to the newsletter!" diff --git a/docs/content/en/blog/Kairos-at-FOSDEM-2023.md b/docs/content/en/blog/Kairos-at-FOSDEM-2023.md deleted file mode 100644 index 671338bfd..000000000 --- a/docs/content/en/blog/Kairos-at-FOSDEM-2023.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: "Kairos at FOSDEM 2023" -date: 2023-02-07T10:53:13+01:00 -author: Mauro Morales ([Twitter](https://twitter.com/mauromrls)) ([GitHub](https://github.com/mauromorales)) ---- - -I recently had the opportunity to attend FOSDEM 2023 and share a bit about the Kairos project. In this post I want to summarize what I presented and share other interesting presentations I attended, which I believe are relevant for Kairos and our community. - -## How we build and maintain Kairos - -I had the opportunity to share about [How we build and maintain Kairos](https://fosdem.org/2023/schedule/event/kairos/). In first half of the presentation, I introduce the different elements that make Kairos a great OS for Edge Kubernetes. During the second half of the presentation you will get an overview of how the Kairos Factory works, starting from those different Linux distributions all the way up to producing Kairos core and standard images. Because my presentation took place in the Distributions Devroom, I put some extra emphasis on the challenges we have to be distribution agnostic. - -The talk is intended to newcomers, so I made an effort to describe things in a simple and welcoming language. However, I think it can also be interesting for those who might already know about Kairos but wonder how to extend the core and standard images, or simply have a better understanding of how all the pieces interconnect. - -Like I mentioned, the presentation took place in the Distributions Devroom and we're very thankful to them for hosting us. While it was a great experience and the talk seemed to have a good reception, I now realize that the topic is probably more relevant for a different devroom, for example, the [ -Image-based Linux and Secure Measured Boot devroom -](https://fosdem.org/2023/schedule/track/image_based_linux_and_secure_measured_boot/), which I'll make sure to send proposals next year. - -## Other talks which are relevant to Kairos - -There were other interesting presentations I had the opportunity to attend, which I think are also relevant to Kairos and our community. These would be my top picks: - -If you're completely new to the concepts of Image-Based Linux, Unified Kernel Image or Discoverable Disk Image, I'd recommend checking Luca Bocassi's talk [Introducing and decoding image-based Linux terminology and concepts](https://fosdem.org/2023/schedule/event/image_linux_secureboot_uki_ddi_ohmy/). As someone who very recently joined the Kairos project, I still get a bit lost with all the different technologies used in Image-Based Linux. The presenter made a good job clarifying some of these technologies and how they work together. - -One of the key presentations in my opinion was Lennart Poettering's, [Measured Boot, Protecting Secrets and you](https://fosdem.org/2023/schedule/event/image_linux_secureboot_tpm/), where he talks about Trusted Plataform Modules and upcoming functionality in systemd. I'm pretty sure there will be some of these features which will be relevant for Kairos sooner rather than later. - -Last but not least, there was an interesting talk by Gabriel Kerneis about [User-friendly Lightweight TPM Remote Attestation over Bluetooth](https://fosdem.org/2023/schedule/event/image_linux_secureboot_ultrablue/). My guess is that we will continue seeing different methods to do and simplify attestation and because one of our goals at the Kairos project is to be as friendly as we can to our user base, then I can only imagine we will end up introducing some sort of remote attestation technologies like Ultrablue in the future. - -## Conclusion - -FOSDEM is a very important conference when it comes to free and open source software and I'm very happy that Kairos was present. First of all because I think the work we're doing with Kairos is helping solve some of the most challenging issues of running cloud native applications on the edge, but also because as an open source project, it was nice to introduce ourselves to the community there and start a conversation. Expect us to keep engaging with you in further editions of FOSDEM and other conferences! diff --git a/docs/content/en/blog/Kairos-at-the-KCD-Amsterdam-and-Paris-2023.md b/docs/content/en/blog/Kairos-at-the-KCD-Amsterdam-and-Paris-2023.md deleted file mode 100644 index 70ec09de6..000000000 --- a/docs/content/en/blog/Kairos-at-the-KCD-Amsterdam-and-Paris-2023.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: "Kairos at the KCD Amsterdam and Paris 2023" -date: 2023-03-09T10:53:13+01:00 -author: Mauro Morales ([Twitter](https://twitter.com/mauromrls)) ([GitHub](https://github.com/mauromorales)) ---- - -We recently had the opportunity to sponsor two Kubernetes events, [KCD Amsterdam][amsterdam] and [KCD Paris][paris]. This blog post, is a summary about my personal experience attending them. - -Let me start by saying, that I'm fairly new to Kubernetes and its community :wave:. I know this project is big and that there are many companies building products and services around it, or have an interest in adopting it. So, I was very curious to see what kind of people I was going to meet and understand how Kairos could help them. - -Most attendees that approached us at the Kairos booths, were hearing about Kairos for the first time, and genuinely wanted to know what the project was about. I feel confident to say this, because we didn't bring fancy prizes to give away and yet most of them would happily stay with us for 5, 10 and up to 15 minutes hearing about our features and engaging in conversation. - -_If you're reading this and would like to know about those cool features I'd recommend going checking out the [Getting Started](/docs/getting-started/), [Web UI](/docs/installation/webui/), [P2P Network](/docs/architecture/network/) and [AuroraBoot](/docs/reference/auroraboot/)_ - -When you're in the trenches building a product, talking to users or potential users is super valuable because it lets you see first hand, what kind of issues they are trying to solve. I don't like building projects just because they are cool. To me, it's important that they make people's life easier. Some of the folks who reached to us, had clear problems in mind, and they didn't shy to make hard questions about the internals of Kairos, our project's governance and beyond. I'm very pleased to say that some of them left the booth with a smile on their face, because they might have found a good fit. - -While I didn't get to attend any of the talks, I saw some really interesting topics, some of them from fantastic organizations like CERN! However, what I did do a bit, was to speak to some of the folks in the other booths, just to see what they were up to :mag: and most importantly to see if there were chances our different projects could leverage each other out :raised_hands:. - -Last but not least, let me thank everyone :bow: who attended our booth for your valuable time and feedback. I think every one of my colleagues will agree that we're committed to building a great product, that solves real world problems and we plan to use that feedback accordingly. We have a passion for open-source and we understand that this means much more than just great engineering and best practices. It also means being there for you, the community. - -[amsterdam]: https://community.cncf.io/events/details/cncf-kcd-netherlands-presents-kubernetes-community-days-amsterdam-2023/ -[paris]: https://community.cncf.io/events/details/cncf-kcd-france-presents-kubernetes-community-days-france-2023/ diff --git a/docs/content/en/blog/_index.md b/docs/content/en/blog/_index.md deleted file mode 100644 index 9ad71a40a..000000000 --- a/docs/content/en/blog/_index.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Blog -menu: {main: {weight: 50}} ---- diff --git a/docs/content/en/blog/home-lab-with-kairos-and-wireguard.md b/docs/content/en/blog/home-lab-with-kairos-and-wireguard.md deleted file mode 100644 index bf6113a6f..000000000 --- a/docs/content/en/blog/home-lab-with-kairos-and-wireguard.md +++ /dev/null @@ -1,296 +0,0 @@ ---- -title: "Access your home-lab Kairos cluster over a Wireguard VPN" -date: 2023-03-29T10:53:13+01:00 -author: Dimitris Karakasilis([Personal page](https://dimitris.karakasilis.me)) ([GitHub](https://github.com/jimmykarily)) ([Codeberg](https://codeberg.org/dkarakasilis/)) ---- - -## The problem - -You got yourself a Rabserry Pi (or more), and you want to put them to good use. -You decide to make a Kubernetes cluster out of them, so that you can utilise the resources better, use familiar tools and implement infrastructure-as-code. - -Up to this point, kudos to you for demanding no less than a real cloud from your home infra. - -Like a smart person you are, you probably used [Kairos](https://kairos.io/) to create your cluster and it's now up and running. -It's now time to run some workloads. - -Here is my list if you need some ideas: - -- A self-hosted Dropbox alternative (e.g. [Seafile](https://www.seafile.com/en/home/), [NextCloud](https://nextcloud.com/) or other) -- [Pihole](https://pi-hole.net/) -- An [mqtt](https://mqtt.org/) broker for your IoT projects -- Your own [Gitea](https://gitea.io/en-us/) instance -- Your own ChatGPT alternative (e.g. using [lama-cli](https://github.com/go-skynet/llama-cli) or [serge](https://github.com/nsarrazin/serge)) - -None of these workloads is intended for public access. There are ways to expose the cluster to the world (e.g. like I described [in another post](https://dimitris.karakasilis.me/2022/12/26/self-hosted-ci.html)) -but it would be better if only devices within a VPN would have access to it. - -Once again, there are many VPN solutions out there, but for this blog post, we'll go with [Wireguard](https://www.wireguard.com/). - -So here is the problem in one sentence: - -> "How do we expose our (possibly behind NAT) cluster, to machines inside the same Wireguard VPN?" - -_"NAT" is the main part of the problem because otherwise this would simply be a blog post on how to create a Wireguard VPN. There are many nice tutorials already out there for that._ - -## A Solution - -While trying to solve the problem, I learned 2 things about Wireguard that I didn't know: - -1. Wireguard doesn't distinguish between a "server" and a "client". All peers are made equal. -2. Wireguard doesn't provide a solution for NAT traversal. How you access nodes behind NAT, is up to you. - -So imagine you have your cluster behind your home router (NAT) and your mobile phone on another network (behind NAT too) trying to access a service on the cluster. -That's not possible, unless there is some public IP address that somehow forwards requests to the cluster. - -And that's the idea this solution is based on. - -### High level view - -![Image describing the solution](/images/kairos-over-wireguard.svg) - -The idea is almost similar to the one I described [in another post](https://dimitris.karakasilis.me/2022/12/26/self-hosted-ci.html). -The only difference is, that this time we expose the cluster only to machines inside the VPN. - -Prerequisites: - -- A VM with a public IP address and SSH access (as small as it gets, it's good enough) -- `kubectl` access to the cluster we want to expose (it doesn't have to be Kairos, even [`k3d`](https://k3d.io) and [`kind`](https://kind.sigs.k8s.io/) will do) -- A machine to test the result (a smartphone where Wireguard can be installed is fine) - -### Step by step - -From this point on, we will use the IP address `1.2.3.4` as the public IP address of the VM in the cloud. -Replace it with the one matching your VM. We also assume, that the user with SSH access is `root`. Replace if necessary. - -#### Setup the cloud VM - -SSH to the machine: - -```bash -$ ssh root@1.2.3.4 -``` - -Create Wireguard keys: - -```bash -$ wg genkey | tee privatekey | wg pubkey > publickey -``` - -Create Wireguard config: - -```bash -$ cat << EOF > /etc/wireguard/wg0.conf -[Interface] -Address = 192.168.6.1/24 -PrivateKey = $(cat privatekey) -ListenPort = 41194 - -# Mobile client -[Peer] -PublicKey = -AllowedIPs = 192.168.6.2/32 -EOF -``` - -Start and enable the Wireguard service: - -``` -$ sudo systemctl enable --now wg-quick@wg0 -``` - -Allow binding non-loopback interfaces when creating an SSH reverse tunnel -by setting `GatewayPorts clientspecified` in `/etc/ssh/sshd_config`. - -#### Setup the test machine (mobile?) - -On some computer with `wg` installed, generate the keys: - -```bash -$ wg genkey | tee privatekey | wg pubkey > publickey -``` - -Create the Wireguard configuration. Follow the instructions for your favorite application. -For Android, you can use this: https://play.google.com/store/apps/details?id=com.wireguard.android - -If setting up a Linux machine, you can create the configuration like this: - -```bash -$ cat << EOF > /etc/wireguard/wg0.conf -[Interface] -Address = 192.168.6.2/24 -PrivateKey = $(cat privatekey) - -# The cloud VM -[Peer] -PublicKey = -AllowedIPs = 192.168.6.1/32 -Endpoint = 1.2.3.4:41194 -EOF -``` - -Start and enable the Wireguard service. If on a Linux machine, something like this will do: - -``` -$ sudo systemctl enable --now wg-quick@wg0 -``` - -On a mobile, follow the instructions of your application. - -After a while, your client should be able to ping the IP address of the VM: `192.168.6.1`. -You may find the output of `wg show` useful, while waiting for the peers to connect. - -#### Setup the cluster - -Deploy the helper Pod. We will use an image created [with this Dockerfile](https://codeberg.org/dkarakasilis/self-hosted-ci/src/branch/main/image) and -published [here](https://quay.io/repository/jimmykarily/nginx-ssh-reverse-proxy). The image's entrypoint works with a config -described [here](https://codeberg.org/dkarakasilis/self-hosted-ci/src/commit/20d7c6cbf70cd5318309362b0897e6aeb9842b82/image/start.sh#L5-L27). -The image is not multiarch, but there is one suitable for RasberryPi 4 (see the comment in the file). - -If you are are going to create a fresh Kairos cluster, you can use a config like the following to automatically set up the helper Pod (make sure you replace the `id_rsa` and `id_rsa.pub` keys). -If you prefer to not have the keys stored on your Kairos host filesystem, you can simply create the same resources using `kubectl apply -f` after your cluster is up an running. - -``` -#cloud-config - -users: -- name: kairos -passwd: kairos - -stages: -after-install-chroot: - - files: - - path: /var/lib/rancher/k3s/server/manifests/rproxy-pod.yaml - content: | - --- - apiVersion: v1 - data: - id_rsa: the_vms_private_key_in_base64 - id_rsa.pub: the_vms_public_key_in_base64 - kind: Secret - metadata: - name: jumpbox-ssh-key - type: Opaque - - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: proxy-config - data: - config.json: | - { - "services": [ - { - "bindIP": "192.168.6.1", - "bindPort": "443", - "proxyAddress": "traefik.kube-system.svc", - "proxyPort": "443" - }, - { - "bindIP": "192.168.6.1", - "bindPort": "80", - "proxyAddress": "traefik.kube-system.svc", - "proxyPort": "80" - } - ], - "jumpbox": { - "url": "1.2.3.4", - "user": "root", - "sshKeyFile": "/ssh/id_rsa" - } - } - - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - annotations: - name: nginx-ssh-reverse-proxy - spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/instance: nginx-ssh-reverse-proxy - app.kubernetes.io/name: nginx-ssh-reverse-proxy - template: - metadata: - labels: - app.kubernetes.io/instance: nginx-ssh-reverse-proxy - app.kubernetes.io/name: nginx-ssh-reverse-proxy - spec: - containers: - - name: proxy - # Change to quay.io/jimmykarily/nginx-ssh-reverse-proxy-arm64:latest - # if you are running on a RasberryPi 4 - image: quay.io/jimmykarily/nginx-ssh-reverse-proxy:latest - command: ["/start.sh", "/proxy-config/config.json"] - imagePullPolicy: Always - volumeMounts: - - name: ssh-key - mountPath: /ssh - - name: config-volume - mountPath: /proxy-config/ - volumes: - - name: ssh-key - secret: - secretName: jumpbox-ssh-key - defaultMode: 0400 - - name: proxy-config - - name: config-volume - configMap: - name: proxy-config - -``` - -In a nutshell, the config above is creating a reverse SSH tunnel from the VM -to the Pod. Inside the Pod, nginx redirects traffic to the traefik load balancer running -on the cluster. This has the effect, that any request landing on the VM on ports 80 and 443 -will eventually reach the Traefik instance inside the cluster on ports 80 and 443. -As a result, you can point any domain you want to the VM and it will reach the corresponding Ingress defined on your cluster. - -{{% alert color="info" %}} - -**NOTE:** The SSH tunnel will only bind the IP address `192.168.6.1` on the VM, which means, anyone trying to access the VM using its public IP address, will not be able to access the cluster. Only machines that can talk to `192.168.6.1` have access, in other words, machines inside the VPN. - -{{% /alert %}} - -#### Test the connection - -- Try to access the cluster with the VPN IP address (should work). - From your test peer, open `http://192.168.6.1`. You should see a 404 message from Traefik. - You can also verify it is a response from Traefik in your cluster, by calling curl - on the `https` endpoint (on a "default" k3s installation): - - ```bash - $ curl -k -v https://192.168.6.1 2>&1 | grep TRAEFIK - * subject: CN=TRAEFIK DEFAULT CERT - * issuer: CN=TRAEFIK DEFAULT CERT - ``` - -- Try to access the cluster with domain pointing to the VPN IP address (should work) - You can create a wildcard DNS record and point it to the VPN IP address if - you want to make it easier for people to access the services you are running. - E.g. by creating an A record like this: `*.mydomainhere.org -> 192.168.6.1` - you will be able create Ingresses for your applications like: - `app1.mydomainhere.org`, `app2.mydomainhere.org`. - -- Try to access the cluster using the public IP address (should not work) - - ```bash - $ curl http://1.2.3.4 - ``` - This command should fail to connect to your cluster - - -### Conclusion - -For non-critical workloads, when 100% uptime is not a hard requirement, the solution we described allows one to use services that would otherwise cost multiple times more by hosting -those on their own hardware. It does so, without exposing the home network to the public. - -If you liked this solution or if you have comments, questions or recommendations for improvements, please reach out! - -### Useful links - -- [Kairos documentation](https://kairos.io/docs/) -- [WireGuard documentation](https://www.wireguard.com/quickstart/) diff --git a/docs/content/en/blog/kairos-canonical.md b/docs/content/en/blog/kairos-canonical.md deleted file mode 100644 index bcbde381f..000000000 --- a/docs/content/en/blog/kairos-canonical.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: "Kairos, SpectroCloud, and Canonical Collaborate to Deliver Revolutionary Telco Radio Edge Solution" -date: 2023-03-13 -linkTitle: "Kairos, SpectroCloud, and Canonical Collaboration at MWC" -description: | - Kairos, the open-source distributed infrastructure platform, has collaborated with SpectroCloud and Canonical to develop a revolutionary Telco Radio Edge solution. The solution leverages the latest advances in OpenRAN automation and distributed compute management, and is set to take center stage at this year's Mobile World Congress. - -author: Ettore Di Giacinto ([Twitter](https://twitter.com/mudler_it)) ([GitHub](https://github.com/mudler)) ---- - -Hello Kairos community! - -We are thrilled to announce that **Kairos** has been used as a key building block for a revolutionary Telco Radio Edge solution developed in collaboration with [Spectro Cloud](https://www.spectrocloud.com/) and [Canonical](https://canonical.com/). This cutting-edge solution showcases the latest advances in OpenRAN automation and distributed compute management, and took center stage at this year's Mobile World Congress. - -The Telco Radio Edge solution leverages the power of Kairos, Ubuntu Pro 22.04 LTS with RT kernel, and MicroK8s CAPI provider to deliver highly distributed edge node onboarding, secure deployment, and substrate provisioning. With this innovative technology stack, we’ve enabled OpenRAN o-DU service orchestration at scale, while optimizing performance, scalability, reliability, and security. - -This is an exciting collaboration between the Kairos project, Spectro Cloud and Canonical to develop a solution that is highly performant, efficient, and scalable. The demos that have been presented at MWC showcase the advanced capabilities of the MicroK8s CAPI provider, and highlight the power of Kairos as a building block for distributed infrastructure substrates that can host even the most demanding modern OpenRAN o-DU, 5G UPF or AI/ML use-cases at scale. - -This is a true testament to the power of open-source technologies and community collaboration, and we can't wait to see what new possibilities this partnership will bring. - -Thank you for your continued support and enthusiasm for Kairos! - -**Details**: You can learn more about the Talco Radio Edge solution on the Ubuntu blog https://ubuntu.com/blog/meet-canonical-at-mwc-barcelona-2023 or at https://ubuntu.com/blog/canonical-at-mwc and watch it in action here: https://www.youtube.com/watch?v=wUCSK0O8Ro4 diff --git a/docs/content/en/blog/media-section.md b/docs/content/en/blog/media-section.md deleted file mode 100644 index 99d9729ef..000000000 --- a/docs/content/en/blog/media-section.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Media Section" -date: 2023-01-04T17:17:11+01:00 -author: Mauro Morales ([Twitter](https://twitter.com/mauromrls)) ([GitHub](https://github.com/mauromorales)) ---- - -We've added a new media section so it's easy to find the different videos and articles about Kairos. To access it, go to the Documentation and at the bottom of the left menu, you will find a link called Media. - -You can also [click here]({{< ref "/media" >}} "Media") to go check it out. diff --git a/docs/content/en/blog/release-v1.5.md b/docs/content/en/blog/release-v1.5.md deleted file mode 100644 index ed5534926..000000000 --- a/docs/content/en/blog/release-v1.5.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: "Kairos release v1.5" -date: 2023-01-27 -linkTitle: "Announcing v1.5 Kairos release" -description: "Introducing Kairos 1.5: A Smarter, More Secure Way to Manage Your Infrastructure" -author: Ettore Di Giacinto ([Twitter](https://twitter.com/mudler_it)) ([GitHub](https://github.com/mudler)) ---- -

-
- kairos-white-column 5bc2fe34 -
-
-

- -We are thrilled to announce the release of Kairos version 1.5, a major update that brings significant improvements to user experience and security. With this release, we have made it even easier for you to install and set up Kairos, as well as better protect your user data. Our community has been an invaluable source of feedback, bug reports, and contributions, and we are grateful for their support. - -You can find Kairos core images at https://github.com/kairos-io/kairos/releases/tag/v1.5.0 and images with k3s pre-bundled here: https://github.com/kairos-io/provider-kairos/releases/tag/v1.5.1. - -## Effortless Installation with the WebUI Installer - -![WebUI](https://user-images.githubusercontent.com/2420543/214573939-31f887b8-890c-4cce-a02a-0100198ea7d9.png) - -Gone are the days of complicated command-line instructions. With the new [WebUI installer](/docs/installation/webui/), installation and setup are a breeze. Simply follow the steps on the web page, and you'll be up and running in no time. You can also use [our core images as an installer](/docs/examples/core/). Take a look at this gif to see the WebUI installer in action: - -![Peek 2023-01-04 01-04](https://user-images.githubusercontent.com/2420543/210461794-fb80ad90-5d11-479d-945d-2e3ba3890435.gif) - -## Protect Your Data with User Data Encryption at the Edge - -Kairos 1.5 now allows you to encrypt your user data with ease, keeping it secure from prying eyes. Encryption is done via TPM and optionally with the Kairos KMS (Key Management Server) for external authentication and management of encrypted secrets. Check out our [documentation](/docs/advanced/partition_encryption) for more information on partition encryption. - -## OS updates - -We've added RockyLinux and Debian to our list of supported releases, giving you more options to run Kairos on both stable and feature-rich operating systems. We've also updated our Alpine support, so you can now run Kairos on the latest version of Alpine Linux. - -## Extend Kairos with Custom Deployment Models (`bundles`) - -Kairos 1.5 allows you to extend the configuration of your node with custom, container-based deployment models defined as `bundles`. Check out our [documentation](/docs/advanced/bundles) and [examples](/docs/examples/bundles) to see how to deploy `MetaLB`. `Kubevirt` and `MetalLB` bundles are also availble in the [community-bundles](https://github.com/kairos-io/community-bundles) repository. - ---- - -For a full list of changes, see the [Changelog](https://github.com/kairos-io/kairos/releases/tag/v1.5.0). We hope you find these updates useful and as always, let us know if you have any questions or feedback. Thanks for using Kairos! diff --git a/docs/content/en/blog/release-v1.6.md b/docs/content/en/blog/release-v1.6.md deleted file mode 100644 index 3be055993..000000000 --- a/docs/content/en/blog/release-v1.6.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: "Kairos release v1.6" -date: 2023-02-26 -linkTitle: "Announcing v1.6 Kairos release" -description: "Introducing Kairos 1.6: Get ready to boot with AuroraBoot!" -author: Ettore Di Giacinto ([Twitter](https://twitter.com/mudler_it)) ([GitHub](https://github.com/mudler)) ---- -

-
- kairos-white-column 5bc2fe34 -
-
-

- -Kairos is a cloud-native meta-Linux distribution that brings the power of public cloud to your on-premises environment. With Kairos, you can build your own cloud with complete control and no vendor lock-in. It allows you to easily spin up a Kubernetes cluster with the Linux distribution of your choice, and manage the entire cluster lifecycle with Kubernetes. - -#### Why you should try Kairos: -Kairos provides a wide range of use cases, from Kubernetes applications to appliances and more. You can provision nodes with your own image or use Kairos releases for added flexibility. Kairos also simplifies day-2 operations like node upgrades. It provides the benefits of a unified, cloud-native approach to OS management. - -#### What you can do with Kairos: -With Kairos, you can create an immutable infrastructure that stays consistent and free of drift with atomic upgrades. You can manage your cluster's entire lifecycle with Kubernetes, from building to upgrading. Kairos also allows you to automatically create multi-node, single clusters that span across regions for maximum flexibility and scalability. - -## Kairos 1.6.0 release - -Kairos 1.6.0 has just been released, and we are thrilled to share the latest updates and improvements to the Kairos project. This release includes bug fixes, small improvements to the Kairos core codebase, and the introduction of AuroraBoot, a tool that simplifies bootstrapping of Kairos nodes. In this post, we will explore how AuroraBoot works and its benefits for users deploying Kairos. - -### What is AuroraBoot? - -[AuroraBoot](https://kairos.io/docs/reference/auroraboot/) is a tool designed to make the process of bootstrapping Kairos machines quick, simple, and efficient. It is specifically designed for the Kairos operating system and provides a comprehensive solution for downloading required artifacts and provisioning a machine, both from network or manually via flashing to USB stick. - -AuroraBoot simplifies the bootstrapping process by automating several steps, such as downloading required files, verifying their authenticity, and providing a streamlined interface for customizing the installation media. With AuroraBoot, users can prepare the environment for network-based bootstrapping, download the necessary release assets, and also customize the installation media for USB-based mass-installations. - -### The Benefits of AuroraBoot -With AuroraBoot, users can prepare multiple nodes in a lab before shipment or deploy Kairos nodes in a network segment where workload can already be sent to (running AuroraBoot in an already-existing downstream cluster). Additionally, AuroraBoot offers a simple, intuitive, and streamlined way to deploy Kairos automatically and manually. It makes the deployment process faster, more efficient, and less error-prone. Besides, it does leverage the DHCP server already existing in the network for booting, requiring zero-configuration. - -You can see AuroraBoot in action here, with [a full e2e example](https://kairos.io/docs/examples/p2p_e2e/) on how to use it with p2p in Kairos, and in the video below: - -{{< youtube id="7Vym18wz9Uw" title="Kairos and libp2p" >}} - -## Improvements to the WebUI for a simplified user experience - -The WebUI got several improvements, we have integrated the documentation inside the web interface, and now can be accessed also offline. The configuration schema is validated and a message is displayed if the configuration is incorrect. You can see how it works here - -[Screencast from 2023-02-21 15-24-59.webm](https://user-images.githubusercontent.com/433958/221510120-ce43eb66-e8c0-4b91-885e-3a213fac896b.webm) - -## Other Improvements in Kairos 1.6.0 - -Aside from AuroraBoot, Kairos 1.6.0 includes several improvements and bugfixes, including: -- Integration of documentation into the Web UI -- Initial support for schema validation in the WebUI and the installer -- Support for Rocky Linux in provider builds -- Renaming of kairos-agent and addition of SHA256 signatures -- Addition of custom mounts -- Fix for DHCP hostname issues -- Fix for encryption reset failures -- Fix for systemd-networkd hostname settings -- Fix for Tumbleweed ISO - -You can check the full changelog at: https://github.com/kairos-io/kairos/releases/tag/v1.6.0 - -## Conclusion - -Kairos 1.6.0 is a significant step forward in simplifying the deployment process of Kairos nodes. With AuroraBoot, users can deploy Kairos faster, more efficiently, and with less risk of error. Additionally, the bug fixes and improvements in this release demonstrate Kairos' commitment to providing a reliable and robust operating system for users. We invite you to download and try Kairos 1.6.0 and experience the benefits of AuroraBoot for yourself. - ---- - -For a full list of changes, see the [Changelog](https://github.com/kairos-io/kairos/releases/tag/v1.6.0). We hope you find these updates useful and as always, let us know if you have any questions or feedback. Thanks for using Kairos! \ No newline at end of file diff --git a/docs/content/en/blog/release-v2.0.md b/docs/content/en/blog/release-v2.0.md deleted file mode 100644 index d56be4c94..000000000 --- a/docs/content/en/blog/release-v2.0.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: "Kairos release v2.0" -date: 2023-04-13 -linkTitle: "Announcing v2.0 Kairos release" -description: "Introducing Kairos 2.0: long live UKI!" -author: Ettore Di Giacinto ([Twitter](https://twitter.com/mudler_it)) ([GitHub](https://github.com/mudler)) ---- -

-
- kairos-white-column 5bc2fe34 -
-
-

- -Kairos is a cloud-native meta-Linux distribution that brings the power of public cloud to your on-premises environment. With Kairos, you can build your own cloud with complete control and no vendor lock-in. It allows you to easily spin up a Kubernetes cluster with the Linux distribution of your choice, and manage the entire cluster lifecycle with Kubernetes. - -#### Why you should try Kairos: -Kairos provides a wide range of use cases, from Kubernetes applications to appliances and more. You can provision nodes with your own image or use Kairos releases for added flexibility. Kairos also simplifies day-2 operations like node upgrades. It provides the benefits of a unified, cloud-native approach to OS management. - -#### What you can do with Kairos: -With Kairos, you can create an immutable infrastructure that stays consistent and free of drift with atomic upgrades. You can manage your cluster's entire lifecycle with Kubernetes, from building to upgrading. Kairos also allows you to automatically create multi-node, single clusters that span across regions for maximum flexibility and scalability. - -## Kairos 2.0.0 release - -Kairos 2.0.0 has just been released, and we are thrilled to share the latest updates and improvements to the Kairos project. This release is a major release as reflect changes to internal core components. - -### What changed? - -We replaced the former dracut modules (a set of bash scripts/dracut/systemd services), which were responsible for the immutability management of Kairos, with https://github.com/kairos-io/immucore , a self-contained binary which doesn't have dependencies and can run without dracut and systemd. While changes shouldn't be impactful for most of our users, as changes impacted only in internal components, we suggest to try the upgrade in a lab environment before upgrading from earlier versions (v1.x). - -The 2.0 release allows us to: -- not depend anymore on systemd while set up immutability on boot ( allowing us to unblock several stories, for instance create Alpine images with vanilla kernels ) -- have hybrid images, that boots both [UKI](https://github.com/uapi-group/specifications/blob/main/specs/unified_kernel_image.md) as a single file image, and as well as pivoting (as we are doing currently) -- pave the way for things like SecureBoot, Static Measured boot and much more -- debug things more cleanly, have a better testbed, and allow to ease out maintenance of the codebase -- be a step closer to our Confidential computing roadmap, indeed now you can try out running [Confidential computing workload](https://kairos.io/docs/advanced/coco/). - -Besides, we have now full SBOM list attached to images, as part of the release process, and `in-toto` attestation, allowing [you to verify attestation also of SBOM lists](https://docs.sigstore.dev/cosign/attestation/), and have full audit of images. We also have integrated `grype` and `trivy` in our pipelines, and as such now releases contains also CVE reports, and finally we upload the generated reports as sarif file to GitHub to have notifications and see with more ease the impact of CVEs to the images. See also our [documentation](https://kairos.io/docs/upgrade/kubernetes/#verify-images-attestation-during-upgrades) on how to gate upgrades and allow only verified images to be used during the process. - -There were also fixes to the Debian flavor (thanks to the community for reporting issues!) and now manual upgrades with private registries are supported, too. - -Finally, it is also now possible to specify custom bind mounts path to overlay on top of the persistent partition, allowing to easily specify paths that you want to be persistent in the system via the cloud config file: https://kairos.io/docs/advanced/customizing/#customizing-the-file-system-hierarchy-using-custom-mounts . - -If you are curious on what's next, check out our [Roadmap](https://github.com/orgs/kairos-io/projects/2) and feel free to engage with our [community](https://kairos.io/community/)! - ---- - -For a full list of changes, see the [Changelog](https://github.com/kairos-io/kairos/releases/tag/v2.0.0). We hope you find these updates useful and as always, let us know if you have any questions or feedback. Thanks for using Kairos! diff --git a/docs/content/en/blog/sena-architecture.md b/docs/content/en/blog/sena-architecture.md deleted file mode 100644 index 888450dca..000000000 --- a/docs/content/en/blog/sena-architecture.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: "Kairos is now part of the Secure Edge-Native Architecture by Spectro Cloud and Intel" -date: 2023-04-18 -linkTitle: "The Secure Edge-Native Architecture" -description: "Learn about how Kairos is now part of SENA, the Secure Edge-Native Architecture announced by Spectro Cloud and developed in collaboration with Intel, enabling organizations to securely deploy, provision, operate and manage at scale edge locations. Discover the benefits of SENA and what's coming up in the future roadmap of Kairos' secure edge computing solutions." -author: Ettore Di Giacinto ([Twitter](https://twitter.com/mudler_it)) ([GitHub](https://github.com/mudler)) ---- - -The Kairos team is thrilled to announce the release of the Secure Edge-Native Architecture (SENA) whitepaper! You can download it [here](https://github.com/kairos-io/kairos/files/11250843/Secure-Edge-Native-Architecture-white-paper-20240417.3.pdf) - -## What is SENA? - -SENA stands for "Secure Edge-Native Architecture." It is a comprehensive solution architecture that outlines the tools and practices to address the modern requirements for deploying and managing Kubernetes-based edge applications at scale. SENA’s objective is to establish a new industry standard in the form of a well-defined framework that leverages best-in-class security and other concepts, design principles and tools, bringing together the most innovative hardware and software security capabilities. - -SENA covers considerations across the full lifecycle of edge hardware and software to enable teams to efficiently deploy, provision, operate and manage edge environments at scale. - -## Kairos and SENA - -Kairos is a core foundation of SENA, providing capabilities in combination with other components across the following areas: - -### When deploying hardware edge devices - -- Ease of deployment: Kairos enables zero-touch provisioning through our [Kubernetes Native API](https://kairos.io/docs/installation/automated/) and locally with [AuroraBoot](https://kairos.io/docs/reference/auroraboot/). -- Self-coordinated deployment: Enable self-coordinated, fully autonomous deployments with [Integrated Kairos P2P support](https://kairos.io/docs/installation/p2p/). -- Flexible deployments: Kairos can be fully customized to meet your Infrastructure needs. Extend [Kairos images easily](https://kairos.io/docs/advanced/customizing/), or [build your own using the Kairos framework](https://kairos.io/docs/reference/build-from-scratch/), even at scale [by leveraging the power of Kubernetes](https://kairos.io/docs/advanced/build/). - -### When provisioning the complete edge stack - -- Ensuring the provenance of the image attestation before deployments and during upgrades via the Kubernetes control plane with [kyverno](https://kyverno.io/docs/writing-policies/verify-images/). Instructions can be found [here](https://kairos.io/docs/upgrade/kubernetes/#verify-images-attestation-during-upgrades). -- Ensuring provenance of the artifacts and comply with SLSA: Kairos releases SBOM artifacts, and builds on Github Actions, allowing you to identify and track components included in the released images with [cosign](https://github.com/sigstore/cosign). - - -## When operating the edge application - -- Immutable, read-only OS stack: Kairos is a single [container image](https://kairos.io/docs/architecture/container/), [immutable system](https://kairos.io/docs/architecture/immutable/) which is read-only and cannot be modified during runtime. -- Ensuring the privacy of user data at rest and in use. You can [encrypt data at rest](https://kairos.io/docs/advanced/partition_encryption/#offline-mode) using the TPM chip and with the Kairos Key Management Server (KMS) 'kcrypt'.The KMS also accepts only hardware devices with a TPM chip, ensuring onboarding of trusted devices. -- Providing the ability for applications to execute in a Trusted Execution Environment (TEE) leveraging [Gramine](https://github.com/gramineproject/gramine). A TEE is an environment where hardware mechanisms are used to ensure the integrity and privacy of process execution, protecting against privileged (root) processes and physical snooping of electrical signals or devices in the system. You can already run workloads in a TEE with Kairos. For instructions check out [Confidential computing](https://kairos.io/docs/advanced/coco/) - -## What's next - -Here are some of the items in our roadmap: - -- Static and Dynamic measured boot: We are planning to have UKI-flavored variants to boot the full OS in a single file. This will enable measurement, signing, and verification, simplifying maintenance and management, and leading to true immutability with a reduced attack surface. -- Ensuring the provenance and integrity of the OS during boot and runtime. We plan to integrate measured boot and SecureBoot on top of UKI images, integrating with Keylime, enabling remote attestation of system integrity after boot -- Ensuring the provenance and integrity of the application stack in runtime. Integration with GSC, [MarbleRun](https://github.com/edgelesssys/marblerun) - to seamlessly run confidential applications in your Kubernetes cluster and running attestation of confidential workloads. -- Management of hardware at scale: OpenAMT - Offering ways to automatically register Kairos boxes to an OpenAMT-supported management platform. - -You can already benefit from the SENA Architecture today with Kairos and you can follow our roadmap to see what's coming up in the next releases [here](https://github.com/orgs/kairos-io/projects/2). - -Stay tuned! More to come! diff --git a/docs/content/en/blog/understanding-immutability.md b/docs/content/en/blog/understanding-immutability.md deleted file mode 100644 index c49edf590..000000000 --- a/docs/content/en/blog/understanding-immutability.md +++ /dev/null @@ -1,141 +0,0 @@ ---- -title: "Understanding Immutable Linux OS: Benefits, Architecture, and Challenges" -date: 2023-03-22 -linkTitle: "Understanding Immutable Linux OS: Benefits, Architecture, and Challenges" -description: "In this post we are trying to answer some of the typical questions that help understanding Immutable OSes principles and we will dive a bit in what solutions are out there, and what are the challenges in the field" -author: Ettore Di Giacinto ([Twitter](https://twitter.com/mudler_it)) ([GitHub](https://github.com/mudler)) ---- - -For years, the traditional Linux operating system has been a top pick for its flexibility and ability to be customized. But as great as it is, there are use cases in which stricter security rules and higher reliability standards are needed. That's where immutable Linux operating systems come in - offering a more secure and reliable option, especially in settings where security is paramount. - -{{< card header="![castle](https://user-images.githubusercontent.com/2420543/226939911-6801ecab-d023-4357-92f2-b782ae086462.png)" subtitle="_An illustration of a fortress surrounded by a moat and guarded by armored knights, with a banner flying the Linux penguin logo, medieval, fortified, secure, trending on Artstation._ Author: _Midjourney AI_" - >}} -{{< /card >}} - -In this post, we'll be addressing some common questions to help you understand the principles behind immutable operating systems. We'll also be exploring the various solutions available and the challenges faced in this field. So, get ready to dive in! - -## What is an Immutable Linux OS? - -Explaining the concept of an immutable Linux OS to a newcomer can often turn into a detailed discussion on system internals. However, we'll simplify it here as much as possible, even for those unfamiliar with the concepts. - -Formally defined, an immutable Linux OS (also known as Immutable Infrastructure or Immutable Deployment) is an operating system designed to be unchangeable and read-only. This means that once the operating system has been installed, the system files and directories cannot be modified. Any changes made to the system are temporary and lost when the system is rebooted. Think of it as a snapshot of a standard Linux system that cannot be changed. Any updates or changes are made by creating a new instance of the OS, deploying it, and switching over to the new instance. You can also find a very good writeup by Adrian Hornsby [here](https://medium.com/the-cloud-architect/immutable-infrastructure-21f6613e7a23). - -If you're already a Linux user, you'll know that as `root` (Administrator), you can write anywhere in the filesystem, potentially corrupting the OS portion responsible for booting or management. In an immutable OS, however, any command that attempts to modify the system files will fail, as those files are only accessible for reading. - -Immutable systems are particularly useful in environments where security is a top priority, such as cloud computing, embedded systems, kiosks, and container execution. Essentially, any environment that needs to scale can benefit from the security and reliability of an immutable OS. - -{{% alert color="info" %}} -_"But what does that really mean? And what problem are Immutable systems trying to solve?"_ -{{% /alert %}} - -There are several advantages to using immutable Linux systems over traditional Linux systems. Firstly, there is an additional layer of **security** as it's not possible to tamper with the runtime OS. Changes, if accepted, are discarded on the next reboot. This means that if a hacker or a malicious actor gains access to the system, they cannot make permanent changes that could compromise the system's security. - -Secondly, **maintenance** of immutable systems is easier because they do not require regular updates or patches at the atomic package level. Instead, the entire OS is updated, similar to how updates are handled on Android phones. - -Finally, because the system is read-only, it is more **reliable** and less prone to failure. A declarative configuration model is usually tied to it, simplifying the configuration of the OS when orchestrated with other tools such as Ansible, Terraform, or similar. - -{{% alert color="info" %}} -_"Right, but how do I manage upgrades?"_ -{{% /alert %}} - -Instead of upgrading the system in place, upgrades are typically handled by creating a new, updated image of the operating system and replacing the existing image, in an atomic operation. This process is commonly referred to as "image-based upgrade". The image can also be delivered to the end system, but this differs depending on the implementation, and there is no building on the node side. - -![Upgrade](https://user-images.githubusercontent.com/2420543/224147132-50d6808e-0a1c-48d0-8f44-627bd0dfa9f2.png) - -In contrast, standard Linux systems typically use package managers such as `apt` or `yum` to upgrade software packages in place. This can be a more complex process because the package manager must ensure that all dependencies are satisfied and that there are no conflicts between different software packages. Additionally, upgrades in standard Linux systems can sometimes cause issues if there are conflicts between different versions of software packages or if the upgrade process is interrupted. - -## Challenges at scale - -In standard Linux systems, the package manager has a lot of responsibilities and interacts directly with the system to apply changes. It can install packages, upgrade packages, merge configurations, and generate additional data required for the package. This makes installing software, upgrading, and running a system easy as a couple of interactions away with the package manager. - -When it comes to upgrading an installed system, the package manager should take care of many aspects, such as: correctly ordering dependencies (which may require a solver), verifying which packages are installed or not, which new packages will be installed, and handling file transmission securely. However, as the complexity of the stack grows, conflicts between packages can arise, and the package manager may prompt the user to solve them. This is not ideal for scaling out deployments, upgrades, and cutting operational costs since it exposes the infrastructure to drift. - -{{< card header="![Screenshot from 2023-03-09 18-25-17](https://user-images.githubusercontent.com/2420543/224106950-7d652652-c8e0-4ee4-980d-b057e4af903f.png)" - footer="">}} - _Huh, didn't we get rid of package conflicts already? ([screenshot](https://www.reddit.com/r/openSUSE/comments/z4ld75/this_seems_to_be_common_in_opensuse_should_i_wait/))_ -{{< /card >}} - -Tools like Ansible, Salt, Puppet, or Chef can manage and control standard systems upgrade mechanisms without requiring any interaction with each system during high-scale upgrades. In the standard model, clients handle certain portions of upgrades and installations, such as updating configuration files, or regenerating the initramfs. However, these actions could eventually raise the infrastructure drift level, causing a configuration merging to block everything or cause damage to your infrastructure and interrupt services. To avoid such issues, preparing fallback or switching services connections after an upgrade has been rolled out is one way to approach it. - -Transactional upgrades, are a step toward making standard mutable Linux systems, act more similarly to image-based upgrades in immutable Linux systems. In a transactional upgrade, the new software packages are prepared, usually into a separate partition, and applied after the first boot, similar to how an image-based upgrade works. However, unlike an immutable system, the existing system files can still be modified during the upgrade process. - -On the other hand, immutable OSes simplify managing the OS stack by not exposing the node to complexities during upgrades or installation. The image is built ahead of time, using a well-tested, reproducible recipe that does not modify the system itself. The package manager is responsible for preparing a new, pristine environment that the real system will boot into afterward. For instance, immutable Linux OSes that use A/B partitioning create a new image of the operating system with the updated software packages or configuration changes. The new image is deployed to a transitive partition, which then becomes the new active partition. If the upgrade fails, the system can simply boot on the passive partition. - -## Immutable OS: a look at the current landscape - -Here are some popular Immutable OS solutions, although this list is not exhaustive. There are much better and updated ones you can find [on Github](https://github.com/castrojo/awesome-immutable). Each of the solutions was created to tackle its own set of challenges, and they differ in their implementation details depending on their target environments. - -The following are some of the most popular Immutable OS solutions: - -- CoreOS: A Linux-based operating system designed for containers and cloud computing, which uses an immutable file system called "Container Linux". CoreOS has now merged with Red Hat Enterprise Linux. -- Project Atomic: A CentOS-based Linux distribution, that focuses on container deployment and management, using a layered approach that allows for easy rollbacks. -- Ubuntu Core: Ubuntu Core is a version of the Ubuntu operating system designed and [engineered for IoT and embedded systems](https://ubuntu.com/core/services/guide/intro-ubuntu-core). It uses snap packages exclusively to create a confined and transaction-based system. It also updates itself and its applications automatically. -- RancherOS: - A Linux-based operating system that is designed to be minimal, lightweight, and optimized for running containers. RancherOS uses Docker for all system processes, and its file system is mounted read-only, making it immutable. -- Talos: An open-source Linux distribution designed to run Kubernetes, K3s, or other container orchestration systems. It features a highly secure, API-managed infrastructure with automated and scalable operations and is suitable for cloud, containers, and general-purpose environments. -- K3OS (discontinued): A minimal Linux distribution designed specifically for running Kubernetes clusters. k3os is built around k3s, a lightweight Kubernetes distribution, and uses the immutable Container Linux file system with an A/B update model to ensure smooth and reliable updates. It is suitable for cloud and container environments. -- Flatcar Container Linux: A Linux-based operating system that is based on CoreOS and is designed for use in containerized environments. Like CoreOS, Flatcar Container Linux uses an immutable file system to provide stability and security. -- Fedora Silverblue: A Fedora-based Linux distribution, that uses an immutable file system and a transactional update model, to provide a stable and secure environment. Fedora Silverblue is designed for use in desktop and containerized environments. A nice overview can be found [here](ttps://www.lifeintech.com/2021/11/19/immutable-os/) or [here](https://www.redhat.com/sysadmin/immutability-silverblue) -- Photon OS: A Linux-based operating system developed by VMware, which is designed to run containerized workloads. Photon OS uses a minimal package set and an immutable file system for enhanced security and manageability. - -To simplify the comparison between the different Immutable OS solutions, the following table highlights their key differences and the environments they are targeted for: - -| Solution | Based on | Update Model | Target Environment | -|---|---|---|---| -| CoreOS | Gentoo | Transactional Updates | Cloud | -| Talos | Nothing | Container image update | Cloud, Containers, General purpose | -| K3OS | Alpine | A/B | Cloud, Containers | -| Project Atomic | CentOS | Layered Packages | Containers | -| Ubuntu Core | Ubuntu | Transactional Updates | IoT, Embedded Systems | -| RancherOS | Linux | Docker for System Processes | Containers | -| Flatcar Container Linux | CoreOS | Transactional Updates | Cloud | -| Red Hat Atomic Host | Red Hat | Transactional Updates | Cloud, optimized for running containers | -| Fedora Silverblue | Fedora | Transactional Updates | Desktop, Containers | -| Photon OS | Linux | Immutable File System | Cloud | -| Kairos | Any Linux distribution | Immutable File System | Cloud, Edge, General purpose | - -{{% alert color="info" %}} -_"So, what's Kairos? What's the challenges that Kairos tries to overcome?"_ -{{% /alert %}} - -## How Kairos fits in the ecosystem - -Kairos is a great fit when you want to deploy a Linux system on real hardware at the Edge[^1] or in a datacenter, whether it's in your cloud on-premises or in the Edge. Specifically, if you're looking for: - -- Zero-touch configuration and high-scalable deployments. [See how to perform automated installs](/docs/installation/automated/) or [how to create custom appliances](/docs/advanced/build/) -- A single distribution center of upgrades across your infrastructure using container registries. [See docs](/docs/architecture/container/#benefits) -- Strong security posture, including [online data encryption at-rest via TPM](/docs/advanced/partition_encryption/), Supply chain verification and Service bill of material -- Good hardware support -- Simplified Kubernetes deployment with [self-coordinated K3s](/docs/installation/p2p/) -- [Flexibility in customization](/docs/advanced/customizing/), including fine-grained control over the OS layer (packages installed, versions), and complete support maintenance level by [building images from scratch](/docs/reference/build-from-scratch) -- Complete control over your infrastructure -- A [community-driven](/community/), [open roadmap](https://github.com/orgs/kairos-io/projects/2), office hours, and the opportunity to get involved - -**Maintenance** - One thing you may have noticed when comparing Kairos to other alternatives, is that it doesn't tie you to a specific OS. Instead, Kairos is flexible and portable, supporting all the popular Linux distributions, such as Ubuntu, Debian, and Fedora, among others. This, unties you from typical vendor lock-in strategies, forcing you to choose a specific distribution only for the immutability aspect. - -The design shines also for its support for **long-term maintenance**. Each framework image released by Kairos allows the conversion of any OS to the given Kairos version, which could potentially enable maintenance for as long as the base OS support model allows. [You can learn more about it here](/docs/reference/build-from-scratch). - -**Container based** - Kairos treats every operating system (OS) as a set of packages and represents the OS with a standard container image that can be executed with tools such as `podman`, `docker`, and so on. [This container image](/docs/architecture/container/) includes all the necessary components for booting. Kairos components manage all node lifecycle operations, such as upgrading, installing, and resetting. These components are packaged within the [framework images](/docs/reference/image_matrix/#framework-images), which can be overlaid while creating a standard container image. Unlike traditional Linux distributions, the kairos-agent handles upgrades by pulling new container images as systems to boot, instead of relying on the OS package manager. - -All installation and upgrades are delivered exclusively through container images, which are overlaid at boot time, eliminating the need for a container engine at runtime. The container image used for booting includes the kernel, initrd, and all other required pieces. This allows for customization directly within a Dockerfile. The container being booted is the image itself, and there is no actual container runtime running the image. The container is used to construct an image internally, which is then used to boot the system in an A/B fashion, without adding any overhead. - -This approach offers several benefits, including the ability to verify the image with security scans and treat it similarly to a standard application that can be distributed via a container registry. - -**Separation of concerns** - The separation of concerns between the OS and the management interface is clear in Kairos. The OS is responsible for providing the booting components and packages necessary for its operation, while Kairos provides the framework for managing the node's lifecycle and immutability interface. The relationship between the image and Kairos is governed by a [contract](/docs/reference/build-from-scratch), which enables package handling without vendor lock-in. - -This separation of concerns simplifies the delegation of package maintenance, CVE monitoring, and security fixes to the OS layer. Upgrades to container images can be achieved by chaining Dockerfiles or manually committing changes to the image. - -**Automatic deployments** - To further [automate](/docs/installation/automated/) custom deployment models, the Kairos Kubernetes Native Extensions can be used to create customized configurations either directly from Kubernetes or via the command line interface (CLI). - -**Self co-ordinated**: [Configuring multiple nodes](/docs/installation/p2p/) at the Edge to form a single cluster can present challenges at various levels, from the network stack (such as assigning IPs to machines) to the configuration of the cluster topology (such as determining which machine will be the master). However, Kairos enables completely self-coordinated deployments, including for high availability (HA), eliminating the need for any configuration templating mechanism or specific role assignments for nodes. - -## Conclusion - -In conclusion, an immutable Linux OS, provides a more secure and reliable environment than a standard Linux system. However, it may not be suitable for all use cases, such as those that require frequent updates or modifications to the system. Upgrades in immutable systems are handled differently from standard Linux systems, using an image-based approach rather than package-based upgrades. While transactional upgrades in standard mutable Linux systems offer some benefits over traditional package-based upgrades, they still do not provide the same level of security and reliability as image-based upgrades in immutable Linux systems. Overall, the decision to use an immutable Linux system should be based on the specific requirements of the use case, and the benefits and limitations should be carefully considered, something that we can't just let ChatGPT decide :wink: - -Immutable Linux OSes offer a higher degree of reliability, security, and fault tolerance compared to traditional Linux systems. By using read-only file systems, separate update partitions and A/B partitioning, Immutable Linux OSes provide a safe, reliable way to update the system without downtime or the risk of breaking the system. Immutable Linux OSes are particularly well-suited for critical systems such as cloud container platforms, embedded systems, or IoT devices, where stability, security and scalability are of the utmost importance. - -## Footnotes - -[^1]: (Author note) As I dislike marketing buzzwords, I prefer to describe the Edge as the last-mile of computing. It involves a dedicated hardware that needs to be controlled by the Cloud in some way, such as a small server running Kubernetes, performing measurements and communicating with the Cloud. The term "Edge" is a broad, generic term that encompasses various computing scenarios, such as near-edge and far-edge computing, each with its own specialized deployment solution. - -To put it simply, Kairos can be deployed on bare-metal hardware, and it provides robust support for hardware deployment. \ No newline at end of file diff --git a/docs/content/en/community/_index.md b/docs/content/en/community/_index.md deleted file mode 100644 index 23f5f9eb0..000000000 --- a/docs/content/en/community/_index.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Community -menu: - main: - weight: 40 ---- diff --git a/docs/content/en/docs/Advanced/_index.md b/docs/content/en/docs/Advanced/_index.md deleted file mode 100644 index ab9036558..000000000 --- a/docs/content/en/docs/Advanced/_index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: "Advanced" -linkTitle: "Advanced" -weight: 5 -description: > - Advanced settings ---- - diff --git a/docs/content/en/docs/Advanced/after-install.md b/docs/content/en/docs/Advanced/after-install.md deleted file mode 100644 index 5d915c9f0..000000000 --- a/docs/content/en/docs/Advanced/after-install.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -title: "Pushing configuration to a node after installation" -linkTitle: "After install" -weight: 1 -description: > ---- - -Kairos configuration mechanism is based on the `cloud-config `file given during installation, however, it's possible to extend the configuration by providing additional cloud-configs in either `/oem` or `/usr/local/cloud-config`. - -By default, `kairos` reads in lexicographic order YAML cloud-config files in the directories above, indeed, after installation you should be able to see the configuration generated by the interactive-installer as `/oem/99_custom.yaml` in the system. - -This mechanism can be used to set and enable persistent configuration on boot after node deployment. - -We are going to see how to do that manually or with Kubernetes by using the - -## Manually - -SSH into the node and copy the config file you want to add into `/oem`. For instance, to add zram on boot we can copy the following file in `/oem/100_zram.yaml` or `/usr/local/cloud-config/100_zram.yaml` and reboot: - -```yaml -stages: - boot: - - name: "zram setup" - commands: - - modprobe zram - - echo lzo > /sys/block/zram0/comp_algorithm - - echo 1G > /sys/block/zram0/disksize - - mkswap --label zram0 /dev/zram0 - - swapon --priority 100 /dev/zram0 -name: "zfs setup" -``` - -## With Kubernetes - -To push configurations to a node, it is necessary [system-upgrade-controller](https://github.com/rancher/system-upgrade-controller) to be deployed in the target cluster which executes plan to the cluster nodes. In the example below, we use a plan to push a swapfile of 3gb enabled during boot, and restart the node afterward. - -To install [system-upgrade-controller](https://github.com/rancher/system-upgrade-controller), use kubectl: - -```bash -kubectl apply -f https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/system-upgrade-controller.yaml -``` - -{{% alert title="Note" %}} - -Several roll-out strategies can be used with `system-upgrade-controller` which are not illustrated here in this example. For instance, it can be specified in the number of hosts which are running the upgrades, filtering by labels, and more. [Refer to the project documentation](https://github.com/rancher/system-upgrade-controller) on how to create efficient strategies to roll plans on the nodes. In the example above, the plans are applied to every host of the cluster, one-by-one in sequence. - -{{% /alert %}} - -The following pushes a new cloud config over the `/oem` directory and reboots the node: - -```bash -cat <<'EOF' | kubectl apply -f - ---- -apiVersion: v1 -kind: Secret -metadata: - name: custom-script - namespace: system-upgrade -type: Opaque -stringData: - swapfile.yaml: | - stages: - boot: - - name: "Setup swapfile" - if: "[ ! -e /usr/local/swapfile ]" - commands: - - dd if=/dev/zero of=/usr/local/swapfile bs=1M count=3K - - mkswap /usr/local/swapfile - - name: "Enable swapfile" - if: "[ -e /usr/local/swapfile ]" - commands: - - swapon /usr/local/swapfile - add-oem-file.sh: | - #!/bin/sh - set -e - if diff /host/run/system-upgrade/secrets/custom-script/swapfile.yaml /host/oem/10_swapfile.yaml >/dev/null; then - echo Swapfile present - exit 0 - fi - # Note: this is a symlink. We can also cp -L, but be aware that standard cp doesn't work. - cat /host/run/system-upgrade/secrets/custom-script/swapfile.yaml > /host/oem/10_swapfile.yaml - sync - - mount --rbind /host/dev /dev - mount --rbind /host/run /run - nsenter -i -m -t 1 -- reboot - exit 1 ---- -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: add-swapfile - namespace: system-upgrade -spec: - concurrency: 1 - # This is the version (tag) of the image. - # The version is referred to the kairos version plus the k3s version. - version: "v1.0.0-rc2-k3sv1.23.9-k3s1" - nodeSelector: - matchExpressions: - - { key: kubernetes.io/hostname, operator: Exists } - serviceAccountName: system-upgrade - cordon: false - upgrade: - # Here goes the image which is tied to the flavor being used. - # Currently can pick between opensuse and alpine - image: quay.io/kairos/kairos-opensuse-leap - command: - - "/bin/bash" - - "-c" - args: - - bash /host/run/system-upgrade/secrets/custom-script/add-oem-file.sh - secrets: - - name: custom-script - path: /host/run/system-upgrade/secrets/custom-script -EOF -``` diff --git a/docs/content/en/docs/Advanced/build.md b/docs/content/en/docs/Advanced/build.md deleted file mode 100644 index 6bb049eb3..000000000 --- a/docs/content/en/docs/Advanced/build.md +++ /dev/null @@ -1,288 +0,0 @@ ---- -title: "Build Kairos appliances" -linkTitle: "Build" -weight: 5 -description: > ---- - -{{% alert title="Note" %}} - -This page is a work in progress! -The feature is experimental and API is likely going to be subject to changes, don't rely on it yet! -{{% /alert %}} - - -This documentation section describes how the Kairos Kubernetes Native API extensions can be used to build custom appliances or booting medium for Kairos. - -While it's possible to just run Kairos from the artifacts provided by our release process, there are specific use-cases which needs extended customization, for example when -additional kernel modules, or custom, user-defined logic that you might want to embed in the media used for installations. - -Note the same can be achieved by using advanced configuration and actually modify the images during installation phase by leveraging the `chroot` stages that takes place in the image - this is discouraged - as it goes in opposite with the "Single Image", "No infrastructure drift" approach of Kairos. The idea here is to create a system from "scratch" and apply that on the nodes - not to run any specific logic on the node itself. - -To achieve that, Kairos provides a set of Kubernetes Native Extensions that allow to programmatically generate Installable mediums, Cloud Images and Netboot artifacts. These provide on-demand customization and exploit Kubernetes patterns to automatically provision nodes using control-plane management clusters - however, the same toolset can be used to build appliances for local development and debugging. - -The [automated](/docs/installation/automated) section already shows some examples of how to leverage the Kubernetes Native Extensions and use the Kairos images to build appliances, in this section we will cover and describe in detail how to leverage the CRDs and the Kairos factory to build custom appliances. - -## Prerequisites - -When building locally, only `docker` is required to be installed on the system. To build with the Kubernetes Native extensions, a Kubernetes cluster is required and `helm` and `kubectl` installed locally. Note [kind](https://github.com/kubernetes-sigs/kind) can be used as well. The Native extensions don't require any special permission, and run completely unprivileged. - -### Kubernetes - -To build with Kubernetes we need to install the Kairos `osbuilder` controller. - -The chart depends on cert-manager. You can install the latest version of cert-manager by running the following commands: - -```bash -kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml -kubectl wait --for=condition=Available deployment --timeout=2m -n cert-manager --all -``` - -Install the Kubernetes charts with `helm`: - -```bash -helm repo add kairos https://kairos-io.github.io/helm-charts -helm repo update -helm install kairos-crd kairos/kairos-crds -helm install kairos-osbuilder kairos/osbuilder -``` - -Among the things deployed by the helm chart, is also an nginx server which is used to -serve the artifact files after they are built. See below for more. - -## Build an ISO - -To build an ISO, consider the following spec, which provides a hybrid bootable ISO (UEFI/MBR), with the `core` kairos image, adding `helm`: - -```yaml -kind: OSArtifact -apiVersion: build.kairos.io/v1alpha1 -metadata: - name: hello-kairos -spec: - imageName: "quay.io/kairos/core-opensuse-leap:latest" - iso: true - bundles: - # Bundles available at: https://packages.kairos.io/Kairos/ - - quay.io/kairos/packages:helm-utils-3.10.1 - cloudConfig: | - #cloud-config - users: - - name: "kairos" - passwd: "kairos" - install: - device: "auto" - reboot: true - poweroff: false - auto: true # Required, for automated installations -``` - -Apply the manifest with `kubectl apply`. - -Note, the CRD allows to specify a custom Cloud config file, [check out the full configuration reference](/docs/reference/configuration). - -As mentioned above, there is an nginx server that will serve the built artifacts as soon as they are ready. -By default, it is exposed with a `NodePort` type of service. Use the following commands -to get its URL: - -The controller will create a pod that builds the ISO ( we can follow the process by tailing to the containers log ) and later makes it accessible to its own dedicated service (nodeport by default): - -```bash -$ PORT=$(kubectl get svc osartifactbuilder-operator-osbuilder-nginx -o json | jq '.spec.ports[0].nodePort') -$ curl http://:$PORT/hello-kairos.iso -o output.iso -``` -## Netboot artifacts - -It is possible to use the CRD to prepare artifacts required for netbooting, by enabling `netboot: true` for instance: - -```yaml -kind: OSArtifact -metadata: - name: hello-kairos -spec: - imageName: "quay.io/kairos/core-opensuse-leap:latest" - netboot: true - netbootURL: ... - bundles: ... - cloudConfig: ... -``` - -## Build a Cloud Image - -Cloud images are images that automatically boots into recovery mode and can be used to deploy whatever image you want to the VM. -Custom user-data from the Cloud provider is automatically retrieved, additionally the CRD allows to embed a custom cloudConfig so that we can use to make configuration permanent also for VM images running outside a cloud provider. - -A Cloud Image boots in QEMU and also in AWS, consider: - -```yaml -apiVersion: build.kairos.io/v1alpha1 -kind: OSArtifact -metadata: - name: hello-kairos -spec: - imageName: "quay.io/kairos/core-opensuse-leap:latest" - cloudImage: true - cloudConfig: | - #cloud-config - users: - - name: "kairos" - passwd: "kairos" - name: "Default deployment" - stages: - boot: - - name: "Repart image" - layout: - device: - label: COS_RECOVERY - add_partitions: - - fsLabel: COS_STATE - size: 16240 # At least 16gb - pLabel: state - - name: "Repart image" - layout: - device: - label: COS_RECOVERY - add_partitions: - - fsLabel: COS_PERSISTENT - pLabel: persistent - size: 0 # all space - - if: '[ -f "/run/cos/recovery_mode" ] && [ ! -e /usr/local/.deployed ]' - name: "Deploy cos-system" - commands: - - | - # Use `elemental reset --system.uri docker:` to deploy a custom image - elemental reset && \ - touch /usr/local/.deployed && \ - reboot -``` - -Note: Since the image come with only the `recovery` system populated, we need to apply a cloud-config similar to this one which tells which container image we want to deploy. -The first steps when the machine boots into is to actually create the partitions needed to boot the active and the passive images, and its populated during the first boot. - -After applying the spec, the controller will create a Kubernetes Job which runs the build process and -then copy the produced `hello-kairos.raw` file to the nginx server (see above). This file is an EFI bootable raw disk, bootable in QEMU and compatible with AWS which automatically provisions the node: - -```bash -$ PORT=$(kubectl get svc osartifactbuilder-operator-osbuilder-nginx -o json | jq '.spec.ports[0].nodePort') -$ curl http://:$PORT/hello-kairos.raw -o output.raw -``` - -Note, in order to use the image with QEMU, we need to resize the disk at least to 32GB, this can be done with the CRD by setting `diskSize: 32000` or by truncating the file after downloading: - -```bash -truncate -s "+$((32000*1024*1024))" hello-kairos.raw -``` - -This is not required if running the image in the Cloud as providers usually resize the disk during import or creation of new instances. - -To run the image locally with QEMU we need `qemu` installed in the system, and we need to be able to run VMs with EFI, for example: - -```bash -qemu-system-x86_64 -m 2048 -bios /usr/share/qemu/ovmf-x86_64.bin -drive if=virtio,media=disk,file=output.raw -``` - -### Use the Image in AWS - -To consume the image, copy it into an s3 bucket: - -```bash -aws s3 cp s3:// -``` - -Create a `container.json` file referring to it: - -```json -{ -"Description": "Kairos custom image", -"Format": "raw", -"UserBucket": { - "S3Bucket": "", - "S3Key": "" -} -} -``` - -Import the image: - -```bash -aws ec2 import-snapshot --description "Kairos custom image" --disk-container file://container.json -``` - -Follow the procedure described in [AWS docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/creating-an-ami-ebs.html#creating-launching-ami-from-snapshot) to register an AMI from snapshot. Use all default settings except for the firmware, set to force to UEFI boot. - -## Build a Cloud Image for Azure - -Similarly we can build images for Azure, consider: - -```yaml -apiVersion: build.kairos.io/v1alpha1 -kind: OSArtifact -metadata: - name: hello-kairos -spec: - imageName: "quay.io/kairos/core-opensuse-leap:latest" - azureImage: true - ... -``` - -Will generate a compressed disk `hello-kairos-azure.vhd` ready to be used in GCE. - -```bash -$ PORT=$(kubectl get svc osartifactbuilder-operator-osbuilder-nginx -o json | jq '.spec.ports[0].nodePort') -$ curl http://:$PORT/hello-kairos-azure.vhd -o output.vhd -``` - -### How to use the image in Azure - -Upload the Azure Cloud VHD disk in `.vhda` format to your bucket: - -```bash -az storage copy --source --destination https://.blob.core.windows.net// -``` - -Import the disk: - -```bash -az image create --resource-group --source https://.blob.core.windows.net// --os-type linux --hyper-v-generation v2 --name -``` - -Note: There is currently no way of altering the boot disk of an Azure VM via GUI, use the `az` to launch the VM with an expanded OS disk if needed - -## Build a Cloud Image for GCE - - -Similarly we can build images for GCE, consider: - -```yaml -apiVersion: build.kairos.io/v1alpha1 -kind: OSArtifact -metadata: - name: hello-kairos -spec: - imageName: "quay.io/kairos/core-opensuse-leap:latest" - gceImage: true - ... -``` - -Will generate a compressed disk `hello-kairos.gce.raw.tar.gz` ready to be used in GCE. - -```bash -$ PORT=$(kubectl get svc osartifactbuilder-operator-osbuilder-nginx -o json | jq '.spec.ports[0].nodePort') -$ curl http://:$PORT/hello-kairos.gce.raw.tar.gz -o output.gce.raw.tar.gz -``` - -### How to use the image in GCE - -To upload the image in GCE (compressed): - -```bash -gsutil cp gs:/// -``` - -Import the disk: - -```bash -gcloud compute images create --source-uri=/ --guest-os-features=UEFI_COMPATIBLE -``` - -See [here how to use a cloud-init with Google cloud](https://cloud.google.com/container-optimized-os/docs/how-to/create-configure-instance#using_cloud-init_with_the_cloud_config_format). diff --git a/docs/content/en/docs/Advanced/bundles.md b/docs/content/en/docs/Advanced/bundles.md deleted file mode 100644 index bf1ecd542..000000000 --- a/docs/content/en/docs/Advanced/bundles.md +++ /dev/null @@ -1,151 +0,0 @@ ---- -title: "Bundles" -linkTitle: "Bundles" -weight: 5 -description: > - Bundles are a powerful feature of Kairos that allow you to customize and configure your operating system. This section explains how to use and build custom bundles. - ---- - -Bundles are a powerful feature of Kairos that allow you to customize and configure your operating system, as well as your Kubernetes cluster. Whether you want to add custom logic, install additional packages, or make any other changes to your system, bundles make it easy to apply these changes after installation or before bootstrapping a node. - -Bundles are container images containing only files (and not full OS) that can be used to install new software or extend the cloud-init syntax. You can find community-supported bundles in the [community-bundles](https://github.com/kairos-io/community-bundles) repository. - -## Consuming Bundles - -To use a bundle in your Kairos configuration, you will need to specify the type of bundle and the target image in your cloud-config file. - -To apply a bundle before Kubernetes starts, you can include it in your config like this: - -```yaml -#cloud-config - -bundles: -- targets: - - run:// -``` - -Replace `` with the URL or path to the bundle image. The prefix (e.g. `run://`) indicates the type of bundle being used. - -To install a bundle after installation instead (for those bundles that explicitly supports that), use the following: - -```yaml -#cloud-config -install: - bundles: - - targets: - - run:// -``` - -One of the benefits of using bundles is that they can also extend the cloud-config keywords available during installation. This means that by adding bundles to your configuration file, you can add new blocks of configuration options and customize your system even further. - -A full config using a bundle from [community-bundles](https://github.com/kairos-io/community-bundles) that configures `metalLB` might look like this: - -```yaml -#cloud-config - -hostname: kairoslab-{{ trunc 4 .MachineID }} -users: -- name: kairos - ssh_authorized_keys: - # Add your github user here! - - github:mudler - -k3s: - enable: true - args: - - --disable=servicelb - -# Specify the bundle to use -bundles: -- targets: - - run://quay.io/kairos/community-bundles:metallb_latest - -# Specify metallb settings -metallb: - version: 0.13.7 - address_pool: 192.168.1.10-192.168.1.20 -``` - -## Bundle types - -Bundles can carry also binaries that can be overlayed in the rootfs, either while [building images](/docs/advanced/build) or with [Live layering](https://kairos.io/docs/advanced/livelayering/). - -Kairos supports three types of bundles: - -- **Container**: This type is a bare container that simply contains files that need to be copied to the system. It is useful for copying over configuration files, scripts, or any other static content that you want to include on your system (prefixed with `container:` or `docker:`). - -- **Run**: This type is also a bare container, but it comes with a script that can be run during the installation phase to add custom logic. This is useful for performing any necessary setup or configuration tasks that need to be done before the cluster is fully deployed (prefixed with `run:`). - -- **Package**: This type is a [luet](https://luet.io) package that will be installed in the system. It requires you to specify a `luet` repository in order to work. Luet packages are a powerful way to manage dependencies and install software on your system (prefixed with `luet:`). - - -{{% alert title="Note" %}} -In the future, Kairos will also support a local type for use in airgap situations, where you can pre-add bundles to the image before deployment. -{{% /alert %}} - -It's important to note that bundles do not have any special meaning in terms of immutability. They install files over paths that are mutable in the system, as they are simply overlaid during the boot process. This means that you can use bundles to make changes to your system at any time, even after it has been deployed. - -## Create bundles - -To build your own bundle, you will need to create a Dockerfile and any necessary files and scripts. A bundle is simply a container image that includes all the necessary assets to perform a specific task. - -To create a bundle, you will need to define a base image and copy over any necessary files and scripts to the image. For example, you might use the following Dockerfile to create a bundle image that deploys everything inside `assets` in the Kubernetes cluster: - -```Dockerfile -FROM alpine -COPY ./run.sh / -COPY ./assets /assets -``` - -And the associated `run.sh` that installs the assets depending on a cloud-config keyword can be: - -```bash -#!/bin/bash - -K3S_MANIFEST_DIR="/var/lib/rancher/k3s/server/manifests/" - -mkdir -p $K3S_MANIFEST_DIR - -# IF the user sets `example.enable` in the input cloud config, we install our assets -if [ "$(kairos-agent config get example.enable | tr -d '\n')" == "true" ]; then - cp -rf assets/* $K3S_MANIFEST_DIR -fi -``` - -This Dockerfile creates an image based on the Alpine base image, and copies over a script file and some assets to the image. -You can then add any additional instructions to the Dockerfile to install necessary packages, set environment variables, or perform any other tasks required by your bundle. - -Once you have created your Dockerfile and any necessary script files, you can build your bundle image by running docker build and specifying the path to your Dockerfile. - -For example: - -```bash -docker build -t . -``` - -This command will build an image with the name you specify ( replace `` accordingly ) based on the instructions in your Dockerfile. - -After building your bundle image, you will need to push it to a registry so that it can be accessed by Kairos. You can use a public registry like Docker Hub. To push your image to a registry, use the docker push command. For example: - -```bash -docker push -``` - -This will push the `` to your specified registry. - -And use it with Kairos: - -```yaml -#cloud-config - -bundles: -- targets: - # e.g. run://quay.io/...:tag - - run:// - -example: - enable: true -``` - -See the [community-bundles repository](https://github.com/kairos-io/community-bundles) for further examples. \ No newline at end of file diff --git a/docs/content/en/docs/Advanced/coco.md b/docs/content/en/docs/Advanced/coco.md deleted file mode 100644 index 87198690e..000000000 --- a/docs/content/en/docs/Advanced/coco.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -title: "Confidential computing setup" -linkTitle: "Confidential Computing" -weight: 9 -description: > ---- - -{{% alert title="Note" %}} -This page describes features that are still experimental in Kairos. There are a lot of things that can be improved and might be more streamlined in the future. -{{% /alert %}} - -Confidential computing is a type of secure computing that allows users to encrypt and decrypt data on a secure, isolated computing environment. -It works by encrypting the data before it is sent to the cloud or other computing resources. This allows users to keep their data private and secure, even if it is accessed by unauthorized parties. -This makes it useful for sensitive data such as financial information, health records, and other confidential data. - -One important aspect of Confidential Computing is the ability to encrypt data even in-memory. This document describes how to setup Kairos to use [`enclave-cc`](https://github.com/confidential-containers/enclave-cc) -in order to run confidential workloads. - -## Create a Kairos cluster - -The [`coco community bundle`](https://github.com/kairos-io/community-bundles/tree/main/coco) is supported since Kairos version `v2.0.0-alpha3` ("coco" stands for "**Co**nfidential **Co**mputing"). - -A configuration file like the following should be used (see the `bundles` section): - -``` -#cloud-config -bundles: - - targets: - - run://quay.io/kairos/community-bundles:system-upgrade-controller_latest - - run://quay.io/kairos/community-bundles:cert-manager_latest - - run://quay.io/kairos/community-bundles:kairos_latest - - run://quay.io/kairos/community-bundles:coco_latest - -install: - auto: true - device: auto - reboot: true - -k3s: - enabled: true - -users: - - name: kairos - passwd: kairos -``` - -The bundle is making some changes on the host's filesystem (installs a customized containerd binary among other things) and a restart of the node is needed in order for the changes to be applied fully. -When this file appears, reboot the node: `/etc/containerd/.sentinel`. - -## Additional steps - -- [Label our node](https://github.com/confidential-containers/documentation/blob/main/quickstart.md#prerequisites): - -``` - kubectl label --overwrite node $(kubectl get nodes -o jsonpath='{.items[].metadata.name}') node-role.kubernetes.io/worker="" -``` - -- [Deploy the operator](https://github.com/confidential-containers/documentation/blob/main/quickstart.md#deploy-the-the-operator) - -``` - kubectl apply -k github.com/confidential-containers/operator/config/release?ref=v0.4.0 -``` - -- [Deploy the `ccruntime` resource] - -``` - kubectl apply -k github.com/confidential-containers/operator/config/samples/ccruntime/ssh-demo?ref=v0.4.0 -``` - - (wait until they are all running: `kubectl get pods -n confidential-containers-system --watch`) - -- [Deploy a workload](https://github.com/confidential-containers/documentation/blob/main/quickstart.md#test-creating-a-workload-from-the-sample-encrypted-image) - - The last part with the verification will only work from within a Pod because the IP address is internal: - - `ssh -i ccv0-ssh root@$(kubectl get service ccv0-ssh -o jsonpath="{.spec.clusterIP}")` - - You can create a Pod like this: - - ``` - apiVersion: v1 - kind: Pod - metadata: - name: kubectl - spec: - containers: - - name: kubectl - image: opensuse/leap - command: ["/bin/sh", "-ec", "trap : TERM INT; sleep infinity & wait"] - ``` - - Get a shell to it and run the verification commands (You will need to install `ssh` in the Pod first). - -## Known limitations - -The above solution has some known limitations that might be addressed in future releases of Kairos. Namely: - -- After a Kairos upgrade, the above process has to be repeated in order to install the customized `containerd` and the relevant configuration. -- There is no simple way to upgrade to newer versions of the bundle ([this is a general bundles limitation](https://github.com/kairos-io/kairos/issues/974)). diff --git a/docs/content/en/docs/Advanced/customizing.md b/docs/content/en/docs/Advanced/customizing.md deleted file mode 100644 index cbbcea64a..000000000 --- a/docs/content/en/docs/Advanced/customizing.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -title: "Customizing the system image" -linkTitle: "Customization" -weight: 2 -description: > ---- - -Kairos is an open source, container-based operating system. To modify Kairos and add a package, you'll need to build a container image from the [Kairos images](/docs/reference/image_matrix/). Here's an example with Docker which adds `figlet`: - -```docker -# Use images from docs/reference/image_matrix/ -FROM quay.io/kairos/kairos:opensuse-latest - -RUN zypper in -y figlet - -RUN export VERSION="my-version" -RUN envsubst '${VERSION}' /myos:0.1 . -Sending build context to Docker daemon 2.048kB -Step 1/3 : FROM quay.io/kairos/kairos-opensuse:latest - ---> 897dc0cddf91 -Step 2/3 : RUN zypper install -y figlet - ---> Using cache - ---> d57ff48546e7 -Step 3/3 : RUN MY_VERSION="my-version" >> /etc/os-release - ---> Running in b7bcb24969f5 -Removing intermediate container b7bcb24969f5 - ---> ca21930a4585 -Successfully built ca21930a4585 -Successfully tagged /myos:0.1 -``` - -Once you have built your image, you can publish it to Docker Hub or another registry with the following command: - -```bash -$ docker push /myos:0.1 -The push refers to repository [docker.io//myos] -c58930881bc4: Pushed -7111ee985500: Pushed -... -``` - -You can use your custom image when [upgrading nodes manually](/docs/upgrade/manual), [with Kubernetes](/docs/upgrade/kubernetes) or [specifying it in the cloud-config during installation](/docs/examples/core). Here's how to do it manually with the `kairos-agent` command: - -``` -node:/home/kairos # kairos-agent upgrade --image docker.io//myos:0.1 -INFO[2022-12-01T13:49:41Z] Starting elemental version v0.0.1 -INFO[2022-12-01T13:49:42Z] Upgrade called -INFO[2022-12-01T13:49:42Z] Applying 'before-upgrade' hook -INFO[2022-12-01T13:49:42Z] Running before-upgrade hook -INFO[2022-12-01T13:49:42Z] deploying image docker.io/oz123/myos:0.1 to /run/initramfs/cos-state/cOS/transition.img -INFO[2022-12-01T13:49:42Z] Creating file system image /run/initramfs/cos-state/cOS/transition.img -INFO[2022-12-01T13:49:42Z] Copying docker.io/oz123/myos:0.1 source... -INFO[0000] Unpacking a container image: docker.io/oz123/myos:0.1 -INFO[0000] Pulling an image from remote repository -... -INFO[2022-12-01T13:52:33Z] Finished moving /run/initramfs/cos-state/cOS/transition.img to /run/initramfs/cos-state/cOS/active.img -INFO[2022-12-01T13:52:33Z] Upgrade completed -INFO[2022-12-01T13:52:33Z] Upgrade completed - -node:/home/kairos # which figlet -which: no figlet in (/sbin:/usr/sbin:/usr/local/sbin:/root/bin:/usr/local/bin:/usr/bin:/bin) -node:/home/kairos # reboot - -``` - -Now, reboot your OS and ssh again to it to use figlet: - -``` -$ ssh -l kairos node: -Welcome to Kairos! - -Refer to https://kairos.io for documentation. -kairos@node2:~> figlet kairos rocks! - _ _ _ _ -| | ____ _(_)_ __ ___ ___ _ __ ___ ___| | _____| | -| |/ / _` | | '__/ _ \/ __| | '__/ _ \ / __| |/ / __| | -| < (_| | | | | (_) \__ \ | | | (_) | (__| <\__ \_| -|_|\_\__,_|_|_| \___/|___/ |_| \___/ \___|_|\_\___(_) -``` - -## Customizing the Kernel - -Kairos allows you to customize the kernel and initrd as part of your container-based operating system. If you are using a glibc-based distribution, such as OpenSUSE or Ubuntu, you can use the distribution's package manager to replace the kernel with the one you want, and then rebuild the initramfs with `dracut`. - -Here's an example of how to do this: - -```bash -# Replace the existing kernel with a new one, depending on the base image it can differ -apt-get install -y ... - -# Create the kernel symlink -kernel=$(ls /boot/vmlinuz-* | head -n1) -ln -sf "${kernel#/boot/}" /boot/vmlinuz - -# Regenerate the initrd, in openSUSE we could just use "mkinitrd" -kernel=$(ls /lib/modules | head -n1) -dracut -v -f "/boot/initrd-${kernel}" "${kernel}" -ln -sf "initrd-${kernel}" /boot/initrd - -# Update the module dependencies -kernel=$(ls /lib/modules | head -n1) -depmod -a "${kernel}" -``` - -{{% alert title="Note" %}} - -If you are using an Alpine-based distribution, modifying the kernel is only possible by rebuilding the kernel and initrd outside of the Dockerfile and then embedding it into the image. This is because dracut and systemd are not supported in musl-based distributions. We are currently exploring ways to provide initramfs that can be generated from musl systems as well. - -{{% /alert %}} - -After you have modified the kernel and initrd, you can use the kairos-agent upgrade command to update your nodes, or [within Kubernetes](/docs/upgrade/kubernetes). - - -## Customizing the file system hierarchy using custom mounts. - - -### Bind mounts - -For clusters that needs to mount network block storage you might want to add -custom mount point that bind mounted to your system. For example, when using -Ceph file system, the OS mounts drives to `/var/lib/ceph` (for example). - -To achieve this you need to add the key `bind_mounts` to the `install` section -you pass the install, and specify a list of one or more bind mounts path. - -``` -install: - auto: true - device: "auto" - # changes persist reboot - mount as BIND - bind_mounts: - - /var/lib/ceph -... -``` - - -### Ephemeral mounts - -One can also specifying custom mounts which are ephemeral. These are writable, -however changes are discarded at boot (like `/etc/` already does). -``` -install: - auto: true - device: "auto" - # changes persist reboot - mount as BIND - bind_mounts: - - /var/lib/ceph - ephemeral_mounts: - - /opt/scratch/ -... -``` -Note, that these paths should exist in the container file-system used to create the ISO. -See [ISO customization](/docs/Advanced/customizing/) above. diff --git a/docs/content/en/docs/Advanced/livelayering.md b/docs/content/en/docs/Advanced/livelayering.md deleted file mode 100644 index 1f038ecf1..000000000 --- a/docs/content/en/docs/Advanced/livelayering.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -title: "Live layering" -linkTitle: "Live layering" -weight: 4 -description: > ---- - -Live layering allows to activates/deactivates system extension images. System extension images may – dynamically at runtime — extend the `/usr/` and `/opt/` directory hierarchies with additional files. - -Kairos supports live layering with `systemd-sysext`. Currently it is supported only on the `openSUSE`, `Fedora` and `Ubuntu` flavors with `systemd-sysext`. - -## Description - -For general reference on how `systemd-sysext` works, please read the [official](https://www.freedesktop.org/software/systemd/man/systemd-sysext.html) documentation. - -Systemd system extensions can be located in the directories `/etc/extensions/`, `/run/extensions/`, `/var/lib/extensions/`, `/usr/lib/extensions/` and `/usr/local/lib/extensions/`. - -In order to install extensions in runtime, they need to be placed into `/usr/local/lib/extensions` which is mounted over the `COS_PERSISTENT` partition. The other paths are reserved for the system image, which could ship extension directly from the container image used for upgrade or deployment. - -## Installing extensions - -In order to install extensions, you can just place them into `/usr/local/lib/extensions`. - -For example, on a running Kairos node to install an extension from a container image: - -```bash -luet util unpack /usr/local/lib/extensions/ -``` - -To load an extension during installation of a Kairos node, it can be supplied as a bundle in the `install` block in the node configuration: - -```yaml -#cloud-config - -# Set username and password -stages: - initramfs: - - name: "Set user and password" - users: - kairos: - passwd: "kairos" - hostname: kairos-{{ trunc 4 .Random }} -# Install configuration block -install: - auto: true - reboot: true - device: auto - # Bundles to install - bundles: - - rootfs_path: /usr/local/lib/extensions/ - targets: - - container:// -``` - -## Building extensions - -Systemd extensions can be images, directory or files, quoting the systemd-sysext documentation: - -- Plain directories or btrfs subvolumes containing the OS tree - -- Disk images with a GPT disk label, following the Discoverable Partitions Specification - -- Disk images lacking a partition table, with a naked Linux file system (e.g. squashfs or ext4) - -All of those can be shipped as a container image and loaded as a bundle. - -For example, a bundle can be defined as a naked container image containing only the files that we want to overlay in the system. - -Consider the following Dockerfile to create an extension which adds `/usr/bin/ipfs` to the system: - - -{{% alert title="Note" %}} -Note that systemd extensions require an extension-release file, which can be used to validate different aspects of the system being run. - -If you don't want to limit to a single OS, you can use the special key `_any` but keep in mind that this is only available in systemd versions 252+. -On the other hand if you do want to have a validation, or if you're running an older version of systemd, you will need to set at least the `ID` and the `VERSION_ID` of the OS. -These need to match with the values in the `/etc/os-release` file. - -Read more here about systemd-sysext [here](https://www.freedesktop.org/software/systemd/man/systemd-sysext.html) -{{% /alert %}} - -```docker -FROM alpine as build - -# Install a binary -RUN wget https://github.com/ipfs/kubo/releases/download/v0.15.0/kubo_v0.15.0_linux-amd64.tar.gz -O kubo.tar.gz -RUN tar xvf kubo.tar.gz -RUN mv kubo/ipfs /usr/bin/ipfs -RUN mkdir -p /usr/lib/extension-release.d/ -RUN echo ID=_any > /usr/lib/extension-release.d/extension-release.kubo - -FROM scratch - -COPY --from=build /usr/bin/ipfs /usr/bin/ipfs -COPY --from=build /usr/lib/extension-release.d /usr/lib/extension-release.d -``` diff --git a/docs/content/en/docs/Advanced/networking.md b/docs/content/en/docs/Advanced/networking.md deleted file mode 100644 index 1269b19cb..000000000 --- a/docs/content/en/docs/Advanced/networking.md +++ /dev/null @@ -1,164 +0,0 @@ ---- -title: "Networking" -linkTitle: "Networking" -weight: 3 -description: > ---- - -By default, Kairos ISOs are configured to automatically get an IP from the network interface. However, depending on the base system you have chosen, there are different way to configure networking. This section collects information on setting network configuration depending on the base that is being chosen (openSUSE, Alpine, Ubuntu). - -There are different network managers depending on the distro: - -- `connman` is available on Alpine-based distribution. By default is enabled on Kairos Alpine flavored variants. -- systemd-based flavors are all using `systemd-networkd` - -## Static IP - -To get a static IP, you can additionally define the following in your configuration file, depending on the network-manager being used: - -{{< tabpane text=true right=true >}} -{{% tab header="connman" %}} -```yaml -stages: - initramfs: - - files: - - path: /var/lib/connman/default.config - permission: 0644 - content: | - [service_eth0] - Type = ethernet - IPv4 = 10.1.1.1/16/10.1.0.1 - Nameservers = 10.1.0.1 -``` -{{% /tab %}} -{{% tab header="systemd-networkd" %}} -```yaml -stages: - initramfs: - - files: - - path: /etc/systemd/network/01-man.network - permissions: 0644 - content: | - [Match] - Name=ens18 - - [Network] - Address=10.1.1.1/16 - Gateway=10.1.0.1 - DNS=10.1.0.1 -``` -{{% /tab %}} -{{< /tabpane >}} - -## Bonding - -Bonding setup with Ubuntu can be configured via systemd-networkd (Ubuntu based images) and wicked (openSUSE based images), consider the following examples: - -{{< tabpane text=true right=true >}} -{{% tab header="systemd-networkd" %}} -```yaml -#cloud-config -name: "My Deployment" -stages: - boot: - - name: "Setup network" - commands: - - systemctl restart systemd-networkd - initramfs: - # Drop network config file - - name: "Setup hostname" - hostname: "hostname" - - name: "Setup network files" - files: - - path: /etc/systemd/network/10-bond0.network - content: | - [Match] - Name=bond0 - [Network] - DHCP=yes - permissions: 0644 - owner: 0 - group: 0 - - path: /etc/systemd/network/10-bond0.netdev - content: | - [NetDev] - Name=bond0 - Kind=bond - [Bond] - Mode=802.3ad - permissions: 0644 - owner: 0 - group: 0 - - path: /etc/systemd/network/15-enp.network - content: | - [Match] - Name=enp* - [Network] - Bond=bond0 - permissions: 0644 - owner: 0 - group: 0 - - path: /etc/systemd/network/05-bond0.link - content: | - [Match] - Driver=bonding - Name=bond0 - [Link] - MACAddress=11:22:33:44:55:66 - permissions: 0644 - owner: 0 - group: 0 - network: - - name: "Setup user ssh-keys" - authorized_keys: - kairos: - - "ssh-rsa AAA..." - - "ssh-rsa AAA..." -# k3s settings -k3s-agent: - enabled: true - env: - K3S_TOKEN: "KubeSecret" - K3S_URL: https://hostname:6443 -``` -{{% /tab %}} -{{% tab header="connman" %}} -```yaml -stages: - boot: - - name: "Setup network" - commands: - - modprobe bonding mode=4 miimon=100 - - ifenslave bond0 eno1 - - ifenslave bond0 eno2 - - ifenslave bond0 eno3 - - ifenslave bond0 eno4 - - ifconfig bond0 up hw ether 11:22:33:44:55:66 - - ifup bond0 - - sleep 5 - - rc-service connman restart - initramfs: - - name: "Setup network files" - files: - - path: /var/lib/connman/default.config - content: | - [service_eth] - Type = ethernet - IPv4 = off - IPv6 = off - - [service_bond0] - Type = ethernet - DeviceName = bond0 - IPv4 = dhcp - MAC = 11:22:33:44:55:66 - permissions: 0644 - owner: 0 - group: 0 -``` -{{% /tab %}} -{{< /tabpane >}} - -### References - -- https://kerlilow.me/blog/setting-up-systemd-networkd-with-bonding/ diff --git a/docs/content/en/docs/Advanced/partition_encryption.md b/docs/content/en/docs/Advanced/partition_encryption.md deleted file mode 100644 index 24815b2b1..000000000 --- a/docs/content/en/docs/Advanced/partition_encryption.md +++ /dev/null @@ -1,301 +0,0 @@ ---- -title: "Encrypting User Data with Kairos" -linkTitle: "Encrypting User Data with Kairos" -weight: 5 -description: > - This section describes how to encrypt partition with LUKS in Kairos. ---- - -{{% alert title="Note" color="warning" %}} - -This feature will be available in Kairos version `1.5.0` and in all future releases. - -{{% /alert %}} - -Kairos offers the ability to encrypt user data partitions with `LUKS`. User-data partitions are dedicated to persist data for a running system, stored separately from the OS images. This encryption mechanism can also be used to encrypt additional partitions created during the installation process. - -Kairos supports the following encryption scenarios: - -1. **Offline mode** - Encryption key for partitions is stored on the machine inside the TPM chip. -1. **Online mode (Automated)** - Keypair used to encrypt the partition passphrase is stored on the TPM chip, and an external server is used to store the encrypted passphrases. -1. **Online mode (Manually configured)** - Plaintext passphrase is stored in the KMS server and returned to the node after TPM challenging. - -![encryption1_1674470732563_0](https://user-images.githubusercontent.com/2420543/214405291-97a30f2d-d70a-45ba-b842-5282c722c79e.png) - -Kairos uses the TPM chip to encrypt partition passphrases, and for offline encryption, it stores the passphrase in the non-volatile registries of the chip. -To enable encryption, you will need to specify the labels of the partitions you want to encrypt, a minimum configuration for offline encryption can be seen below: - -```yaml -#cloud-config - -install: - # Label of partitions to encrypt - # COS_PERSISTENT is the OS partition - # dedicated to user-persistent data. - encrypted_partitions: - - COS_PERSISTENT -``` - -Please note that for online mode, you will also need to specify the key management server address that will be used to store the keys, a complete configuration reference is the following: - -```yaml -#cloud-config - -# Install block -install: - # Label of partitions to encrypt - # COS_PERSISTENT is the OS partition - # dedicated to user-persistent data. - encrypted_partitions: - - COS_PERSISTENT - -# Kcrypt configuration block -kcrypt: - challenger: - # External KMS Server address. This must be reachable by the node - challenger_server: "http://192.168.68.109:30000" - # (optional) Custom Non-Volatile index to use to store encoded blobs - nv_index: "" - # (optional) Custom Index for the RSA Key pair - c_index: "" - # (optional) Custom TPM device - tpm_device: "" -``` - -| Option | Description | -| --- | --- | -| `install.encrypted_partitions` | Label of partitions to encrypt | -| `kcrypt.challenger.challenger_server` | External KMS Server address | -| `kcrypt.challenger.nv_index` | Custom Non-Volatile index to use to store encoded blobs | -| `kcrypt.challenger.c_index` | Custom Index for the RSA Key pair | -| `kcrypt.challenger.tpm_device` | Custom TPM device | - -## Requirements - -The host machine must have a TPM chip version 2.0 or higher to use encryption with Kairos. A list of TPM chips/HW can be found [in the list of certified products](https://trustedcomputinggroup.org/membership/certification/tpm-certified-products/), however, any modern machine has a TPM 2.0 chip. - -## Components - -The Kairos encryption design involves three components to manage partitions encryption and decryption lifecycle: - -- [kcrypt](https://github.com/kairos-io/kcrypt) runs on the machine and attempts to unlock partitions by using plugins to delegate encryption/decryption business logic. -- [kcrypt-discovery-challenger](https://github.com/kairos-io/kcrypt-challenger) runs on the machine, it is called by `kcrypt` and uses the TPM chip to retrieve the passphrase as described below. -- [kcrypt-challenger](https://github.com/kairos-io/kcrypt-challenger) is the KMS (Key Management Server) component, deployed in Kubernetes, which manages secrets and partitions of the nodes. - -## Offline mode - -This scenario covers encryption of data at rest without any third party or KMS server. The keys used to encrypt the partitions are stored in the TPM chip. - -### Scenario: Offline encryption - -A high level overview of the interaction between the components can be observed here: - -![offline](https://user-images.githubusercontent.com/2420543/214795800-f7d54309-2a3c-4d29-b6da-c74644424244.png) - -A complete cloud config example for this scenario can be: - -```yaml -#cloud-config - -install: - encrypted_partitions: - - COS_PERSISTENT - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - ssh_authorized_keys: - # Replace with your github user and un-comment the line below: - # - github:mudler -``` - -Note, we define a list of partition labels that we want to encrypt. In the example above we set `COS_PERSISTENT` to be encrypted, which in turns will encrypt all the user-data of the machine (this includes, for instance, Kubernetes pulled images, or any runtime persisting data on the machine). - -## Online mode - -Online mode involves an external service (the Key Management Server, **KMS**) to boot the machines. The KMS role is to enable machine to boot by providing the encrypted secrets, or passphrases to unlock the encrypted drive. Authentication with the KMS is done via TPM challenging. - -In this scenario, we need to first deploy the KMS server to an existing Kubernetes cluster, and associate the TPM hash of the nodes that we want to manage. During deployment, we specify the KMS server inside the cloud-config of the nodes to be provisioned. - -### Requirements - -- A Kubernetes cluster -- Kcrypt-challenger reachable by the nodes attempting to boot - -### Install the KMS (`kcrypt-challenger`) - -To install the KMS (`kcrypt-challenger`), you will first need to make sure that certificate manager is installed. You can do this by running the following command: - -``` -kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml -kubectl wait --for=condition=Available deployment --timeout=2m -n cert-manager --all -``` - -To install `kcrypt-challenger` on a Kubernetes cluster with `helm`, you can use the commands below: - -``` -# Install the helm repository -helm repo add kairos https://kairos-io.github.io/helm-charts -helm repo update - -# Install the Kairos CRDs -helm install kairos-crd kairos/kairos-crds - -# Deploy the KMS challenger -helm install kairos-challenger kairos/kairos-challenger --set service.challenger.type="NodePort" - -# we can also set up a specific port and a version: -# helm install kairos-challenger kairos/kairos-challenger --set image.tag="v0.2.2" --set service.challenger.type="NodePort" --set service.challenger.nodePort=30000 -``` - -A service must be used to expose the challenger. If using the node port, we can retrieve the address with: - -```bash -export EXTERNAL_IP=$(kubectl get nodes -o jsonpath='{.items[].status.addresses[?(@.type == "ExternalIP")].address}') -export PORT=$(kubectl get svc kairos-challenger-escrow-service -o json | jq '.spec.ports[0].nodePort') -``` - -### Register a node - -In order to register a node on the KMS, the TPM hash of the node needs to be retrieved first. -You can get a node TPM hash by running `/system/discovery/kcrypt-discovery-challenger` as root from the LiveCD: - -``` -kairos@localhost:~> ID=$(sudo /system/discovery/kcrypt-discovery-challenger) -kairos@localhost:~> echo $ID -7441c78f1976fb23e6a5c68f0be35be8375b135dcb36fb03cecc60f39c7660bd -``` - -This is the hash you should use in the definition of the `SealedVolume` in the -examples below. - -### Scenario: Automatically generated keys - -![encryption3_1674472162848_0](https://user-images.githubusercontent.com/2420543/214405310-78f7deec-b43e-4581-a99b-a358492cc7ac.png) - -The TPM chip generates unique RSA keys for each machine during installation, which are used to encrypt a generated secret. These keys can only be accessed by the TPM and not by the KMS, thus ensuring that both the KMS and the TPM chip are required to boot the machine. As a result, even if the machine or its disks are stolen, the drive remains sealed and encrypted. -Deployment using this method, will store the encrypted key used to boot into the KMS, and the keypair used to encrypt it in the TPM chip of the machine during installation. This means that, only the TPM chip can decode the passphrase, and the passphrase is stored in the KMS such as it can't be decrypted by it. As such, nodes can boot only with the KMS, and the disk can be decrypted only by the node. - -To register a node to kubernetes, use the TPM hash retrieved before (see section ["Register a node"](#register-a-node)) -and replace it in this example command: - -```bash -cat < ---- - diff --git a/docs/content/en/docs/Architecture/cloud-init.md b/docs/content/en/docs/Architecture/cloud-init.md deleted file mode 100644 index e57f3ae08..000000000 --- a/docs/content/en/docs/Architecture/cloud-init.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -title: "Cloud init based" -linkTitle: "Cloud init based" -weight: 3 -date: 2022-11-13 -description: > ---- - -Kairos supports the [standard cloud-init syntax](https://github.com/mudler/yip#compatibility-with-cloud-init-format) and [its own extended syntax](https://github.com/mudler/yip) to allow to configure a system declaratively with a cloud-config centric approach. - -If you are not familiar with the concepts of cloud-init, [official cloud-init](https://cloud-init.io/) is a recommended read. - -## Configuration persistency - -Kairos is an Immutable OS and the only configuration that is persistent across reboots is the cloud-init configuration. -Multiple cloud-init files can be present in the system and Kairos will read them and process them in sequence (lexicographic order) allowing to extend the configuration with additional pieces also after deployment, or to manage logical configuration pieces separately. - -In Kairos the `/oem` directory keeps track of all the configuration of the system and stores the configuration files. Multiple files are allowed and they are all executed during the various system stages. `/usr/local/cloud-config` can be optionally used as well to store cloud config files in the persistent partition instead. `/system/oem` is instead reserved to default cloud-init files that are shipped by the base OS image. - -By using the standard cloud-config syntax, a subset of the functionalities are available and the settings will be executed in the boot stage. - -## Boot stages - -During boot the stages are emitted in an event-based pattern until a system completes its boot process - -![Kairos-boot-events](https://user-images.githubusercontent.com/2420543/195111193-3167eab8-8058-4676-a1a0-f64aea745646.png) - -The events can be used in the cloud-config extended syntax to hook into the various stages, which can allow to hook inside the different stages of a node lifecycle. - -For instance, to execute something before reset is sufficient to add the following to the config file used to bootstrap a node: - -```yaml -name: "Run something before reset" -stages: - before-reset: - - name: "Setting" - commands: - - | - echo "Run a command before reset the node!" - -``` - -Below there is a detailed list of the stages available that can be used in the cloud-init configuration files: - -| **Stage** | **Description** | -|------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| _rootfs_ | This is the earliest stage, running before switching root, just right after the root is mounted in /sysroot and before applying the immutable rootfs configuration. This stage is executed over initrd root, no chroot is applied. | -| _initramfs_ | This is still an early stage, running before switching root. Here you can apply radical changes to the booting setup of Elemental. Despite this is executed before switching root this exection runs chrooted into the target root after the immutable rootfs is set up and ready. | -| _boot_ | This stage is executed after initramfs has switched root, during the systemd bootup process. | -| _fs_ | This stage is executed when fs is mounted and is guaranteed to have access to the state and persistent partitions ( `COS_STATE` and `COS_PERSISTENT` respectively). | -| _network_ | This stage is executed when network is available | -| _reconcile_ | This stage is executed 5m after boot and periodically each 60m. | -| _after-install_ | This stage is executed after installation of the OS has ended | -| _after-install-chroot_ | This stage is executed after installation of the OS has ended. | -| _after-upgrade_ | This stage is executed after upgrade of the OS has ended. | -| _after-upgrade-chroot_ | This stage is executed after upgrade of the OS has ended (chroot call). | -| _after-reset_ | This stage is executed after reset of the OS has ended. | -| _after-reset-chroot_ | This stage is executed after reset of the OS has ended (chroot call). | -| _before-install_ | This stage is executed before installation | -| _before-upgrade_ | This stage is executed before upgrade | -| _before-reset_ | This stage is executed before reset | - -Note: Steps executed at the `chroot` stage are running inside the new OS as chroot, allowing to write persisting changes to the image, for example by downloading and installing additional software. - - -### Sentinels - -When a Kairos boots it creates sentinel files in order to allow to execute cloud-init steps programmaticaly. - -- /run/cos/recovery_mode is being created when booting from the recovery partition -- /run/cos/live_mode is created when booting from the LiveCD - -To execute a block using the sentinel files you can specify: `if: '[ -f "/run/cos/..." ]'`, for instance: \ No newline at end of file diff --git a/docs/content/en/docs/Architecture/container.md b/docs/content/en/docs/Architecture/container.md deleted file mode 100644 index f98b838c4..000000000 --- a/docs/content/en/docs/Architecture/container.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -title: "Container based" -linkTitle: "Container" -weight: 2 -date: 2022-11-13 -description: > ---- - -Kairos is a container-based operating system (OS). - -A container-based operating system is an OS that is shipped via containers. Indeed, if it happens to be based on Linux (most probably), you can run the container image as well on your Docker daemon. The image being booted is the container, which contains all the required pieces in order to boot (Kernel, Initrd, Init system). There is no real container runtime running the image. The container is used to construct an image internally that is then used to boot the system in an A/B fashion, so there is no overhead introduced. The system being booted is actually a snapshot of the container. - -- **Single-image** The OS is a single container image which contains all the OS components, including Kernel and Initrd. -- **Tamper-proof upgrades** Upgrades are atomic, A/B swaps with fallback mechanisms and automatic boot assessment. -- **Distributed via container registries** Bootable images are standard OCI artifacts that can be hosted in any container regist -- **Platform Engineer-friendly** Adapt the infrastructure to your needs by plugging images into your already-existing workflow pipeline. Customizing an immutable OS becomes as easy as writing a Dockerfile. - -## A/B Upgrades - -![upgrade](https://user-images.githubusercontent.com/2420543/197806999-587632a1-0292-44df-bb8f-176ff702f62d.png) - -Upgrades are atomic operations that can be triggered manually or via Kubernetes. The node will create a transition image that will be swapped for the Active system, and the Active system becomes Passive. This ensures tamper-proof upgrades and automated fallback and boot assessment strategies are in place to automatically boot from the fallback system. The recovery image can be furthermore exploited to completely automatize node recovery. - -## Benefits - -- Container registries are already widely supported and used by anyone. -- Reduce infrastructure drift, by pushing upgrades as single images, with atomic upgrades. - -If you are operating a Kubernetes cluster and deploying applications on top, chances are that you already have a container registry deployed somewhere and configured to store them or manage your infrastructure stack. By using container images, you can reuse the same infrastructure to propagate upgrades to the nodes and handle customizations. - -![kairos-factory](https://user-images.githubusercontent.com/2420543/197808767-e213709d-af21-4e32-9a78-818f34170077.png) - -Container images can be extended after a build by using standard container building practices and seamlessly plug into your existing pipelines. Kairos allows to seamlessly upgrade to container images that are derived from other versions. - -We believe that bringing rollbacks, or incremental patches upgrades increases the exposure to infrastructure drift. In opposition, immutable, single images are deployed to the nodes as they were apps - no more discrepancies in your nodes - no need of configuration management tools like Chef, Ansible, or alikes. - -This means that to customize a Kairos version, all that is required is to build a standard container image with a plain Dockerfile—plus, the bits that are actually needed - we can't touch a system as we are typically used to. - -If you are familiar with Dockerfiles, then you are good to go to roll your own custom OS version to provision in the nodes. That removes any friction to questions like, "How do I add this package to my nodes?", or more complex ones as, "How can I replace with my own Kernel?". - -## Container Image based OS - -The Image support matrix in [here](/docs/reference/image_matrix) lists all the container images built from our CI on every release of Kairos. - -To inspect an image and run it locally, you can use a container engine like Docker or Podman: - -``` -$ docker pull {{< registryURL >}}/core-{{< flavor >}}:{{< kairosVersion >}} -``` - -We can run it locally with docker as a container to inspect it, as it is runnable: - -``` -$ docker run -ti --rm {{< registryURL >}}/core-{{< flavor >}}:{{< kairosVersion >}} -/ # cat /etc/os-release -... -KAIROS_NAME="kairos-core-{{< flavor >}}" -KAIROS_VERSION="{{< kairosVersion >}}" -KAIROS_ID="kairos" -KAIROS_ID_LIKE="kairos-core-{{< flavor >}}" -KAIROS_VERSION_ID="{{< kairosVersion >}}" -KAIROS_PRETTY_NAME="kairos-core-{{< flavor >}} {{< kairosVersion >}}" -KAIROS_BUG_REPORT_URL="https://github.com/kairos-io/kairos/issues" -KAIROS_HOME_URL="https://github.com/kairos-io/kairos" -KAIROS_IMAGE_REPO="{{< registryURL >}}/core-{{< flavor >}}" -KAIROS_IMAGE_LABEL="latest" -KAIROS_GITHUB_REPO="kairos-io/kairos" -KAIROS_VARIANT="core" -KAIROS_FLAVOR="{{< flavor >}}" -``` - -And check out things like what's the kernel inside: - -```bash -/ $ ls -liah /boot/ -total 102M -6692018 drwxr-xr-x 2 root root 4.0K Apr 16 2020 . -6817515 drwxr-xr-x 1 root root 4.0K Oct 10 16:11 .. -6692019 -rw-r--r-- 1 root root 65 Apr 16 2020 .vmlinuz-5.14.21-150400.24.21-default.hmac -6692020 -rw-r--r-- 1 root root 4.9M Apr 16 2020 System.map-5.14.21-150400.24.21-default -6692021 -rw-r--r-- 1 root root 1.7K Apr 16 2020 boot.readme -6692022 -rw-r--r-- 1 root root 245K Apr 16 2020 config-5.14.21-150400.24.21-default -6692023 lrwxrwxrwx 1 root root 35 Apr 16 2020 initrd -> initrd-5.14.21-150400.24.21-default -6692024 -rw------- 1 root root 69M Apr 16 2020 initrd-5.14.21-150400.24.21-default -6692025 -rw-r--r-- 1 root root 443K Apr 16 2020 symvers-5.14.21-150400.24.21-default.gz -6692026 -rw-r--r-- 1 root root 484 Apr 16 2020 sysctl.conf-5.14.21-150400.24.21-default -6692027 -rw-r--r-- 1 root root 17M Apr 16 2020 vmlinux-5.14.21-150400.24.21-default.gz -6692028 lrwxrwxrwx 1 root root 36 Apr 16 2020 vmlinuz -> vmlinuz-5.14.21-150400.24.21-default -6692029 -rw-r--r-- 1 root root 11M Apr 16 2020 vmlinuz-5.14.21-150400.24.21-default -``` - -The CI process generates bootable medium by the container images, and similarly, we can modify this image to introduce our changes and remaster an ISO as described in [Automated installation](/docs/installation/automated), but that can be resumed in the following steps: - -```bash -$ docker run -ti --name custom-container {{< registryURL >}}/core-{{< flavor >}}:{{< kairosVersion >}} -# # Do your changes inside the container.. -# echo "foo" > /foo -# ... -# exit -$ docker commit custom-container custom-image - > sha256:37176f104a870480f9c3c318ab51f6c456571b6612b6a47b96af71b95a0a27c7 -# Builds an ISO from it -$ docker run -v $PWD:/cOS -v /var/run/docker.sock:/var/run/docker.sock -i --rm quay.io/kairos/osbuilder-tools:v0.1.1 --name "custom-iso" --debug build-iso --date=false --local custom-image --output /cOS/ - > ... - > ... - > xorriso : UPDATE : Writing: 147456s 84.0% fifo 100% buf 50% 60.5xD - > ISO image produced: 175441 sectors - > Written to medium : 175472 sectors at LBA 48 - > Writing to '/cOS/custom-iso.iso' completed successfully. -$ ls -custom-iso.iso custom-iso.iso.sha256 -``` - -In order to go further and upgrade nodes using this image, now the only requirement is to push it in a container registry and upgrade the nodes using that container image. - -For upgrading to a container image see [manual upgrades](/docs/upgrade/manual) and [kubernetes upgrades](/docs/upgrade/kubernetes). - -## See also - -- [ISO remastering](/docs/installation/automated#iso-remastering) diff --git a/docs/content/en/docs/Architecture/immutable.md b/docs/content/en/docs/Architecture/immutable.md deleted file mode 100644 index 98e24147d..000000000 --- a/docs/content/en/docs/Architecture/immutable.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: "Immutable" -linkTitle: "Immutable" -weight: 1 -date: 2022-11-13 -description: > ---- - -Kairos adopts an immutable layout and derivatives created with its toolkit, inherit the same immutability attributes. - -An immutable OS is a carefully engineered system which boots in a restricted, permissionless mode, where certain paths of the system are not writable. For instance, after installation it's not possible to add additional packages to the system, and any configuration change is discarded after reboot. - -A running Linux-based OS system will have the following paths: - -``` -/usr/local - persistent ( partition label COS_PERSISTENT) -/oem - persistent ( partition label COS_OEM) -/etc - ephemeral -/usr - read only -/ immutable -``` - -`/usr/local` will contain all the persistent data which will be carried over in-between upgrades, unlike the changes made to `/etc` which will be discarded. - -## Benefits of using an Immutable System - -There are many reasons why you would like to use an immutable system, in this article we'll present two of them. - -1. From a security standpoint, it's far more secure than traditional systems. This is because most attack vectors rely on writing on the system, or installing persistent tools after a vector has been exploited. -2. From a maintenance perspective, configuration management tools like Chef, Puppet, or the likes aren't needed because immutable systems only have one configuration entry point. Every other configuration is cleaned up automatically after a reboot. - -The benefit of rolling out the same system over a set of machines are the following: - -- **No snowflakes** - All the machines are based on the same image, configuration settings and behavior. This allows to have a predictable infrastructure, predictable upgrades, and homogeneous configurations across your cluster. -- **Configuration is driven via cloud-init** - There is only one source of truth for the configuration, and that happens at bootstrap time. Anything else is handled afterwards—natively via Kubernetes, so no configuration management software is required. -- **Reduced attack surface** - Immutable systems cannot be modified or tampered at runtime. This enhances the security of a running OS, as changes on the system are not allowed. - -Tools like Chef, Puppet, and Ansible share the same underlying issues when it comes to configuration management. That is, nodes can have different version matrices of software and OS, which makes your set of nodes inhomogeneous and difficult to maintain and orchestrate from day 1 to day 2. - -Kairos tackles the issue from different angle, as can turn _any_ distribution to an "immutable" system, distributed as a standard container image, which gets provisioned to the devices as declared. This allows to treat OSes with the same repeatable portability as containers for apps, removing snowflakes in your cluster. Container registries can be used either internally or externally to the cluster to propagate upgrades with customized versions of the OS (kernel, packages, and so on). - -## Design - -Kairos after installation will create the following partitions: - -- A state partition that stores the container images, which are going to be booted (active and passive, stored in `.img` format which are loopback mounted) -- A recovery partition that stores the container images, used for recovery (in `.squashfs` format) -- An OEM partition (optional) that stores user configuration and cloud-config files -- A persistent partition to keep the data across reboots - -![Kairos-installation-partitioning](https://user-images.githubusercontent.com/2420543/195111190-3bdfb917-312a-40f4-b0bc-4a65a701c06b.png) - -The persistent partition is mounted during boot on `/usr/local`, and additional paths are mount-bind to it. Those configuration aspects are defined in a [cloud-config](https://github.com/kairos-io/kairos/blob/a1a9bef4dff30e0718fa4d2697f075ce37c7ed90/overlay/files/system/oem/11_persistency.yaml#L11) file. It is possible to override such configuration, via a custom cloud-config, during installation. - -The Recovery system allows to perform emergency tasks, in case of failure from the active and passive images. Furthermore a fallback mechanism will take place, so that in case of failures the booting sequence will be as follows: “A -> B -> Recovery”. - -The upgrade happens in a transition image and takes place only after all the necessary steps are completed. An upgrade of the ‘A/B’ partitions can be done [with Kubernetes](/upgrade/kubernetes) or [manually](/upgrade/manual). The upgrade will create a new pristine image, that will be selected as active for the next reboot, the old one will be flagged as passive. If we are performing the same from the passive system, only the active is subject to changes. - -### Kernel and Initrd - -The Kernel and Initrd are loaded from the system images and are expected to be present in the container, that is pulled down and used for upgrades. Differently from standard approaches, Kairos focuses on having static Initrds, which are generated while building images used for upgrades - in opposite of generating Initramfs locally on the node. A typical setup has kernels and initrd in a special boot partition dedicated for boot files - in Kairos instead the Kernel and Initrd are being loaded from the images, which are chainloaded from the bootloader (GRUB). This is a design choice to keep the entire OS stack confined as a single layer which gets pulled and swapped atomically during upgrades. diff --git a/docs/content/en/docs/Architecture/meta.md b/docs/content/en/docs/Architecture/meta.md deleted file mode 100644 index 0b15e122c..000000000 --- a/docs/content/en/docs/Architecture/meta.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: "Meta-Distribution" -linkTitle: "Meta-Distribution" -weight: 4 -date: 2022-11-13 -description: > ---- - -We like to define Kairos as a meta-Linux Distribution, as its goal is to convert other distros to an immutable layout with Kubernetes Native components. - -## Kairos - -The Kairos stack is composed of the following: - -- A core OS image release for each flavor in ISO, qcow2, and other similar formats (see [the list of supported distributions](/docs/reference/image_matrix)) provided for user convenience -- A release with K3s embedded. -- A set of Kubernetes Native API components (CRDs) to install into the control-plane node, to manage deployment, artifacts creation, and lifecycle (WIP). -- A set of Kubernetes Native API components (CRDs) to install into the target nodes to manage and control the node after deployment (WIP). -- An agent installed into the nodes to be compliant with Kubernetes Native API components mentioned above. - -Every component is extensible and modular such as it can be customized and replaced in the stack and built off either locally or with Kubernetes. - -### Internal components - -Kairos encompasses several components, external and internal. - -Internal: -- [kairos](https://github.com/kairos-io/kairos) is the main repository, building the `kairos-agent` and containing the image definitions which runs on our CI pipelines. -- [immucore](https://github.com/kairos-io/immucore) is the immutability management interface. -- [AuroraBoot](https://github.com/kairos-io/AuroraBoot) is the Kairos Node bootstrapper -- [elemental-cli](https://github.com/kairos-io/elemental-cli) manages the installation, reset, and upgrade of the Kairos node. -- [system packages](https://github.com/kairos-io/packages) contains additional packages, cross-distro, partly used in framework images -- [kcrypt](https://github.com/kairos-io/kcrypt) is the component responsible for encryption and decryption of data at rest -- [kcrypt-challenger](https://github.com/kairos-io/kcrypt-challenger) is the `kairos` plugin that works with the TPM chip to unlock LUKS partitions -- [osbuilder](https://github.com/kairos-io/osbuilder) is used to build bootable artifacts from container images -- [entangle](https://github.com/kairos-io/entangle) a CRD to interconnect Kubernetes clusters -- [entangle-proxy](https://github.com/kairos-io/entangle-proxy) a CRD to control interconnetted clusters - -External: -- [K3s](https://k3s.io) as a Kubernetes distribution -- [edgevpn](https://mudler.github.io/edgevpn) (optional) as fabric for the distributed network, node coordination and bootstrap. Provides also embedded DNS capabilities for the cluster. Internally uses [libp2p](https://github.com/libp2p/go-libp2p) for the P2P mesh capabilities. -- [nohang](https://github.com/hakavlad/nohang) A sophisticated low memory handler for Linux. diff --git a/docs/content/en/docs/Architecture/network.md b/docs/content/en/docs/Architecture/network.md deleted file mode 100644 index a58bb2849..000000000 --- a/docs/content/en/docs/Architecture/network.md +++ /dev/null @@ -1,167 +0,0 @@ ---- -title: "P2P Network" -linkTitle: "P2P Network" -weight: 5 -date: 2023-02-15 -description: > - How Kairos leverage Peer-to-peer (P2P) to self-coordinate clusters at the edge. ---- - -## Introduction - -As more organizations seek to take advantage of the benefits of Kubernetes for their edge applications, the difficulties of managing large-scale clusters become apparent. Managing, configuring, and coordinating multiple clusters can be a complex and time-consuming process. We need solutions that offer zero-touch configuration and self-coordination. - -To address these challenges, Kairos provides an easy and robust solution for deploying Kubernetes workloads at the edge. By utilizing peer-to-peer (p2p) technology, Kairos can automatically coordinate and create Kubernetes clusters without requiring a control management interface. This frees users up to concentrate on running and scaling their applications instead of spending time on cluster management. - -In this document, we will examine the advantages of using Kairos to deploy Kubernetes clusters at the edge, and how p2p technology facilitates self-coordination for a zero-touch configuration experience. We will also explore how Kairos' highly adaptable and container-based approach, combined with an immutable OS and meta-distribution, makes it an excellent choice for edge deployments. - -{{% alert title="Note" %}} -You can also watch our [Kairos and libp2p video]({{< ref "docs/media/#how-kairos-uses-libp2p" >}} "Media") in the [Media Section]({{< ref "docs/media" >}} "Media") -{{% /alert %}} - -## Overview: P2P for self-coordination - - - -Kairos creates self-coordinated, fully meshed clusters at the edge by using a combination of P2P technology, VPN, and Kubernetes. - -This design is made up of several components: - -- The Kairos base OS with support for different distribution flavors and k3s combinations (see our support matrix [here](/docs/reference/image_matrix)). -- A Virtual private network interface ([EdgeVPN](https://github.com/mudler/edgevpn) which leverages [libp2p](https://github.com/libp2p/go-libp2p)). -- K3s/CNI configured to work with the VPN interface. -- A shared ledger accessible to all nodes in the p2p private network. - -By using libp2p as the transport layer, Kairos can abstract connections between the nodes and use it as a coordination mechanism. The shared ledger serves as a cache to store additional data, such as node tokens to join nodes to the cluster or the cluster topology, and is accessible to all nodes in the P2P private network. The VPN interface is automatically configured and self-coordinated, requiring zero-configuration and no user intervention. - -Moreover, any application at the OS level can use P2P functionalities by using Virtual IPs within the VPN. The user only needs to provide a generated shared token containing OTP seeds for rendezvous points used during connection bootstrapping between the peers. It's worth noting that the VPN is optional, and the shared ledger can be used to coordinate and set up other forms of networking between the cluster nodes, such as KubeVIP. (See this example [here](/docs/examples/multi-node-p2p-ha-kubevip)) - -## Implementation - -Peer-to-peer (P2P) networking is used to coordinate and bootstrap nodes. When this functionality is enabled, there is a distributed ledger accessible over the nodes that can be programmatically accessed and used to store metadata. - -Kairos can automatically set up a VPN between nodes using a shared secret. This enables the nodes to automatically coordinate, discover, configure, and establish a network overlay spanning across multiple regions. [EdgeVPN](https://github.com/mudler/edgevpn) is used for this purpose. - -The private network is bootstrapped in three phases, with discovery driven by a distributed hash table (DHT) and multicast DNS (mDNS), which can be selectively disabled or enabled. The three phases are: - -1. Discovery -1. Gossip network -1. Full connectivity - -During the discovery phase, which can occur via mDNS (for LAN) or DHT (for WAN), nodes discover each other by broadcasting their presence to the network. - -In the second phase, rendezvous points are rotated by OTP (one-time password). A shared token containing OTP seeds is used to generate these rendezvous points, which serve as a secure way to bootstrap connections between nodes. This is essential for establishing a secure and self-coordinated P2P network. - -In the third phase, a gossip network is formed among nodes, which shares shared ledger blocks symmetrically encrypted with AES. The key used to encrypt these blocks is rotated via OTP. This ensures that the shared ledger is secure and that each node has access to the most up-to-date version of the shared configuration. The ledger is used to store arbitrary metadata from the nodes of the network. On each update, a new block is created with the new information and propagated via gossip. - -Optionally, full connectivity can be established by bringing up a TUN interface, which routes packets via the libp2p network. This enables any application at the OS level to leverage P2P functionalities by using VirtualIPs accessible within the VPN. - -The coordination process in Kairos is designed to be resilient and self-coordinated, with no need for complex network configurations or control management interfaces. By using this approach, Kairos simplifies the process of deploying and managing Kubernetes clusters at the edge, making it easy for users to focus on running and scaling their applications. - -

- -

- -### Why Peer-to-Peer? - -Kairos has chosen Peer-to-Peer as an internal component to enable automatic coordination of Kairos nodes. To understand why [EdgeVPN](https://github.com/mudler/edgevpn) has been selected, see the comparison table below, which compares EdgeVPN with other popular VPN solutions: - -| | Wireguard | OpenVPN | EdgeVPN | -|------|-----------|-------------|----------------------------------------------------| -| Memory Space | Kernel-module | Userspace | Userspace | -| Protocol | UDP | UDP, TCP | TCP, UDP/quick, UDP, ws, everything supported by libp2p | -| P2P | Yes | Yes | Yes | -| Fully meshed | No | No | Yes | -| Management Server (SPOF) | Yes | Yes | No | -| Self-coordinated | No | No | Yes | - -Key factors, such as self-coordination and the ability to share metadata between nodes, have led to the selection of EdgeVPN. However, there are tradeoffs and considerations to note in the current architecture, such as: - -- Routing all traffic to a VPN can introduce additional latency -- Gossip protocols can be chatty, especially if using DHT, creating VPNs that span across regions -- EdgeVPN is in user-space, which can be slower compared to kernel-space solutions such as Wireguard -- For highly trafficked environments, there will be an increase in CPU usage due to the additional encryption layers introduced by EdgeVPN - -Nonetheless, these tradeoffs can be overcome, and new features can be added due to EdgeVPN's design. For example: - -- There is no need for any server to handle traffic (no SPOF), and no additional configuration is necessary -- The p2p layer is decentralized and can span across different networks by using DHT and a bootstrap server -- Self-coordination simplifies the provisioning experience -- Internal cluster traffic can also be offloaded to other mechanisms if network performance is a prerequisite -- For instance, with [KubeVIP](/docs/examples/multi-node-p2p-ha-kubevip), new nodes can join the network and become cluster members even after the cluster provisioning phase, making EdgeVPN a scalable solution. - -### Why a VPN ? - -A VPN allows for the configuration of a Kubernetes cluster without depending on the underlying network configuration. This design model is popular in certain use cases at the edge where fixed IPs are not a viable solution. We can summarize the implications as follows: - -| | K8s Without VPN | K8s With VPN | -|----------|---------------------|----------------------------------------------------------| -| IP management | Needs to have static IP assigned by DHCP or manually configured (can be automated) | Automatically coordinated Virtual IPs for nodes. Or manually assign them | -| Network Configuration | `etcd` needs to be configured with IPs assigned by your network/fixed | Automatically assigned, fixed VirtualIPs for `etcd`. | -| Networking | Cluster IPs, and networking is handled by CNIs natively (no layers) | Kubernetes Network services will have Cluster IPs sitting below the VPN.
Every internal kubernetes communication goes through VPN.
The additional e2e encrypted network layer might add additional latency, 0-1ms in LAN.| - -The use of a VPN for a Kubernetes cluster has significant implications. With a VPN, IP management is automatic and does not require static IP addresses assigned by DHCP or manually configured. Nodes can be assigned virtual IPs that are automatically coordinated or manually assigned, which eliminates the need for manual configuration of IP addresses. Additionally, EdgeVPN implements distributed DHCP, so there are no Single point of Failures. - -Additionally, network configuration is simplified with a VPN. Without a VPN, `etcd` needs to be configured with IPs assigned by your network or fixed. With a VPN, virtual IPs are automatically assigned for `etcd`. - -In terms of networking, a Kubernetes cluster without a VPN handles cluster IPs and networking natively without additional layers. However, with a VPN, Kubernetes network services will have Cluster IPs below the VPN. This means that all internal Kubernetes communication goes through the VPN. While the additional end-to-end encrypted network layer might add some latency, it is observed typically to be only 0-1ms in LAN. However, due to the Encryption layers, the CPU usage might be high if used for high-demanding traffic. - -It's also worth noting that while a VPN provides a unified network environment, it may not be necessary or appropriate for all use cases. Users can choose to opt-out of using the VPN and leverage only the coordination aspect, for example, with KubeVIP. Ultimately, the decision to use a VPN should be based on the specific needs and requirements of your Kubernetes cluster, and as such you can just use the co-ordination aspect and leverage for instance [KubeVIP](/docs/examples/multi-node-p2p-ha-kubevip). - -### Packet flow - -The Virtual Private Network used is [EdgeVPN](https://github.com/mudler/edgevpn), which leverages [libp2p](https://github.com/libp2p/go-libp2p) for the transport layer. - -To explain how the packet flow works between two nodes, Node A and Node B, refer to the diagram below: - -

- -

- -While participating actively on a network, each node keeps the shared ledger up-to-date with information about itself and how to be reached by advertizing its own IP and the libp2p identity, allowing nodes to discover each other and how to route packets. - -Assuming that we want to establish an SSH connection from Node A to Node B through the VPN network, which exposes the `sshd` service, the process is as follows: - -1. Node A (`10.1.0.1`) uses `ssh` to dial the VirtualIP of the Node B (`10.1.0.2`) in the network. -2. EdgeVPN reads the frame from the TUN interface. -3. If EdgeVPN finds a match in the ledger between the VirtualIP and an associated Identity, it opens a p2p stream to Node B using the libp2p Identity. -4. Node B receives the incoming p2p stream from EdgeVPN. -5. Node B performs a lookup in the shared ledger. -6. If a match is found, Node B routes the packet back to the TUN interface, up to the application level. - -### Controller - -A set of Kubernetes Native Extensions ([Entangle](/docs/reference/entangle)) provides peer-to-peer functionalities also to existing clusters by allowing to bridge connection with the same design architecture described above. - -It can be used to: - -- Bridge services between clusters -- Bridge external connections to cluster -- Setup EdgeVPN as a daemonset between cluster nodes - -See also the Entangle [documentation](/docs/reference/entangle) to learn more about it. - -## Benefits - -

- -

- -The use of p2p technology to enable self-coordination of Kubernetes clusters in Kairos offers a number of benefits: - -1. **Simplified deployment**: Deploying Kubernetes clusters at the edge is greatly simplified. Users don’t need to specify any network settings or use a control management interface to set up and manage their clusters. -1. **Easy customization**: Kairos offers a highly customizable approach to deploying Kubernetes clusters at the edge. Users can choose from a range of meta distributions, including openSUSE, Ubuntu, Alpine and [many others](/docs/reference/image_matrix), and customize the configuration of their clusters as needed. -1. **Automatic coordination**: With Kairos, the coordination of Kubernetes clusters is completely automated. The p2p network is used as a coordination mechanism for the nodes, allowing them to communicate and coordinate with each other without the need for any external management interface. This means that users can set up and manage their Kubernetes clusters at the edge with minimal effort, freeing up their time to focus on other tasks. -1. **Secure and replicated**: The use of rendezvous points and a shared ledger, encrypted with AES and rotated via OTP, ensures that the p2p network is secure and resilient. This is especially important when deploying Kubernetes clusters at the edge, where network conditions can be unpredictable. -1. **Resilient**: Kairos ensures that the cluster remains resilient, even in the face of network disruptions or failures. By using VirtualIPs, nodes can communicate with each other without the need for static IPs, and the cluster's etcd database remains unaffected by any disruptions. -1. **Scalable**: Kairos is designed to be highly scalable. With the use of p2p technology, users can easily add or remove nodes from the cluster, without the need for any external management interface. -By leveraging p2p technology, Kairos makes it easy for users to deploy and manage their clusters without the need for complex network configurations or external management interfaces. The cluster remains secure, resilient, and scalable, ensuring that it can handle the challenges of deploying Kubernetes at the edge. - -## Conclusions - -In conclusion, Kairos offers an innovative approach to deploying and managing Kubernetes clusters at the edge. By leveraging peer-to-peer technology, Kairos eliminates the need for a control management interface and enables self-coordination of clusters. This makes it easier to deploy and manage Kubernetes clusters at the edge, saving users time and effort. - -The use of libp2p, shared ledger, and OTP for bootstrapping and coordination thanks to [EdgeVPN](https://github.com/mudler/edgevpn) make the solution secure and resilient. Additionally, the use of VirtualIPs and the option to establish a TUN interface ensures that the solution is flexible and can be adapted to a variety of network configurations without requiring exotic configurations. - -With Kairos, users can boost large-scale Kubernetes adoption at the edge, achieve zero-touch configuration, and have their cluster's lifecycle completely managed, all while enjoying the benefits of self-coordination and zero network configuration. This allows users to focus on running and scaling their applications, rather than worrying about the complexities of managing their Kubernetes clusters. - diff --git a/docs/content/en/docs/Development/_index.md b/docs/content/en/docs/Development/_index.md deleted file mode 100644 index f368ded63..000000000 --- a/docs/content/en/docs/Development/_index.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Development" -linkTitle: "Development" -weight: 7 -description: > ---- diff --git a/docs/content/en/docs/Development/debugging-station.md b/docs/content/en/docs/Development/debugging-station.md deleted file mode 100644 index ead7ac92f..000000000 --- a/docs/content/en/docs/Development/debugging-station.md +++ /dev/null @@ -1,246 +0,0 @@ ---- -title: "Debugging station" -linkTitle: "Debugging station" -weight: 4 -date: 2023-03-15 -description: > - Debugging station ---- - -When developing or troubleshooting Kairos, it can be useful to share a local cluster with another peer. This section illustrates how to use [Entangle](/docs/reference/entangle) to achieve that. We call this setup _debugging-station_. - -## Configuration - - -{{% alert title="Note" color="warning" %}} - -This section describes the configuration step by step. If you are in a hurry, you can skip this section and directly go to **Deploy with AuroraBoot**. - -{{% /alert %}} - -When deploying a new cluster, we can use [Bundles](/docs/advanced/bundles) to install the `entangle` and `cert-manager` chart automatically. We specify the bundles in the cloud config file as shown below: - -```yaml -bundles: -- targets: - - run://quay.io/kairos/community-bundles:cert-manager_latest - - run://quay.io/kairos/community-bundles:kairos_latest -``` - -We also need to enable entangle by setting `kairos.entangle.enable: true`. - -Next, we generate a new token that we will use to connect to the cluster later. - -```bash -docker run -ti --rm quay.io/mudler/edgevpn -b -g -``` - -In order for `entangle` to use the token, we can define a `Entanglement` to expose ssh in the mesh network like the following: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: ssh-entanglement - namespace: kube-system -type: Opaque -stringData: - network_token: ___GENERATED TOKEN HERE___ ---- -apiVersion: entangle.kairos.io/v1alpha1 -kind: Entanglement -metadata: - name: ssh-entanglement - namespace: kube-system -spec: - serviceUUID: "ssh" - secretRef: "ssh-entanglement" - host: "127.0.0.1" - port: "22" - hostNetwork: true -``` - -{{% alert title="Note" color="warning" %}} - -If you have already a kubernetes cluster, you can install the [Entangle](/docs/reference/entangle) chart and just apply the manifest. - -{{% /alert %}} - -This entanglement will expose the port `22` in the node over the mesh network with the `ssh` service UUID so we can later connect to it. Replace `___GENERATED TOKEN HERE___` with the token you previously generated with the `docker` command (check out the [documentation](/docs/reference/entangle) for advanced usage). - -In order to deploy the `Entanglement` automatically, we can add it to the `k3s` manifests folder in the cloud config file: - -```yaml -write_files: -- path: /var/lib/rancher/k3s/server/manifests/expose-ssh.yaml - permissions: "0644" - owner: "root" - content: | - apiVersion: v1 - kind: Secret - metadata: - name: ssh-entanglement - namespace: kube-system - type: Opaque - stringData: - network_token: ___GENERATED TOKEN HERE___ - --- - apiVersion: entangle.kairos.io/v1alpha1 - kind: Entanglement - metadata: - name: ssh-entanglement - namespace: kube-system - spec: - serviceUUID: "ssh" - secretRef: "ssh-entanglement" - host: "127.0.0.1" - port: "22" - hostNetwork: true -``` - -Here's an example of a complete cloud configuration file which automatically install a Kairos node in the bigger disk, and exposes ssh with `entangle`: - -```yaml -#cloud-config - -install: - device: "auto" - auto: true - reboot: true - -hostname: debugging-station-{{ trunc 4 .MachineID }} - -users: -- name: kairos - passwd: kairos - ssh_authorized_keys: - - github:mudler - -k3s: - enabled: true - -# Specify the bundle to use -bundles: -- targets: - - run://quay.io/kairos/community-bundles:system-upgrade-controller_latest - - run://quay.io/kairos/community-bundles:cert-manager_latest - - run://quay.io/kairos/community-bundles:kairos_latest - -kairos: - entangle: - enable: true - -write_files: -- path: /var/lib/rancher/k3s/server/manifests/expose-ssh.yaml - permissions: "0644" - owner: "root" - content: | - apiVersion: v1 - kind: Secret - metadata: - name: ssh-entanglement - namespace: kube-system - type: Opaque - stringData: - network_token: ___GENERATED TOKEN HERE___ - --- - apiVersion: entangle.kairos.io/v1alpha1 - kind: Entanglement - metadata: - name: ssh-entanglement - namespace: kube-system - spec: - serviceUUID: "ssh" - secretRef: "ssh-entanglement" - host: "127.0.0.1" - port: "22" - hostNetwork: true -``` - -In this file, you can specify various settings for your debugging station. For example, the `hostname` field sets the name of the machine, and the `users` field creates a new user with the name "kairos" and a pre-defined password and SSH key. The `k3s` field enables the installation of the k3s Kubernetes distribution. - -## Deploy with AuroraBoot - -To automatically boot and install the debugging station, we can use [Auroraboot](/docs/reference/auroraboot). The following example shows how to use the cloud config above with it: - -```bash -cat < - Guidelines when developing Kairos ---- - -Here you can find development notes intended for maintainers and guidance for new contributors. - -## Repository structure - -Kairos uses [earthly](https://earthly.dev/) as a build system instead of Makefiles. This ensures that despite the environment you should be able to build `Kairos` seamlessly. To track external packages (like kernels, additional binaries, and so on) which follow their own versioning [luet](https://luet.io) is used and there is a separate [repository](https://github.com/kairos-io/packages) with package building specifications. - -- [The Kairos repository](https://github.com/kairos-io/kairos) contains the `kairos-agent` code, the OS definitions (`Dockerfile`s) and configuration. The releases generate core ISOs without any Kubernetes engine. -- [The packages repository](https://github.com/kairos-io/packages) contains package specifications used by `kairos` while building OS images. -- [The provider-kairos repository](https://github.com/kairos-io/provider-kairos) contains the kairos provider component which uses the SDK to bring up a Kubernetes cluster with `k3s`. It uses images from `kairos` core to remaster images with `k3s` and the `provider` embedded. This allows to automatically bootstrap Kubernetes cluster. Note, Kairos core in runtime can be extended to add providers or deploy automatically images with the embedded provider. - -## Build Kairos - -To build Kairos you need only Docker installed locally, and there is a convenience script in the root of the repository (`earthly.sh`) which wraps `earthly` inside Docker to avoid to install locally which can be used instead of `earthly` (e.g. `./earthly.sh +iso ...`). However, for daily development, it is strongly suggested to install it in your workstation. The `earthly.sh` script runs `earthly` in a container, and as such there are limitations on image caching between builds. - -To build a Kairos ISO, you need to specify the flavor. For example, to build Kairos Alpine with `earthly` installed locally: - -```bash -earthly -P +iso --FLAVOR=alpine -``` - -This will build a container image from scratch and create an ISO which is ready to be booted. - -Note earthly targets are prefixed with `+` while variables are passed as flags, and `ARGS` can be passed as parameters with `--`. - -### Adding flavors - -Every source image used as a flavor is inside the `images` folder in the top-level directory. Any Dockerfile have the extension corresponding to the flavor which can be used as an argument for earthly builds (you will find a `Dockerfile.alpine` that will be used by our `earthly -P +iso --FLAVOR=alpine` above). - -To add a flavor is enough to create a Dockerfile corresponding to the flavor and check if any specific setting is required for it in the `+framework` target. - -Generally to add a flavor the image needs to have installed: - -- An init system (systemd or openRC are supported) -- Kernel -- GRUB -- rsync - -If you are building a flavor without Earthly, be sure to consume the packages from our repository to convert it to a Kairos-based version. - -### Bumping packages - -Let's assume there is some change you introduce in a package consumed by kairos -(e.g. [kcrypt](https://github.com/kairos-io/kcrypt)). In order to build a kairos image -with the updated package, first tag the repository (`kcrypt` in our example). -Then trigger [the auto-bump pipeline](https://github.com/kairos-io/packages/actions/workflows/autobump.yaml) -on the packages repository. This should create at least on PR which bumps the desired package to the latest tag. -It may also create more PRs if other packages had new tags recently. When PR passes CI, merge it. -Next, in order to bump the packages on kairos, manually trigger [the bump-repos pipeline](https://github.com/kairos-io/kairos/actions/workflows/bump_repos.yml). -This will automatically open a PR on the kairos repository which can be merged when it passes CI. -After this, any images produced by the kairos repository, will have the latest version of the package(s). - -## New controllers - -Kairos-io adopts [operator-sdk](https://github.com/operator-framework/operator-sdk). - -To install `operator-sdk` locally you can use the `kairos` repositories: - -1. Install Luet: - `curl https://luet.io/install.sh | sudo sh` -2. Enable the Kairos repository locally: - `luet repo add kairos --url quay.io/kairos/packages --type docker` -3. Install operator-sdk: - `luet install -y utils/operator-sdk` - -### Create the controller - -Create a directory and let's init our new project it with the operator-sdk: - -```bash - -$ mkdir kairos-controller-foo -$ cd kairos-controller-foo -$ operator-sdk init --domain kairos.io --repo github.com/kairos-io/kairos-controller-foo - -``` - -### Create a resource - -To create a resource boilerplate: - -``` -$ operator-sdk create api --group --version v1alpha1 --kind --resource --controller -``` - -### Convert to a Helm chart - -operator-sdk does not have direct support to render Helm charts (see [issue](https://github.com/operator-framework/operator-sdk/issues/4930)), we use [kubesplit](https://github.com/spectrocloud/kubesplit) to render Helm templates by piping kustomize manifests to it. `kubesplit` will split every resource and add a minimal `helm` templating logic, that will guide you into creating the Helm chart. - -If you have already enabled the `kairos` repository locally, you can install `kubesplit` with: - -``` -$ luet install -y utils/kubesplit -``` - -### Test with Kind - -Operator-sdk will generate a Makefile for the project. You can add the following and edit as needed to add kind targets: - -``` -CLUSTER_NAME?="kairos-controller-e2e" - -kind-setup: - kind create cluster --name ${CLUSTER_NAME} || true - $(MAKE) kind-setup-image - -kind-setup-image: docker-build - kind load docker-image --name $(CLUSTER_NAME) ${IMG} - -.PHONY: test_deps -test_deps: - go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo - go install github.com/onsi/gomega/... - -.PHONY: unit-tests -unit-tests: test_deps - $(GINKGO) -r -v --covermode=atomic --coverprofile=coverage.out -p -r ./pkg/... - -e2e-tests: - GINKGO=$(GINKGO) KUBE_VERSION=${KUBE_VERSION} $(ROOT_DIR)/script/test.sh - -kind-e2e-tests: ginkgo kind-setup install undeploy deploy e2e-tests -``` diff --git a/docs/content/en/docs/Development/nvidia.md b/docs/content/en/docs/Development/nvidia.md deleted file mode 100644 index ec8c71239..000000000 --- a/docs/content/en/docs/Development/nvidia.md +++ /dev/null @@ -1,269 +0,0 @@ ---- -title: "Booting Kairos on Nvidia Jetson ARM" -linkTitle: "Booting Kairos on Nvidia Jetson ARM" -weight: 5 -date: 2022-11-13 -description: > - This page contains a reference on how to run Kairos on Nvidia Jetson ARM ---- - -{{% alert title="Note" %}} -Please note that the following page contains only development reference. At the time of writing, we have tried porting Kairos to Jetson Nano eMMC without success. This is due to the old kernel supported (4.9), not properly working with `EFISTUB` and `U-boot` (you can see the [issue here](https://github.com/kairos-io/kairos/issues/45)). However, the steps outlined _should_ be a good reference to port Kairos to those architecture _when_ a new kernel version is available. We have tested, and have successfully booted a Jetson Nano with the 5.15 kernel, however, due to the lack of driver support, eMMC partitions are not properly recognized. -{{% /alert %}} - -This page is a development reference in order to boot Kairos in Nvidia Jetson devices. Nvidia Jetson images by default ship `extlinux` as bootloader, without EFI boot. This guide explains how to get instead u-boot to chainload to `grub2`, which can be used to boot and load `Kairos`. - -Note that currently there are no official Kairos core images for Jetson images, this page will refer to Jetson Nano eMMC version as the current reference, but the steps should be similar, as outline how to use the Nvidia SDK to flash the OS onboard in the eMMC of the device. - -The steps involved are: - -- Prepare the Kernel (if you have one, compatible with `EFISTUB`, you can skip this part) -- Flash u-boot (If the U-boot version support booting efi shells, you might skip this part too) -- Prepare the Kairos partitions -- Flash the image to the board - -## Prerequisites - -You need the Nvidia SDK and few other dependencies in the system. Note that for the Jetson Nano you can't use the latest SDK version as it is not anymore supporting it. The latest version available with support for Jetson Nano is [r32.7.3](https://developer.nvidia.com/embedded/linux-tegra-r3273): - -```bash -# Build dependencies -apt update && apt install -y git-core build-essential bc wget xxd kmod flex libelf-dev bison libssl-dev - -mkdir build -build_dir=$PWD/build -cd build - -# Get Jetson SDK compatible with Jetson NANO - -wget https://developer.nvidia.com/downloads/remetpack-463r32releasev73t210jetson-210linur3273aarch64tbz2 -O Jetson-210_Linux_R32.7.3_aarch64.tbz2 -tar xvf Jetson-210_Linux_R32.7.3_aarch64.tbz2 -``` - -## Prepare the Kernel - -The only requirement of the kernel in order to this to work is that has to have `CONFIG_EFI_STUB` and `CONFIG_EFI` enabled. - -The default kernel with the Nvidia Jetson Nano is `4.9` and it turns out to not have those enabled. - -### Build from official Nvidia sources - -If your kernel is not compiled to boot as _EFI stub_ you can refer to the steps below to compile the official Nvidia kernel with `EFISTUB`: - -```bash -cd build -wget https://developer.nvidia.com/downloads/remack-sdksjetpack-463r32releasev73sourcest210publicsourcestbz2 -O public_sources.tbz2 -wget https://developer.nvidia.com/embedded/dlc/l4t-gcc-7-3-1-toolchain-64-bit -tar xvf https://developer.nvidia.com/embedded/dlc/l4t-gcc-7-3-1-toolchain-64-bit -# gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/.... -export CROSS_COMPILE_AARCH64_PATH=$PWD/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/ - -cd Linux_for_Tegra/source/public -tar xvf kernel_src.bz2 -mkdir kernel_out -echo "CONFIG_EFI_STUB=y" >> ./kernel/kernel-4.9/arch/arm64/configs/tegra_defconfig -echo "CONFIG_EFI=y" >> ./kernel/kernel-4.9/arch/arm64/configs/tegra_defconfig - -# https://forums.developer.nvidia.com/t/kernel-build-script-nvbuild-sh-with-output-dir-option-not-working/173087 -sed -i '86s/.*/ O_OPT=(O="${KERNEL_OUT_DIR}")/' nvbuild.sh -## See workaround for DTB errors in Troubleshooting (edit Kconfig.include..) -./nvbuild.sh -o $PWD/kernel_out -``` - -Note that, with the Jetson NANO, the kernel will fail to boot allocating the memory during the EFI stub boot phase. - -### Build from official linux kernel - -Seems the kernel `5.15` boots fine on the Jetson Nano, however, it fails to load eMMC drivers to detect eMMC partitions. A configuration reference can be found [here](https://github.com/kairos-io/packages/blob/main/packages/kernels/linux-tegra/config). - -```bash -build_dir=$PWD/build -cd build - -# Clone the kernel -git clone --branch v5.15 --depth 1 https://github.com/torvalds/linux.git kernel-4.9 - -wget https://developer.nvidia.com/downloads/remack-sdksjetpack-463r32releasev73sourcest210publicsourcestbz2 -O public_sources.tbz2 -tar xvf public_sources.tbz2 -wget https://developer.nvidia.com/embedded/dlc/l4t-gcc-7-3-1-toolchain-64-bit -tar xvf l4t-gcc-7-3-1-toolchain-64-bit - -# Replace the kernel in the SDK -pushd Linux_for_Tegra/source/public && tar xvf kernel_src.tbz2 && rm -rf kernel/kernel-4.9 && mv $build_dir/kernel-4.9 ./kernel/ && popd - -# Use the tegra config, patch nvbuild.sh -mkdir kernel_out && \ -wget https://raw.githubusercontent.com/kairos-io/packages/main/packages/kernels/linux-tegra/config -O ./kernel/kernel-4.9/arch/arm64/configs/defconfig && \ -wget https://raw.githubusercontent.com/kairos-io/packages/main/packages/kernels/linux-tegra/nvbuild.sh -O nvbuild.sh && chmod +x nvbuild.sh - -# gcc 12 patches -pushd Linux_for_Tegra/source/public/kernel/kernel-4.9 && curl -L https://raw.githubusercontent.com/kairos-io/packages/main/packages/kernels/linux-tegra/patch.patch | patch -p1 && popd - -# Build the kernel -pushd Linux_for_Tegra/source/public && \ - CROSS_COMPILE_AARCH64_PATH=$build_dir/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu/ ./nvbuild.sh -o $PWD/kernel_out -``` - -## Prepare container image (Kairos) - -Now we need a container image with the OS image. The image need to contain the kernel and the initramfs generated with `dracut`. - -For instance, given that the kernel is available at `/boot/Image`, and the modules at `/lib/modules`: - -```Dockerfile -FROM .... - -RUN ln -sf Image /boot/vmlinuz -RUN kernel=$(ls /lib/modules | head -n1) && \ - dracut -f "/boot/initrd-${kernel}" "${kernel}" && \ - ln -sf "initrd-${kernel}" /boot/initrd && \ - depmod -a "${kernel}" -``` - -## Flashing - -In order to flash to the `eMMC` we need the Nvidia SDK. - -```bash -mkdir work -cd work -wget https://developer.nvidia.com/downloads/remetpack-463r32releasev73t210jetson-210linur3273aarch64tbz2 -tar xvf Jetson-210_Linux_R32.7.3_aarch64.tbz2 -``` - -### Replace U-boot (optional) - -If the version of `u-boot` is old and doesn't support EFI booting, you can replace the `u-boot` binary like so: - -```bash -wget http://download.opensuse.org/ports/aarch64/tumbleweed/repo/oss/aarch64/u-boot-p3450-0000-2023.01-2.1.aarch64.rpm -mkdir u-boot -cd u-boot -rpm2cpio ../u-boot-p3450-0000-2023.01-2.1.aarch64.rpm | cpio -idmv -cd .. -cd Linux_for_Tegra -# "p3450-0000" Depends on your board -cp -rfv ../u-boot/boot/u-boot.bin bootloader/t210ref/p3450-0000/u-boot.bin -``` - -### Disable Extlinux - -We need to disable extlinux, in order for u-boot to scan for EFI shells: - -```bash -# Drop extlinux -echo "" > ./bootloader/extlinux.conf -``` - -### Prepare Partitions - -We need to prepare the partitions from the container image we want to boot, in order to achieve this, we can use `osbuilder`, which will prepare the `img` files ready to be flashed for the SDK: - -```bash -cd Linux_for_Tegra -docker run --privileged -e container_image=$IMAGE -v $PWD/bootloader:/bootloader --entrypoint /prepare_arm_images.sh -ti --rm quay.io/kairos/osbuilder-tools -``` - -This command should create `efi.img`, `oem.img`, `persistent.img`, `recovery_partition.img`, `state_partition.img` in the `bootloader` directory - -### Configure the SDK - -In order to flash the partitions to the eMMC of the board, we need to configure the SDK to write the partitions to the board via its configuration files. - -For the Jetson Nano, the configuration file for the partitions is located at `bootloader/t210ref/cfg/flash_l4t_t210_emmc_p3448.xml`, where we replace the `partition name=APP` with: - -```xml - - sequential - basic - 20971520 - 0 - C12A7328-F81F-11D2-BA4B-00A0C93EC93B - 0x8 - 0 - efi.img - **Required.** Contains a redundant copy of CBoot. - - - - sequential - basic - 2298478592 - 0x8 - recovery_partition.img - - - - sequential - basic - 5234491392 - 0x8 - state_partition.img - - - - sequential - basic - 67108864 - 0x8 - oem.img - - - - sequential - basic - 2147483648 - 0x8 - persistent.img - - -``` - -Note: The order matters here. We want to replace the default "APP" partition with our set of partitions. - -If you didn't changed the default size of the images you should be fine, however, you should check the `` of each of the blocks if corresponds to the files generated from your container image: - -```bash -stat -c %s bootloader/efi.img -stat -c %s bootloader/recovery_partition.img -stat -c %s bootloader/state_partition.img -stat -c %s bootloader/oem.img -stat -c %s bootloader/persistent.img -``` - -### Flash - -Turn the board in recovery mode, depending on the model this process might differ: -- Turn off the board -- Jump the FCC REC pin to ground -- Plug the USB cable -- Power on the board - -If you see the board ready to be flashed, you should see the following: - -```bash -$ lsusb -Bus 003 Device 092: ID 0955:7f21 NVIDIA Corp. APX -``` - -To flash the configuration to the board, run: - -```bash -./flash.sh -r jetson-nano-devkit-emmc mmcblk0p1 -``` - -## Troubleshooting notes - -You can use `picom` to see the serial console: - -```bash -picocom -b 115200 /dev/ttyUSB0 -``` - -## References - -- https://docs.nvidia.com/jetson/archives/r35.1/DeveloperGuide/text/SD/SoftwarePackagesAndTheUpdateMechanism.html#update-with-partition-layout-changes -- https://docs.nvidia.com/jetson/archives/r34.1/DeveloperGuide/text/SD/Kernel/KernelCustomization.html?highlight=kernel -- https://en.opensuse.org/HCL:Jetson_Nano#Update_Firmware -- https://nullr0ute.com/2020/11/installing-fedora-on-the-nvidia-jetson-nano/ -- https://forums.developer.nvidia.com/t/support-nano-on-openwrt/219168/7 \ No newline at end of file diff --git a/docs/content/en/docs/Examples/_index.md b/docs/content/en/docs/Examples/_index.md deleted file mode 100644 index bf6bf26eb..000000000 --- a/docs/content/en/docs/Examples/_index.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: "Examples" -linkTitle: "Examples" -weight: 5 -description: > - This section contains various examples, how-to and tutorial to use Kairos ---- - -Welcome to the examples section of the Kairos documentation! Here, you will find a variety of examples that demonstrate how to use Kairos to create and manage Kubernetes clusters on bare metal. - -## Getting Started - -- [Quick Start Guide](/docs/getting-started): This guide will walk you through the process of installing Kairos and creating your first Kubernetes cluster on bare metal. - -## Troubleshooting - -- [Troubleshooting common issues](/docs/reference/troubleshooting): This page provides solutions to some common issues that you may encounter while using Kairos. \ No newline at end of file diff --git a/docs/content/en/docs/Examples/airgap.md b/docs/content/en/docs/Examples/airgap.md deleted file mode 100644 index 9f34be402..000000000 --- a/docs/content/en/docs/Examples/airgap.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -title: "How to Create an Airgap K3s Installation with Kairos" -linkTitle: "Airgapped ISO with AuroraBoot" -weight: 4 -description: > - This section describe examples on how to use AuroraBoot and Kairos bundles to create ISOs for airgapped installs ---- - -If you want to create an [airgap K3s installation](https://docs.k3s.io/installation/airgap), Kairos provides a convenient way to do so using AuroraBoot. In this guide, we will go through the process of creating a custom ISO of Kairos that contains a configuration file and a [bundle](/docs/advanced/bundles/) that executes preparatory steps after installation. The bundle will overlay new files in the system and prepare the node for having an airgapped K3s installation. - -{{% alert title="Note" %}} -If you already have a Kubernetes cluster, you can use the osbuilder controller to generate container images with your additional files already inside. -{{% /alert %}} - -## Prerequisites - -Docker running in the host - -## Creating the Bundle - -First, we need to create a bundle that contains the K3s images used for the airgap installation. The bundle will place the images in the `/var/lib/rancher/k3s/agent/images` directory. The `/var/lib/rancher` is already configured as persistent by Kairos defaults and every change to that directory persist reboots. You can add additional persistent paths in the system with [the cloud config](/docs/advanced/customizing/#bind-mounts) - -1. Create a new directory named `images-bundle`, and create a new file inside it called `Dockerfile`. -2. Paste the following code into the `Dockerfile`: - -```Dockerfile -FROM alpine -WORKDIR /build -RUN wget https://github.com/k3s-io/k3s/releases/download/v1.23.16%2Bk3s1/k3s-airgap-images-amd64.tar.gz - -FROM scratch -COPY ./run.sh / -COPY --from=alpine /build/k3s-airgap-images-amd64.tar.gz /assets -``` -3. Create a new file called `run.sh` inside the `images-bundle` directory, and paste the following code: - -```bash -#!/bin/bash - -mkdir -p /usr/local/.state/var-lib-rancher.bind/k3s/agent/images/ -cp -rfv ./k3s-airgap-images-amd64.tar.gz /usr/local/.state/var-lib-rancher.bind/k3s/agent/images/ -``` -4. Make the `run.sh` file executable by running the following command: -```bash -chmod +x run.sh -``` -5. Build the container image by running the following command inside the images-bundle directory. This will save the image as `data/bundle.tar`: -```bash -docker build -t images-bundle . -``` -6. Save the bundle: - -``` -$ ls -images-bundle - -# create a directory -$ mkdir data -$ docker save images-bundle -o data/bundle.tar -``` - -## Building the Offline ISO for Airgap - -Now that we have created the bundle, we can use it to build an offline ISO for the airgap installation. - -1. Create a cloud config for the ISO and save it as config.yaml. The config.yaml file should contain your cloud configuration for Kairos and is used to set up the system when it is installed. An example can be: -```yaml -#cloud-config - -install: - auto: true - device: "auto" - reboot: true - bundles: - # This bundle needs to run after-install as it consumes assets from the LiveCD - # which is not accessible otherwise at the first boot (there is no live-cd with any bundle.tar) - - targets: - - run:///run/initramfs/live/bundle.tar - local_file: true - -# Define the user accounts on the node. -users: -- name: "kairos" # The username for the user. - passwd: "kairos" # The password for the user. - ssh_authorized_keys: # A list of SSH keys to add to the user's authorized keys. - - github:mudler # A key from the user's GitHub account. - -k3s: - enabled: true -``` - -2. Build the ISO with [AuroraBoot](/docs/reference/auroraboot) by running the following command: - - -```bash -IMAGE=quay.io/kairos/kairos-opensuse-leap:v1.6.1-k3sv1.26.1-k3s1 - -docker pull $IMAGE - -docker run -v $PWD/config.yaml:/config.yaml \ - -v $PWD/build:/tmp/auroraboot \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v $PWD/data:/tmp/data \ - --rm -ti quay.io/kairos/auroraboot:v0.2.0 \ - --set "disable_http_server=true" \ - --set "disable_netboot=true" \ - --set "container_image=docker://$IMAGE" \ - --set "iso.data=/tmp/data" \ - --cloud-config /config.yaml \ - --set "state_dir=/tmp/auroraboot" -``` - -The resulting ISO should be available at: `build/iso/kairos.iso` - -This example is also available in the [AuroraBoot repository](https://github.com/kairos-io/AuroraBoot/tree/master/examples/airgap) in the `examples/airgap` directory, where you can run `build_docker.sh` to reproduce the example. - -## See also - -- [Customize the OS image](/docs/advanced/customizing/) -- [Live layer bundles](/docs/advanced/livelayering/) -- [Create ISOs with Kubernetes](/docs/installation/automated/#kubernetes) -- [Bundles reference](https://kairos.io/docs/advanced/bundles/) \ No newline at end of file diff --git a/docs/content/en/docs/Examples/bundles.md b/docs/content/en/docs/Examples/bundles.md deleted file mode 100644 index 87728f3ee..000000000 --- a/docs/content/en/docs/Examples/bundles.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: "Bundles" -linkTitle: "Bundles" -weight: 4 -description: > - This section describe examples on how to use a Kairos bundle to deploy MetalLB on top of K3s ---- - -Welcome to the guide on setting up MetalLB on a Kairos cluster with K3s! This tutorial will walk you through the steps of using a Kairos [bundle](/docs/advanced/bundles) to automatically configure MetalLB on your local network with an IP range of `192.168.1.10-192.168.1.20`. Check out the [MetalLB](/docs/examples/metallb) example to configure it without a [bundle](/docs/advanced/bundles). - -For those unfamiliar with [MetalLB](https://metallb.universe.tf/), it is an open-source load balancer implementation for bare metal Kubernetes clusters that utilizes standard routing protocols. When used with K3s on Kairos, it provides load balancing capabilities and helps manage IP addresses within a cluster. - - -## Prerequisites - -Before we begin, you will need to have the following: - -1. Kairos [provider-kairos](https://github.com/kairos-io/provider-kairos) artifacts which includes K3s -1. A baremetal node to run the installation - -## Installation - -1. Follow the [Installation](/docs/installation) documentation for Kairos. -1. Use the following cloud configuration file when setting up Kairos: - -```yaml -#cloud-config - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - ssh_authorized_keys: - # Replace with your github user and un-comment the line below: - # - github:mudler - -k3s: - enabled: true - args: - - --disable=traefik,servicelb - -# Specify the bundle to use -bundles: -- targets: - - run://quay.io/kairos/community-bundles:metallb_latest - -# Specify metallb settings, available only with the bundle. -metallb: - version: 0.13.7 - address_pool: 192.168.1.10-192.168.1.20 -``` - -There are a few key points to note in the configuration file: - -- The `metallb` block is provided by the MetalLB bundle and allows us to specify the version of MetalLB that we want to deploy, as well as the `address_pool` available for our services. -- The `bundles` block enables the `run` [bundle](/docs/advanced/bundles) type. The bundle we are using is part of the [community-bundles](https://github.com/kairos-io/community-bundles) repository. - -And that's it! With these steps, you should now have MetalLB configured and ready to use on your Kairos cluster. If you have any questions or run into any issues, don't hesitate to check out the [bundle documentation](/docs/advanced/bundles) or reach out to the community for support. \ No newline at end of file diff --git a/docs/content/en/docs/Examples/core.md b/docs/content/en/docs/Examples/core.md deleted file mode 100644 index 479ed3f01..000000000 --- a/docs/content/en/docs/Examples/core.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: "Using Kairos Core Images as an Installer" -linkTitle: "Using Kairos Core Images as an Installer" -weight: 4 -description: > - Core images serve as the foundation for creating downstream images or as an installer for deploying other images during the installation process. In this guide, we'll take a closer look at using Kairos core images as an installer to deploy other container images. ---- - -Kairos is a powerful, open-source meta-distribution that allows you to easily deploy and manage nodes on your Immutable infrastructure. - -One key feature of Kairos is the use of its core images, which are released as part of the [kairos-io/kairos](https://github.com/kairos-io/kairos) repository and can be found in the releases section. These core images serve as the foundation for creating [downstream images](/docs/advanced/customizing) or as an installer for deploying other images during the installation process. In this guide, we'll take a closer look at using Kairos core images as an installer to deploy other container images. - -## Getting started - -To begin using Kairos core images as an installer, you'll need to start by using the artifacts from the [Kairos core](https://github.com/kairos-io/kairos/releases) repository. These images do not include the Kubernetes engine, so you'll need to configure the container image you want to deploy in the `install.image` field of your cloud config file. A list of available images can be found in [our support matrix](/docs/reference/image_matrix). - -For example, let's say you want to use an image from the provider-kairos repository. Your cloud config file might look something like this: - -```yaml -#cloud-config -install: - # Here we specify the image that we want to deploy - image: "docker:quay.io/kairos/kairos-opensuse-leap:v1.4.0-k3sv1.26.0-k3s1" -``` - -Once you've chosen your image, you can move on to the installation process by following the steps outlined in our [Installation](/docs/installation) documentation. - -For example, a full cloud-config might look like this: - -```yaml -#cloud-config - -install: - device: "auto" - auto: true - reboot: true - # Here we specify the image that we want to deploy - image: "docker:quay.io/kairos/kairos-opensuse-leap:v1.4.0-k3sv1.26.0-k3s1" - -hostname: "test" -users: -- name: "kairos" - passwd: "kairos" - ssh_authorized_keys: - - github:mudler - -k3s: - enable: true -``` - -## Configuring the installation - -As you move through the installation process, there are a few key points to keep in mind when configuring your cloud config file: - -- We set `install.image` to the container image that we want to deploy. This can be an image from [our support matrix](/docs/reference/image_matrix), a [custom image](/docs/advanced/customizing) or an [image from scratch](/docs/reference/build-from-scratch). -- After the installation is complete, the configuration in the `k3s` block will take effect. This is because after the installation, the system will boot into the image specified in the `install.image` field, which in the example above is an image with the Kairos K3s provider, as such the configuration in the k3s block will become active. - -With these steps, you should now be able to use Kairos core images as an installer to deploy other container images. The process is straightforward and gives you the flexibility to customize your deployments and build custom images as needed. - -You can also refer our [troubleshoot](/docs/reference/troubleshooting) document if you are facing any issue while following the installation process. \ No newline at end of file diff --git a/docs/content/en/docs/Examples/ha.md b/docs/content/en/docs/Examples/ha.md deleted file mode 100644 index 9be3de596..000000000 --- a/docs/content/en/docs/Examples/ha.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -title: "High Availability K3s deployments" -linkTitle: "HA with K3s" -weight: 3 -description: > - This section contains instructions how to deploy Kairos with a High Available control-plane for K3s ---- - -Please refer to the [k3s HA](https://docs.k3s.io/installation/ha-embedded) documentation. - -This document describes how to configure Kairos with `k3s` by following the same documentation outline, to show how to apply `k3s` configuration to `Kairos`. It is implied that you are using a Kairos version with `k3s` included. - -## New cluster - -To run Kairos and k3s in this mode, you must have an odd number of server nodes. K3s documentation recommends starting with three nodes. - -To get started, first launch a server node with the cluster-init flag added in `k3s.args` to enable clustering. A token here can be specified, and will be used as a shared secret to join additional servers to the cluster. Note, if you don't provide one, a token will be generated automatically on your behalf and available at `/var/lib/rancher/k3s/server/node-token`. - -```yaml -#cloud-config - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - #ssh_authorized_keys: - ## Add your github user here! - #- github:mudler - -k3s: - enabled: true - args: - - --cluster-init - # Token will be generated if not specified at /var/lib/rancher/k3s/server/node-token - env: - K3S_TOKEN: "TOKEN_GOES_HERE" -``` - -After launching the first server, join the other servers to the cluster using the shared secret (`K3S_TOKEN`): - -```yaml -#cloud-config - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - ssh_authorized_keys: - # Add your github user here! - - github:mudler - -k3s: - enabled: true - args: - - --server https://:6443 - env: - K3S_TOKEN: "TOKEN_GOES_HERE" -``` - -Now you have a highly available control plane. Any successfully clustered server can be used in the `--server` argument to join additional server and worker nodes. - -### Joining a worker - -Joining additional worker nodes to the cluster follows the same procedure as a single server cluster. - -To join a worker when deploying a Kairos node, use the `k3s-agent` block: - -```yaml -#cloud-config - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - #ssh_authorized_keys: - ## Add your github user here! - #- github:mudler - -k3s-agent: - enabled: true - env: - K3S_TOKEN: "TOKEN_GOES_HERE" - K3S_URL: "https://:6443" -``` - -## External DB - -K3s requires two or more server nodes for this HA configuration. See the [K3s requirements guide](https://docs.k3s.io/installation/requirements) for minimum machine requirements. - -When running the k3s as a server, you must set the datastore-endpoint parameter so that K3s knows how to connect to the external datastore. - -```yaml -#cloud-config - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - #ssh_authorized_keys: - ## Add your github user here! - #- github:mudler - -k3s: - enabled: true - args: - - --datastore-endpoint mysql://username:password@tcp(hostname:3306)/database-name - # Token will be generated if not specified at /var/lib/rancher/k3s/server/node-token - env: - K3S_TOKEN: "TOKEN_GOES_HERE" -``` -## Resources - -- [High Availability with Embedded DB](https://docs.k3s.io/installation/ha-embedded) -- [High Availability with External DB](https://docs.k3s.io/installation/ha) \ No newline at end of file diff --git a/docs/content/en/docs/Examples/metallb.md b/docs/content/en/docs/Examples/metallb.md deleted file mode 100644 index 949ed7ce9..000000000 --- a/docs/content/en/docs/Examples/metallb.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -title: "MetalLB" -linkTitle: "MetalLB" -weight: 4 -description: > - This section describe examples on how to deploy Kairos with k3s and MetalLB ---- - -Welcome to the guide on using MetalLB with Kairos and K3s on a bare metal host! - -In this tutorial, we'll walk through the steps of setting up a Kairos node on your local network using the `192.168.1.10-192.168.1.20` IP range, with MetalLB and K3s. - -But first, let's talk a little bit about what [MetalLB](https://metallb.universe.tf/) and [K3s](https://k3s.io/) are. MetalLB is a load balancer implementation for bare metal Kubernetes clusters that uses standard routing protocols. It's particularly useful when used with K3s in Kairos, as it provides load balancing for bare metal clusters and helps manage IP addresses within the cluster. K3s is a lightweight Kubernetes distribution that is easy to install and maintain. - -Now that you have an understanding of what we'll be working with, let's dive into the installation process. - -Check out the [bundle](/docs/examples/bundles) example to configure `MetalLB` with bundles. Bundles provides a streamlined way to publish and re-use configuration between nodes. - -To get started, you'll need to use the [provider-kairos](https://github.com/kairos-io/provider-kairos) artifacts, which include k3s. We'll be using the [k3s manifest method](/docs/reference/configuration#kubernetes-manifests) to deploy MetalLB. - -Follow the [Installation](/docs/installation) documentation, and use the following cloud config file with Kairos: - -```yaml -#cloud-config - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - ssh_authorized_keys: - # Add your github user here! - - github:mudler - -k3s: - enabled: true - args: - - --disable=traefik,servicelb - -# Additional manifests that are applied by k3s on boot -write_files: -- path: /var/lib/rancher/k3s/server/manifests/metallb.yaml - permissions: "0644" - content: | - apiVersion: v1 - kind: Namespace - metadata: - name: metallb-system - --- - apiVersion: helm.cattle.io/v1 - kind: HelmChart - metadata: - name: metallb - namespace: metallb-system - spec: - chart: https://github.com/metallb/metallb/releases/download/metallb-chart-0.13.7/metallb-0.13.7.tgz -- path: /var/lib/rancher/k3s/server/manifests/addresspool.yaml - permissions: "0644" - content: | - apiVersion: metallb.io/v1beta1 - kind: IPAddressPool - metadata: - name: default - namespace: metallb-system - spec: - addresses: - - 192.168.1.10-192.168.1.20 - --- - apiVersion: metallb.io/v1beta1 - kind: L2Advertisement - metadata: - name: default - namespace: metallb-system - spec: - ipAddressPools: - - default -``` - -There are a few things to note in this configuration file: - -- In the `k3s` block, we use the `--disable` flag to disable `traefik` and `servicelb`, which are the default load balancers for k3s. -- In the `write_files` block, we write manifests (in `/var/lib/rancher/k3s/server/manifests/` see [docs](/docs/reference/configuration#kubernetes-manifests)) to deploy MetalLB and configure it to use the `192.168.1.10-192.168.1.20` IP range. Make sure to choose an IP range that doesn't interfere with your local DHCP network. - -And that's it! You should now have MetalLB and K3s set up on your Kairos node. - -## Resources - -- [TNS blog post](https://thenewstack.io/livin-kubernetes-on-the-immutable-edge-with-kairos-project/) \ No newline at end of file diff --git a/docs/content/en/docs/Examples/multi-node-p2p-ha-kubevip.md b/docs/content/en/docs/Examples/multi-node-p2p-ha-kubevip.md deleted file mode 100644 index f06fc9dd8..000000000 --- a/docs/content/en/docs/Examples/multi-node-p2p-ha-kubevip.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: "Deploying a High-Availability K3s Cluster with KubeVIP" -linkTitle: "Deploying a High-Availability K3s Cluster with KubeVIP" -weight: 6 -date: 2022-11-13 -description: > - This guide walks through the process of deploying a highly-available, P2P self-coordinated k3s cluster with KubeVIP, which provides a high available Elastic IP for the control plane. ---- - -{{% alert title="Note" %}} - -This feature is crazy and experimental! Do not run in production servers. -Feedback and bug reports are welcome, as we are improving the p2p aspects of Kairos. - -{{% /alert %}} - -K3s is a lightweight Kubernetes distribution that is easy to install and operate. It's a great choice for small and edge deployments, but it can also be used to create a high-availability (HA) cluster with the help of [KubeVIP](https://kube-vip.io/). In this guide, we'll walk through the process of deploying a highly-available k3s cluster with KubeVIP, which provides a high available ip for the control plane. - -The first step is to set up the cluster. Kairos automatically deploys an HA k3s cluster with KubeVIP to provide a high available ip for the control plane. KubeVIP allows to setup an ElasticIP that is advertized in the node's network and, as managed as a daemonset in kubernetes it is already running in HA. - - - -The difference between this setup is that we just use the p2p network to automatically co-ordinate nodes, while the connection of the cluster is not being routed to a VPN. The p2p network is used for co-ordination, self-management, and used to add nodes on day 2. - -In order to deploy this setup you need to configure the cloud-config file. You can see the example of the yaml file below. You need to configure the hostname, user and ssh_authorized_keys. You need also to configure kubevip with the elastic ip and the p2p network with the options you prefer. - -```yaml -#cloud-config - -hostname: kairoslab-{{ trunc 4 .MachineID }} -users: -- name: kairos - ssh_authorized_keys: - # Replace with your github user and un-comment the line below: - # - github:mudler - -kubevip: - eip: "192.168.1.110" - -p2p: - # Disabling DHT makes co-ordination to discover nodes only in the local network - disable_dht: true #Enabled by default - - vpn: - create: false # defaults to true - use: false # defaults to true - # network_token is the shared secret used by the nodes to co-ordinate with p2p. - # Setting a network token implies auto.enable = true. - # To disable, just set auto.enable = false - network_token: "" - - # Automatic cluster deployment configuration - auto: - # Enables Automatic node configuration (self-coordination) - # for role assignment - enable: true - # HA enables automatic HA roles assignment. - # A master cluster init is always required, - # Any additional master_node is configured as part of the - # HA control plane. - # If auto is disabled, HA has no effect. - ha: - # Enables HA control-plane - enable: true - # Number of HA additional master nodes. - # A master node is always required for creating the cluster and is implied. - # The setting below adds 2 additional master nodes, for a total of 3. - master_nodes: 2 -``` - -When configuring the `p2p` section, start by adding your desired `network_token` under the p2p configuration in the cloud-config file. To generate a network token, see [documentation](/docs/installation/p2p/#network_token). - -Next, set up an Elastic IP (`kubevip.eip`) with a free IP in your network. KubeVIP will advertise this IP, so make sure to select an IP that is available for use on your network. - -In the VPN configuration, the create and use options are disabled, so the VPN setup is skipped and not used to route any traffic into. \ No newline at end of file diff --git a/docs/content/en/docs/Examples/multi-node-p2p-ha.md b/docs/content/en/docs/Examples/multi-node-p2p-ha.md deleted file mode 100644 index 4cb4613de..000000000 --- a/docs/content/en/docs/Examples/multi-node-p2p-ha.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: "Configuring Automatic High Availability in Kairos" -linkTitle: "Configuring Automatic High Availability in Kairos" -weight: 6 -date: 2022-11-13 -description: > - Kairos makes it easy to configure automatic High Availability (HA) in your cluster by using cloud-config. With just a few simple steps, you can have a fully-functioning HA setup in your cluster. ---- - -{{% alert title="Note" %}} - -This feature is crazy and experimental! Do not run in production servers. -Feedback and bug reports are welcome, as we are improving the p2p aspects of Kairos. - -{{% /alert %}} - -To enable automatic HA rollout, enable the `p2p.auto.ha.enable` option in your cloud-config, and set up a number of `master_nodes`. The number of `master_nodes` is the number of additional masters in addition to the initial HA role. There will always be a minimum of 1 master, which is already taken into account. For example, setting up `master_nodes` to two will result in a total of 3 master nodes in your cluster. - -To make this process even easier, Kairos automatically configures each node in the cluster from a unique cloud-config. This way, you don't have to manually configure each node, but provide instead a config file for all of the machines during [Installation](/docs/installation). - -Here is an example of what your cloud-config might look like: -```yaml -#cloud-config - -hostname: kairoslab-{{ trunc 4 .MachineID }} -users: -- name: kairos - ssh_authorized_keys: - # Replace with your github user and un-comment the line below: - # - github:mudler - -p2p: - # Disabling DHT makes co-ordination to discover nodes only in the local network - disable_dht: true #Enabled by default - - # network_token is the shared secret used by the nodes to co-ordinate with p2p. - # Setting a network token implies auto.enable = true. - # To disable, just set auto.enable = false - network_token: "" - - # Automatic cluster deployment configuration - auto: - # Enables Automatic node configuration (self-coordination) - # for role assignment - enable: true - # HA enables automatic HA roles assignment. - # A master cluster init is always required, - # Any additional master_node is configured as part of the - # HA control plane. - # If auto is disabled, HA has no effect. - ha: - # Enables HA control-plane - enable: true - # Number of HA additional master nodes. - # A master node is always required for creating the cluster and is implied. - # The setting below adds 2 additional master nodes, for a total of 3. - master_nodes: 2 -``` - -Note: In order for the automatic HA rollout to work, you need to generate a network token. You can find more information on how to do this in the [dedicated section](/docs/installation/p2p/#network_token). - diff --git a/docs/content/en/docs/Examples/multi-node-p2p.md b/docs/content/en/docs/Examples/multi-node-p2p.md deleted file mode 100644 index eac146cd0..000000000 --- a/docs/content/en/docs/Examples/multi-node-p2p.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: "P2P multi-node cluster" -linkTitle: "P2P multi-node cluster" -weight: 6 -date: 2022-11-13 -description: > - Install Kairos with p2p support, on a multi-node cluster ---- - -{{% alert title="Note" %}} - -This feature is crazy and experimental! Do not run in production servers. -Feedback and bug reports are welcome, as we are improving the p2p aspects of Kairos. - -{{% /alert %}} - -A multi-node scenario with non-HA is the default peer-to-peer (P2P) configuration in Kairos. To set this up, you will need to configure the `network_token` under the `p2p` configuration in your cloud-config file. Once you have set this, Kairos will handle the configuration of each node. - -Consider the following example, which uses cloud-config to automatically configure the cluster: - - -```yaml -#cloud-config - -hostname: kairoslab-{{ trunc 4 .MachineID }} -users: -- name: kairos - ssh_authorized_keys: - # Replace with your github user and un-comment the line below: - # - github:mudler - -p2p: - # Disabling DHT makes co-ordination to discover nodes only in the local network - disable_dht: true #Enabled by default - - # network_token is the shared secret used by the nodes to co-ordinate with p2p. - # Setting a network token implies auto.enable = true. - # To disable, just set auto.enable = false - network_token: "" - -``` - -To set up a multi-node P2P scenario with non-HA in Kairos, start by adding your desired `network_token` under the p2p configuration in the cloud-config file. To generate a network token, see [documentation](/docs/installation/p2p/#network_token). - -Be sure to set `disable_dht` to true. This will ensure that coordination to discover nodes only happens on the local network. - -Once you done with the above step, you can also customize the hostname to your liking by modifying the `hostname` field, adding your github user to the `ssh_authorized_keys` field, and adding any other necessary configurations. \ No newline at end of file diff --git a/docs/content/en/docs/Examples/multi-node.md b/docs/content/en/docs/Examples/multi-node.md deleted file mode 100644 index 501f97793..000000000 --- a/docs/content/en/docs/Examples/multi-node.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: "Multi Node k3s cluster" -linkTitle: "Multi node k3s cluster" -weight: 1 -description: > - This section describe examples on how to deploy Kairos with k3s as a multi-node cluster ---- - -In the example below we will use a bare metal host to provision a Kairos cluster in the local network with K3s and one master node. - -## Installation - -Use the [provider-kairos](https://github.com/kairos-io/provider-kairos) artifacts which contains `k3s`. - -Follow the [Installation](/docs/installation) documentation, and use the following cloud config file with Kairos for the master and worker: - -{{< tabpane text=true right=true >}} -{{% tab header="server" %}} -```yaml -#cloud-config - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - ssh_authorized_keys: - # Replace with your github user and un-comment the line below: - # - github:mudler - -k3s: - enabled: true - args: - - --disable=traefik,servicelb -``` -{{% /tab %}} -{{% tab header="worker" %}} -```yaml -#cloud-config - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - ssh_authorized_keys: - # Add your github user here! - - github:mudler - -k3s-agent: - enabled: true - env: - K3S_TOKEN: ... - K3S_URL: ... -``` -{{% /tab %}} -{{< /tabpane >}} - -Deploy first the server; the value to use for `K3S_TOKEN` in the worker is stored at /var/lib/rancher/k3s/server/node-token on your server node. - -Notably: - -- we use the `k3s` block to disable `traefik` and `servicelb` (the default `k3s` load balancer) -- You can add additional configuration as args to k3s here, see [k3s](https://docs.k3s.io/reference/server-config#listeners) documentation diff --git a/docs/content/en/docs/Examples/p2p_e2e.md b/docs/content/en/docs/Examples/p2p_e2e.md deleted file mode 100644 index dd70e20c1..000000000 --- a/docs/content/en/docs/Examples/p2p_e2e.md +++ /dev/null @@ -1,165 +0,0 @@ ---- -title: "P2P multi-node cluster with AuroraBoot" -linkTitle: "P2P multi-node cluster with AuroraBoot" -weight: 6 -date: 2023-02-15 -description: > - Full end to end example to bootstrap a self-coordinated cluster with Kairos and AuroraBoot ---- - -{{% alert title="Note" %}} - -The p2p feature of Kairos is crazy and experimental! Do not run in production servers. -Feedback and bug reports are welcome, as we are improving the p2p aspects of Kairos. - -{{% /alert %}} - -Deploying Kubernetes at the Edge can be a complex and time-consuming process, especially when it comes to setting up and managing multiple clusters. To make this process easier, Kairos leverages peer-to-peer technology to automatically coordinate and create Kubernetes clusters without the need of a control management interface. - -To leverage p2p self-coordination capabilities of Kairos, you will need to configure the `network_token` under the `p2p` configuration block in your cloud-config file. Once you have set this, Kairos will handle the configuration of each node. - -{{% alert title="Note" %}} -You can see this example live in the [Kairos and libp2p video]({{< ref "docs/media/#how-kairos-uses-libp2p" >}} "Media") in the [Media Section]({{< ref "docs/media" >}} "Media") -{{% /alert %}} - -## Description - -In the following example we are going to bootstrap a new multi-node, single cluster with Kairos. We will use at least 2 nodes, one as a master and one as a worker. Note how we don't specify any role, or either pin any IP in the following configurations. - -We will first create a cloud config file for our deployment, and then run [AuroraBoot](/docs/reference/auroraboot) locally. We then start 2 VMs configured for netbooting. - -## Prepare your `cloud-config` file - -Consider the following example, which uses cloud-config to automatically configure the cluster: - -We start by creating a cloud config file locally, that could look similar to this: -``` yaml -#cloud-config - -hostname: kairoslab-{{ trunc 4 .MachineID }} -users: -- name: kairos - passwd: kairos - ssh_authorized_keys: - # Replace with your github user and un-comment the line below: - - github:mudler - - github:mauromorales - -p2p: - disable_dht: true # Enable for LAN-only clusters - network_token: "" -``` - -As we want the installation to be triggered automatically, we add also the `install block`: -``` yaml -install: - auto: true - device: "auto" - reboot: true -``` - -In order to leverage p2p and automatic node co-ordination, we need to generate a unique pre-shared token that will be used by all the nodes that we want to be part of our cluster. - -We can generate a network token by using the `edgevpn` images, by running it locally: - -``` -$ docker run -ti --rm quay.io/mudler/edgevpn -b -g -b3RwOgogIGRodDoKICAgIGludGVydmFsOiA5MDAwCiAgICBrZXk6IGtkdGtoY21sMHVJM2hzVUFUMXpUY1B2aDhBblkzNDZUbHJ3NklVRmUxYUoKICAgIGxlbmd0aDogNDMKICBjcnlwdG86CiAgICBpbnRlcnZhbDogOTAwMAogICAga2V5OiBIcEJGaGxxdlFrcTZVd3BPSTBPVkJWQ1daRjNRYlE3WGdDa1R1bnI0cGV3CiAgICBsZW5ndGg6IDQzCnJvb206IGFBUE5oRTdlODgyZUZhM2NMTW56VkM0ZDZjWFdpTU5EYlhXMDE4Skl2Q3oKcmVuZGV6dm91czogOHVzaGhzNnFrTU92U2ZvQmZXMHZPaEY1ZFlodVZlN1Flc00zRWlMM2pNMwptZG5zOiBJZ0ljaGlvRlVYOFN6V1VKQjNXQ0NyT2UzZXZ3YzE4MWVIWm42SmlYZjloCm1heF9tZXNzYWdlX3NpemU6IDIwOTcxNTIwCg== -``` - -This command will generate a network token that we can use in the configuration, which now looks like the following: - -``` yaml -#cloud-config - -# https://github.com/kairos-io/kairos/issues/885 -config_url: "" - -install: - auto: true - device: "auto" - reboot: true - -hostname: kairoslab-{{ trunc 4 .MachineID }} -users: -- name: kairos - passwd: kairos - ssh_authorized_keys: - - github:mudler - - github:mauromorales - -p2p: - disable_dht: true #Enabled by default - network_token: "b3RwOgogIGRodDoKICAgIGludGVydmFsOiA5MDAwCiAgICBrZXk6IGtkdGtoY21sMHVJM2hzVUFUMXpUY1B2aDhBblkzNDZUbHJ3NklVRmUxYUoKICAgIGxlbmd0aDogNDMKICBjcnlwdG86CiAgICBpbnRlcnZhbDogOTAwMAogICAga2V5OiBIcEJGaGxxdlFrcTZVd3BPSTBPVkJWQ1daRjNRYlE3WGdDa1R1bnI0cGV3CiAgICBsZW5ndGg6IDQzCnJvb206IGFBUE5oRTdlODgyZUZhM2NMTW56VkM0ZDZjWFdpTU5EYlhXMDE4Skl2Q3oKcmVuZGV6dm91czogOHVzaGhzNnFrTU92U2ZvQmZXMHZPaEY1ZFlodVZlN1Flc00zRWlMM2pNMwptZG5zOiBJZ0ljaGlvRlVYOFN6V1VKQjNXQ0NyT2UzZXZ3YzE4MWVIWm42SmlYZjloCm1heF9tZXNzYWdlX3NpemU6IDIwOTcxNTIwCg==" -``` - -Change also accordingly the users that can access to the machine: - -``` yaml -ssh_authorized_keys: -- github:mudler <--- put your GitHub handle here -``` - -## Provisioning with AuroraBoot - -We now can run [AuroraBoot](/docs/reference/auroraboot) with `quay.io/kairos/kairos-opensuse-leap:v1.5.1-k3sv1.21.14-k3s1` to provision `openSUSE Leap` machines with `k3s 1.21.14` and Kairos `1.5.1`. - -AuroraBoot takes `cloud-config` files also from _STDIN_, so we will pipe the configuration file to it, and specify the container image that we want to use for our nodes: - -``` bash -cat < kubeconfig -$ KUBECONFIG=kubeconfig k9s -``` - -## Notes - -By default, the Kubernetes API endpoint is not exposed outside the VPN. This is an opinionated configuration from Kairos. To check out configurations without VPN, see also [the KubeVIP example](/docs/examples/multi-node-p2p-ha-kubevip). - -## Troubleshooing - -During the first-boot, you can check the provisioning status by looking at the `kairos-agent` logs: - -``` bash -$ systemctl status kairos-agent -$ journalctl -fu kairos-agent -``` - -## See also - -- [Installation with p2p](/docs/installation/p2p) -- [P2P Architecture](/docs/architecture/network) \ No newline at end of file diff --git a/docs/content/en/docs/Examples/single-node-p2p.md b/docs/content/en/docs/Examples/single-node-p2p.md deleted file mode 100644 index c67907a17..000000000 --- a/docs/content/en/docs/Examples/single-node-p2p.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: "P2P single-node cluster" -linkTitle: "P2P single-node cluster" -weight: 6 -date: 2022-11-13 -description: > - This documentation page provides instructions on how to install Kairos with P2P support on a single-node cluster ---- - -{{% alert title="Note" color="warning" %}} - -This feature is crazy and experimental! Do not run in production servers. -Feedback and bug reports are welcome, as we are improving the p2p aspects of Kairos. - -{{% /alert %}} - -Installing Kairos with P2P support on a single-node cluster requires a few specific steps. To begin, it's important to note that in a single-node scenario, the role must be enforced to a specific role. In a non-HA (high availability) setup, that role can be either `master` or `worker`. In a single-node cluster, there will be only one master node that needs to be configured explicitly. - -To set up a single-node cluster over P2P, consider the following example, which uses cloud-config to automatically configure the cluster: - -```yaml -#cloud-config - -hostname: kairoslab-{{ trunc 4 .MachineID }} -users: -- name: kairos - ssh_authorized_keys: - # Add your github user here! - - github:mudler - -p2p: - role: "master" - # Disabling DHT makes co-ordination to discover nodes only in the local network - disable_dht: true #Enabled by default - - # network_token is the shared secret used by the nodes to co-ordinate with p2p. - # Setting a network token implies auto.enable = true. - # To disable, just set auto.enable = false - network_token: "" - -``` - -{{% alert title="Note" %}} - -One important note is that this example requires the YAML format when editing the configuration file, and that the indentation needs to be accurate, otherwise the configuration will fail. - -{{% /alert %}} - -The above cloud-config configures the hostname, creates a new user `kairos`, and sets the `role` to `master`. Additionally, it disables DHT (distributed hash table) to make the VPN functional only within the local network and use *mDNS* for discovery. If you wish to make the VPN work across different networks, you can set `disable_dht` to `false` or unset it. - -The `network_token` field is a shared secret used by the nodes to coordinate with P2P. Setting a network token implies `auto.enable`. If you wish to disable it, simply set `auto.enable` to false. To generate a network token, see [documentation](/docs/installation/p2p/#network_token). - -Keep in mind that, this example is a minimal configuration, and you can add more options depending on your needs. The above configuration can be used as a starting point and can be customized further. - diff --git a/docs/content/en/docs/Examples/single-node.md b/docs/content/en/docs/Examples/single-node.md deleted file mode 100644 index 942c61e28..000000000 --- a/docs/content/en/docs/Examples/single-node.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: "Single Node k3s cluster" -linkTitle: "Single node k3s cluster" -weight: 1 -description: > - This section describe examples on how to deploy Kairos with k3s as a single-node cluster ---- - -In the example below we will use a bare metal host to provision a Kairos node in the local network with K3s. - -## Installation - -Use the [provider-kairos](https://github.com/kairos-io/provider-kairos) artifacts which contains `k3s`. - -Follow the [Installation](/docs/installation) documentation, and use the following cloud config file with Kairos: - -```yaml -#cloud-config - -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - # Change to your pass here - passwd: kairos - ssh_authorized_keys: - # Replace with your github user and un-comment the line below: - # - github:mudler - -k3s: - enabled: true - args: - - --disable=traefik,servicelb -``` - -Notably: - -- We use the `k3s` block to disable `traefik` and `servicelb` (the default `k3s` load balancer). -- In a single-node setup, you may wish to use a non-generated node name. This can be achieved with these options: - ``` - k3s: - enabled: true - replace_args: true - args: - - --node-name=my-node - ``` - {{% alert title="Note" %}} - `replace_args` replaces all arguments otherwise passed to k3s by Kairos with those supplied here. Make sure you pass all the arguments you need. - {{% /alert %}} -- We use `write_files` to write manifests to the default `k3s` manifest directory (`/var/lib/rancher/k3s/server/manifests/`) see [docs](/docs/reference/configuration#kubernetes-manifests) to deploy `MetalLB` and configure it with the `192.168.1.10-192.168.1.20` IP range. Make sure to pick up a range which doesn't interfere with your local DHCP network. diff --git a/docs/content/en/docs/Getting started/_index.md b/docs/content/en/docs/Getting started/_index.md deleted file mode 100644 index b48feba32..000000000 --- a/docs/content/en/docs/Getting started/_index.md +++ /dev/null @@ -1,320 +0,0 @@ ---- -title: "Getting Started" -linkTitle: "Getting Started" -weight: 2 -description: > - Getting started with Kairos ---- - -{{% alert title="Note" %}} -If you prefer video format, you can also watch our [Introduction to Kairos video]({{< ref "docs/media/#introduction-to-kairos" >}} "Media") on the [Media Section]({{< ref "docs/media" >}} "Media") -{{% /alert %}} - -Ready to launch your Kubernetes cluster with ease? With Kairos, deployment is a breeze! Simply download the pre-packaged artifacts, boot up on a VM or bare metal, and let Kairos handle the rest. Whether you're a Linux or Windows user, our quickstart guide will have you up and running in no time. Kairos can build a Kubernetes cluster for you with just a few simple steps! - -The goal of this quickstart is to help you quickly and easily deploy a Kubernetes cluster using Kairos releases. With Kairos, you can easily build a k3s cluster in a VM, or a baremetal using our pre-packaged artifacts, even if you don't already have a cluster. This process can also be used on bare metal hosts with some configuration adjustments. Check out our documentation further for more detailed instructions and [examples](/docs/examples). - -To create a Kubernetes cluster with Kairos, the only thing needed is one or more machines that will become the Kubernetes nodes. No previously existing clusters is needed. - -Once the installation is complete, you can begin using your Kubernetes cluster. - -## Prerequisites - -- A VM (hypervisor) or a physical server (bare-metal) that boots ISOs -- A Linux or a Windows machine where to run the Kairos CLI (optional, we will see) -- A `cloud-init` configuration file (example below) -- At least 30+ Gb of available disk space. - -## Download - -1. Visit the Kairos [release page on GitHub](https://github.com/kairos-io/provider-kairos/releases) -1. Select the latest release and download the assets of your flavor. For example, - pick the [kairos-opensuse-{{}}-{{}}.iso](https://github.com/kairos-io/provider-kairos/releases/download/v/kairos-opensuse-{{}}-{{}}.iso) - ISO file for the openSUSE based version, where `{{< k3sVersion >}}` in the name is the `k3s` version and `{{< kairosVersion >}}` is the Kairos one to deploy on a VM. -1. You can also use [netboot](/docs/installation/netboot) to boot Kairos over the network - -{{% alert title="Note" %}} -The releases in the [kairos-io/kairos](https://github.com/kairos-io/kairos/releases) repository are the Kairos -core images that ship **without** K3s and P2P full-mesh functionalities; Core images can be used as a -generic installer to [deploy container images](/docs/examples/core). - -The releases in [kairos-io/provider-kairos](https://github.com/kairos-io/provider-kairos/releases) -**contains** already k3s and P2P full-mesh instead. These options need to be explicitly enabled. -In follow-up releases, _k3s-only_ artifacts will also be available. - -See [Image Matrix Support](/docs/reference/image_matrix) for additional supported images and kernels. - -{{% /alert %}} - - -## Checking artifacts signatures - -{{% alert title="Note" color="warning" %}} - -This feature will be available in Kairos version `1.5.0` and in all future releases. - -{{% /alert %}} - -Our ISO releases have sha256 files to checksum the validity of the artifacts. At the same time, our sha256 files are signed automatically in the CI during the -release workflow to verify that they haven't been tampered with, adding an extra step to the supply chain. - -It is recommended that before starting any installation the whole security chain is validated by verifying our sha256 signature and validating that the checksum matches with the download artifacts. - - -To validate the whole chain you would need: - -1. `sha256sum` which is usually installed by default on most linux distributions. -2. `cosign` to verify the signatures of the sha256 file. You can install cosign via their [installation docs](https://docs.sigstore.dev/cosign/installation/) -3. ISO, sha256, certificate and signature files for the release/flavor that you want to verify. All the artifacts are available on the [kairos release page](https://github.com/kairos-io/kairos/releases) - - -In this example we will use the `v1.5.1` version and `opensuse-leap` flavor - -First we check that we have all needed files: - -```bash -$ ls -kairos-opensuse-leap-v1.5.1.iso kairos-opensuse-leap-v1.5.1.iso.sha256.pem -kairos-opensuse-leap-v1.5.1.iso.sha256 kairos-opensuse-leap-v1.5.1.iso.sha256.sig -``` - -We first verify that the sha256 checksums haven't been tampered with: - -```bash -$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --cert kairos-opensuse-leap-v1.5.1.iso.sha256.pem --signature kairos-opensuse-leap-v1.5.1.iso.sha256.sig kairos-opensuse-leap-v1.5.1.iso.sha256 -tlog entry verified with uuid: 51ef927a43557386ad7912802607aa421566772524319703a99f8331f0bb778f index: 11977200 -Verified OK -``` - -Once we see that `Verified OK` we can be sure that the file hasn't been tampered with, and we can continue verifying the iso checksum. - -For an example of a failure validation see below: - -```bash -$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --enforce-sct --cert kairos-opensuse-leap-v1.5.1.iso.sha256.pem --signature kairos-opensuse-leap-v1.5.1.iso.sha256.sig kairos-opensuse-leap-v1.5.1.iso.sha256.modified -Error: verifying blob [kairos-opensuse-leap-v1.5.1.iso.sha256.modified]: invalid signature when validating ASN.1 encoded signature -main.go:62: error during command execution: verifying blob [kairos-opensuse-leap-v1.5.1.iso.sha256.modified]: invalid signature when validating ASN.1 encoded signature -``` -{{% alert title="Info" %}} -We use `COSIGN_EXPERIMENTAL=1` to verify the blob using the keyless method. That means that only ephemeral keys are created to sign, and it relays on using -OIDC Identity Tokens to authenticate so not even Kairos developers have access to the private keys and can modify an existing signature. All signatures are done -via the CI with no external access to the signing process. For more information about keyless signing please check the [cosign docs](https://github.com/sigstore/cosign/blob/main/KEYLESS.md) -{{% /alert %}} - - -Now we can verify that the integrity of the ISO hasnt been compromise: - -```bash -$ sha256sum -c kairos-opensuse-leap-v1.5.1.iso.sha256 -kairos-opensuse-leap-v1.5.1.iso: OK -``` - -Once we reached this point, we can be sure that from the ISO hasn't been tampered with since it was created by our release workflow. - -## Booting - -Now that you have the ISO at hand, it's time to boot! - -Here are some additional helpful tips depending on the physical/virtual machine you're using. - -{{< tabpane text=true right=true >}} - {{% tab header="**Machine**:" disabled=true /%}} - {{% tab header="Bare-Metal" %}} - - When deploying on a bare metal server, directly flash the image into a USB stick. There are multiple ways to do this: - - **From the command line using the `dd` command** - - ```bash - dd if=/path/to/iso of=/path/to/dev bs=4MB - ``` - -
- - **From the GUI** - - For example using an application like [balenaEtcher](https://www.balena.io/etcher/) but can be any other application which allows you to write bootable USBs. - {{% /tab %}} - {{< tab header="QEMU" >}} - {{% alert title="Warning" %}} - Make sure you have KVM enabled, this will improve the performance of your VM significantly! - {{% /alert %}} - - This would be the way to start it via the command line, but you can also use the GUI - - {{< highlight bash >}} - virt-install --name my-first-kairos-vm \ - --vcpus 1 \ - --memory 1024 \ - --cdrom /path/to/kairos-opensuse-{{< kairosVersion >}}-{{< k3sVersion >}}.iso \ - --disk size=30 \ - --os-variant opensuse-factory \ - --virt-type kvm - - {{< / highlight >}} - Immediately after open a viewer so you can interact with the boot menu: - {{< highlight bash >}} - virt-viewer my-first-kairos-vm - {{< / highlight >}} - - {{% /tab %}} -{{< /tabpane >}} - -After booting you'll be greeted with a GRUB boot menu with multiple options. -The option you choose will depend on how you plan to install Kairos: - -- The first entry will boot into installation with a QR code or [WebUI](/docs/installation/webui), - which we'll cover in the next step. -- The second entry will boot into [Manual installation mode](/docs/installation/manual), - where you can install Kairos manually using the console. -- The third boot option boots into [Interactive installation mode](/docs/installation/interactive), - where you can use the terminal host to drive the installation and skip the Configuration and Provisioning step. - -To begin the installation process, select the first entry and let the machine boot. Eventually, a QR code will be printed on the screen. Follow the next step in the documentation to complete the installation. - -![livecd](https://user-images.githubusercontent.com/2420543/189219806-29b4deed-b4a1-4704-b558-7a60ae31caf2.gif) - -## Configuration - -After booting up the ISO, the machine will wait for you to provide configuration details before continuing with the installation process. There are different ways to provide these details: - -- Use the [WebUI](/docs/installation/webui) to continue the installation. -- Serve the configuration via QR code. -- Connect to the machine via [SSH](/docs/installation/manual) and start the installation process with a configuration file ( with `kairos-agent manual-install `). -- [Use a datasource iso, or a generating a custom one](/docs/installation/automated) - -The configuration file is a YAML file with `cloud-init` syntax and additional Kairos configuration details. In this example, we'll configure the node as a single-node Kubernetes cluster using K3s. We'll also set a default password for the Kairos user and define SSH keys. - -Here's an example configuration file that you can use as a starting point: - -{{% alert title="Warning" %}} -The `#cloud-config` at the top is not a comment. Make sure to start your configuration file with it. -{{% /alert %}} - -```yaml -#cloud-config - -# Define the user accounts on the node. -users: -- name: "kairos" # The username for the user. - passwd: "kairos" # The password for the user. - ssh_authorized_keys: # A list of SSH keys to add to the user's authorized keys. - - github:mudler # A key from the user's GitHub account. - - "ssh-rsa AAA..." # A raw SSH key. - -# Enable K3s on the node. -k3s: - enabled: true # Set to true to enable K3s. -``` - -Save this file as config.yaml and use it to start the installation process with kairos-agent manual-install config.yaml. This will configure the node as a single-node Kubernetes cluster and set the default password and SSH keys as specified in the configuration file. - -[Check out the full configuration reference](/docs/reference/configuration). - -**Note**: - -- `users`: This block defines the user accounts on the node. In this example, it creates a user named `kairos` with the password `kairos` and adds two SSH keys to the user's authorized keys. -- `k3s`: This block enables K3s on the node. -- If you want to enable experimental P2P support, check out [P2P installation](/docs/installation/p2p) - -{{% alert title="Note" %}} - -Several configurations can be added at this stage. [See the configuration reference](/docs/reference/configuration) for further reading. - -{{% /alert %}} - -## Provisioning - -{{% alert title="Note" %}} - -You can find instructions showing how to use the Kairos CLI below. In case you prefer to install via SSH and log in to the box, see the [Manual installation](/docs/installation/manual) section or the [Interactive installation](/docs/installation/interactive) section to perform the installation manually from the console. - -{{% /alert %}} - -To trigger the installation process via QR code, you need to use the Kairos CLI. The CLI is currently available only for Linux and Windows. It can be downloaded from the release artifact: - -```bash -curl -L https://github.com/kairos-io/provider-kairos/releases/download/v1.0.0/kairos-cli-v1.0.0-Linux-x86_64.tar.gz -o - | tar -xvzf - -C . -``` - -```bash -# optionally, install the CLI locally -mv kairos-cli /usr/local/bin/kairos -chmod +x /usr/local/bin/kairos - -``` - -The CLI allows to register a node with a screenshot, an image, or a token. During pairing, the configuration is sent over, and the node will continue the installation process. - -In a terminal window from your desktop/workstation, run: - -``` -kairos register --reboot --device /dev/sda --config config.yaml -``` - -**Note**: - -- By default, the CLI will automatically take a screenshot to get the QR code. Make sure it fits into the screen. Alternatively, an image path or a token can be supplied via arguments (e.g. `kairos register /img/path` or `kairos register `). -- The `--reboot` flag will make the node reboot automatically after the installation is completed. -- The `--device` flag determines the specific drive where Kairos will be installed. Replace `/dev/sda` with your drive. Any existing data will be overwritten, so please be cautious. -- The `--config` flag is used to specify the config file used by the installation process. - -After a few minutes, the configuration is distributed to the node and the installation starts. At the end of the installation, the system is automatically rebooted. - -## Accessing the Node - -After the boot process, the node starts and is loaded into the system. You should already have SSH connectivity when the console is available. - -To access to the host, log in as `kairos`: - -```bash -ssh kairos@IP -``` - -**Note**: - -- `sudo` permissions are configured for the Kairos user. - -You will be greeted with a welcome message: - -``` -Welcome to Kairos! - -Refer to https://kairos.io for documentation. -kairos@kairos:~> -``` - -It can take a few moments to get the K3s server running. However, you should be able to inspect the service and see K3s running. For example, with systemd-based flavors: - -``` -$ sudo systemctl status k3s -● k3s.service - Lightweight Kubernetes - Loaded: loaded (/etc/systemd/system/k3s.service; enabled; vendor preset: disabled) - Drop-In: /etc/systemd/system/k3s.service.d - └─override.conf - Active: active (running) since Thu 2022-09-01 12:02:39 CEST; 4 days ago - Docs: https://k3s.io - Main PID: 1834 (k3s-server) - Tasks: 220 -``` - -The K3s `kubeconfig` file is available at `/etc/rancher/k3s/k3s.yaml`. Please refer to the [K3s](https://rancher.com/docs/k3s/latest/en/) documentation. - -## See Also - -There are other ways to install Kairos: - -- [Automated installation](/docs/installation/automated) -- [Manual login and installation](/docs/installation/manual) -- [Create decentralized clusters](/docs/installation/p2p) -- [Take over installation](/docs/installation/takeover) -- [Installation via network](/docs/installation/netboot) -- [Raspberry Pi](/docs/installation/raspberry) -- [CAPI Lifecycle Management (TODO)]() - -## What's Next? - -- [Upgrade nodes with Kubernetes](/docs/upgrade/kubernetes) -- [Upgrade nodes manually](/docs/upgrade/manual) -- [Encrypt partitions](/docs/advanced/partition_encryption) -- [Immutable architecture](/docs/architecture/immutable) diff --git a/docs/content/en/docs/Installation/_index.md b/docs/content/en/docs/Installation/_index.md deleted file mode 100644 index 031b5b74c..000000000 --- a/docs/content/en/docs/Installation/_index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: "Installation" -linkTitle: "Installation" -weight: 2 -description: > - Kairos Installation reference ---- - diff --git a/docs/content/en/docs/Installation/automated.md b/docs/content/en/docs/Installation/automated.md deleted file mode 100644 index da8d5a85e..000000000 --- a/docs/content/en/docs/Installation/automated.md +++ /dev/null @@ -1,224 +0,0 @@ ---- -title: "Automated" -linkTitle: "Automated" -weight: 3 -date: 2022-11-13 -description: > - Install Kairos automatically, with zero touch provisioning ---- - -To automate Kairos installation, you can configure a specific portion of the installation configuration file. The configuration file can then be supplied in a few different ways, such as creating an additional ISO to mount, specifying a URL, or even creating an ISO from a container image with an embedded configuration file. - -Here's an example of how you might customize the install block: - -```yaml -install: - # Device for automated installs - device: "/dev/sda" - # Reboot after installation - reboot: true - # Power off after installation - poweroff: true - # Set to true to enable automated installations - auto: true - # A list of bundles - bundles: - - quay.io/kairos/packages:k9s-utils-0.26.7 -``` - -This block allows you to specify the device on which to install Kairos, whether to reboot or power off after installation, and which bundles to include. - -## Data source - -To supply your Kairos configuration file, you can create an ISO that contains both a user-data file (which contains your configuration) and a meta-data file. - -Here's an example `user-data` configuration that is set up to automatically install Kairos onto /dev/sda and reboot after installation: - -```yaml -#cloud-config - -install: - device: "/dev/sda" - reboot: true - poweroff: false - auto: true # Required, for automated installations - -kairos: - network_token: .... -# extra configuration -``` - -Save this file as `cloud_init.yaml`, then create an ISO with the following steps: - -1. Create a new directory and navigate to it: -```bash -$ mkdir -p build -$ cd build -``` -2. Create empty `meta-data` and copy your config as `user-data`: -```bash -$ touch meta-data -$ cp -rfv cloud_init.yaml user-data -``` -3. Use `mkisofs` to create the ISO file: -```bash -$ mkisofs -output ci.iso -volid cidata -joliet -rock user-data meta-data -``` - -Once the ISO is created, you can attach it to your machine and boot up as usual, along with the Kairos ISO. - -## Via config URL - -Another way to supply your Kairos configuration file is to specify a URL as a boot argument during startup. To do this, add `config_url=` as a boot argument. This will allow the machine to download your configuration from the specified URL and perform the installation using the provided settings. - -After installation, the configuration will be available on the system at `/oem/90_custom.yaml`. - -If you're not sure where to host your configuration file, a common option is to upload it as a GitHub gist. - -## ISO remastering - -It is possible to create custom ISOs with an embedded cloud configuration. This allows the machine to automatically boot with a pre-specified configuration file, which will be installed on the system after provisioning is complete. See also [AuroraBoot](/docs/reference/auroraboot) for documentation. - -### Locally - -To create a custom ISO, you will need Docker installed on your machine. - -Here's an example of how you might do this: - -{{% alert title="Warning" %}} -The image passed to the osbuilder-tools, needs to have one of the accepted schemes: `docker`, `oci`, `file`, `dir` or `channel`. - -If you don't pass one, we will make an attempt to read it as a web URL but depending on your URL this might throw an error. -{{% /alert %}} - -{{< tabpane text=true >}} -{{% tab header="AuroraBoot" %}} - -We can use [AuroraBoot](/docs/reference/auroraboot) to handle the the ISO build process, for example: - -```bash -$ IMAGE= -$ docker pull $IMAGE -# Build the ISO -$ docker run -v $PWD/cloud_init.yaml:/cloud_init.yaml \ - -v $PWD/build:/tmp/auroraboot \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --rm -ti quay.io/kairos/auroraboot \ - --set container_image=docker://$IMAGE \ - --set "disable_http_server=true" \ - --set "disable_netboot=true" \ - --cloud-config /cloud_init.yaml \ - --set "state_dir=/tmp/auroraboot" -# Artifacts are under build/ -$ sudo ls -liah build/iso -total 778M -34648528 drwx------ 2 root root 4.0K Feb 8 16:39 . -34648526 drwxr-xr-x 5 root root 4.0K Feb 8 16:38 .. -34648529 -rw-r--r-- 1 root root 253 Feb 8 16:38 config.yaml -34649370 -rw-r--r-- 1 root root 389M Feb 8 16:38 kairos.iso -34649371 -rw-r--r-- 1 root root 76 Feb 8 16:39 kairos.iso.sha256 -``` -{{% /tab %}} -{{% tab header="Manually" %}} - -```bash -$ IMAGE= -$ mkdir -p files-iso/boot/grub2 -# You can replace this step with your own grub config. This GRUB configuration is the boot menu of the ISO -$ wget https://raw.githubusercontent.com/kairos-io/kairos/master/overlay/files-iso/boot/grub2/grub.cfg -O files-iso/boot/grub2/grub.cfg -# Copy the config file -$ cp -rfv cloud_init.yaml files-iso/cloud_config.yaml -# Pull the image locally -$ docker pull $IMAGE -# Optionally, modify the image here! -# docker run --entrypoint /bin/bash --name changes -ti $IMAGE -# docker commit changes $IMAGE -# Build an ISO with $IMAGE -$ docker run -v $PWD:/cOS -v /var/run/docker.sock:/var/run/docker.sock -i --rm quay.io/kairos/osbuilder-tools:latest --name "custom-iso" --debug build-iso --date=false --local --overlay-iso /cOS/files-iso $IMAGE --output /cOS/ -``` -{{% /tab %}} -{{< /tabpane >}} - -This will create a new ISO with your specified cloud configuration embedded in it. You can then use this ISO to boot your machine and automatically install Kairos with your desired settings. - -You can as well modify the image in this step and add additional packages before deployment. See [customizing the system image](/docs/advanced/customizing). - -Check out the [AuroraBoot documentation](/docs/reference/auroraboot) and the [examples](/docs/examples) for learn more on how to generate customized images for installation. - -### Kubernetes - -It is possible to create custom ISOs and derivatives using extended Kubernetes API resources with an embedded configuration file. This allows you to drive automated installations and customize the container image without breaking the concept of immutability. - -To do this, you will need a Kubernetes cluster. Here's an example of how you might use Kubernetes to create a custom ISO with Kairos: - - -1. Add the Kairos Helm repository: -```bash -$ helm repo add kairos https://Kairos-io.github.io/helm-charts -"kairos" has been added to your repositories -``` -2. Update your Helm repositories: -```bash -$ helm repo update -Hang tight while we grab the latest from your chart repositories... -...Successfully got an update from the "kairos" chart repository -Update Complete. ⎈Happy Helming!⎈ -``` -3. Install the Kairos CRD chart: -```bash -$ helm install kairos-crd kairos/kairos-crds -NAME: kairos-crd -LAST DEPLOYED: Tue Sep 6 20:35:34 2022 -NAMESPACE: default -STATUS: deployed -REVISION: 1 -TEST SUITE: None -``` -4. Install the Kairos `osbuilder` chart: -```bash -$ helm install kairos-osbuilder kairos/osbuilder -NAME: kairos-osbuilder -LAST DEPLOYED: Tue Sep 6 20:35:53 2022 -NAMESPACE: default -STATUS: deployed -REVISION: 1 -TEST SUITE: None -``` -5. Use `kubectl` to apply an `OSArtifact` spec: -```bash -cat <<'EOF' | kubectl apply -f - -apiVersion: build.kairos.io/v1alpha1 -kind: OSArtifact -metadata: - name: hello-kairos -spec: - imageName: "quay.io/kairos/core-opensuse-leap:latest" - iso: true - bundles: - # Bundles available at: https://packages.kairos.io/Kairos/ - - quay.io/kairos/packages:helm-utils-3.10.1 - cloudConfig: | - #cloud-config - users: - - name: "kairos" - passwd: "kairos" - install: - device: "auto" - reboot: true - poweroff: false - auto: true # Required, for automated installations -EOF -``` - -This will create a new ISO with Kairos and the specified bundles included. You can then use this ISO to boot your machine and automatically install Kairos with the specified configuration. - -Note: If you're using kind, you'll need to use the IP address and port of the nginx service to access the ISO. You can get this with: - -```bash -# Note on running with kind: -$ IP=$(docker inspect kind-control-plane | jq -r '.[0].NetworkSettings.Networks.kind.IPAddress') -$ PORT=$(kubectl get svc osartifactbuilder-operator-osbuilder-nginx -o json | jq '.spec.ports[0].nodePort') -$ curl http://$IP:$PORT/hello-kairos.iso -o test.iso -``` - -Check out the [dedicated section in the documentation](/docs/advanced/build) for further examples. diff --git a/docs/content/en/docs/Installation/interactive.md b/docs/content/en/docs/Installation/interactive.md deleted file mode 100644 index 3425d57eb..000000000 --- a/docs/content/en/docs/Installation/interactive.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: "Interactive" -linkTitle: "Interactive" -weight: 2 -date: 2022-11-13 -description: > - Install Kairos interactively ---- - -The interactive installation can be accessed from the LiveCD ISO and guides the user into the installation process. - -It generates a configuration file, which is later accessible after installation in the `/oem/99_custom.yaml` file. - -## From the boot menu - -When loading any Kairos ISOs, a GRUB menu, like the following will be displayed. To access the interactive installation, select the third entry (`kairos (interactive install)`). - -![interactive](https://user-images.githubusercontent.com/2420543/189219819-6b16d13d-c409-4b9b-889b-12792f800a08.gif) - -## Manually - -The interactive installer can be also started manually with `kairos-agent interactive-install` from the LiveCD. diff --git a/docs/content/en/docs/Installation/manual.md b/docs/content/en/docs/Installation/manual.md deleted file mode 100644 index fc6feb310..000000000 --- a/docs/content/en/docs/Installation/manual.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: "Manual installation" -linkTitle: "Manual installation" -weight: 1 -date: 2022-11-13 -description: > - Install Kairos manually ---- - -To install manually, follow the [quickstart](/docs/getting-started). When the QR code is prompted at the screen, you will be able to log in via SSH to the box with the password `kairos` as `kairos` user. - -{{% alert title="Note" %}} - -**Note**: After the installation, the password login is disabled, users, and SSH keys to log in must be configured via cloud-init. - -{{% /alert %}} - - -## Installation - -To start the installation, run the following command from the console: - -```bash -sudo kairos-agent manual-install --device "auto" $CONFIG -``` - -Where the configuration can be a `cloud-init` file or a URL to it: - -```yaml -#cloud-init - -p2p: - network_token: .... -# extra configuration -``` - -**Note**: -- The command is disruptive and will erase any content on the drive. -- The parameter **"auto"** selects the biggest drive available in the machine. diff --git a/docs/content/en/docs/Installation/netboot.md b/docs/content/en/docs/Installation/netboot.md deleted file mode 100644 index 5dde062fe..000000000 --- a/docs/content/en/docs/Installation/netboot.md +++ /dev/null @@ -1,128 +0,0 @@ ---- -title: "Network booting" -linkTitle: "Network booting" -weight: 5 -date: 2022-12-1 -description: > - Install Kairos from network ---- - -Most hardware these days, supports booting an operating system from the network. -The technology behind this is called [Preboot Execution Environment](https://en.wikipedia.org/wiki/Preboot_Execution_Environment). -Kairos releases include artifacts to allow booting from the network. In general, the following files are needed: - -- The initrd image: It's the system that loads first. It's responsible to load the kernel. -- The kernel: This is the kernel of the operating system that will boot. -- The squashfs: The filesystem of the operating system that will boot. - -Booting using these files can happen in multiple ways: - -- Either with direct support from the machine BIOS plus network configuration (DHCP server etc). -- Software based network booting. This works with a special ISO, built with - [ipxe](https://ipxe.org/) project. Kairos releases include pre-built ISOs for - netbooting (named like `*.ipxe.iso.ipxe`). -- Use [AuroraBoot](/docs/reference/auroraboot) - -Generic hardware based netbooting is out of scope for this document. -Below we give instructions on how to use the Kairos release artifacts to netboot and how to use [AuroraBoot](/docs/reference/auroraboot) to boot from network. - -## Boot with pre-built ISOs - -The ipxe ISOs from the Kairos release artifacts, were built with a ipxe script that points directly to the -`kernel`, `initrd` and `squashfs` artifacts of the same release on GitHub. - -E.g.: - - -```bash -#!ipxe -set url https://github.com/kairos-io/kairos/releases/download/v1.3.0 -set kernel kairos-alpine-opensuse-leap-v1.3.0-kernel -set initrd kairos-alpine-opensuse-leap-v1.3.0-initrd -set rootfs kairos-alpine-opensuse-leap-v1.3.0.squashfs - -# Configure interface -ifconf - -# set config https://example.com/machine-config -# set cmdline extra.values=1 -kernel ${url}/${kernel} initrd=${initrd} rd.neednet=1 ip=dhcp rd.cos.disable root=live:${url}/${rootfs} netboot nodepair.enable config_url=${config} console=tty1 console=ttyS0 ${cmdline} -initrd ${url}/${initrd} -boot -``` - -Booting the ISO will automatically download and boot those artifacts. E.g. using qemu: - -```bash -#!/bin/bash - -qemu-img create -f qcow2 disk.img 40g -qemu-system-x86_64 \ - -m 4096 \ - -smp cores=2 \ - -nographic \ - -drive if=virtio,media=disk,file=disk.img \ - -drive if=ide,media=cdrom,file=${1:-kairos.iso} - -``` - -## Use AuroraBoot - -[AuroraBoot](/docs/reference/auroraboot) is a Kairos convinience tool that can be used to quickly deploy Kairos from Network with zero-touch configuration, for instance: - -```bash -docker run --rm -ti --net host quay.io/kairos/auroraboot \ - --set "container_image=quay.io/kairos/kairos-opensuse-leap:v1.5.1-k3sv1.21.14-k3s1" - # Optionally: - # --cloud-config .... -``` - -Will netboot the `quay.io/kairos/kairos-opensuse-leap:v1.5.1-k3sv1.21.14-k3s1` image. You can find more details in the [AuroraBoot documentation section](/docs/reference/auroraboot). - -## Notes on booting from network - -Another way to boot with the release artifacts is using [pixiecore](https://github.com/danderson/netboot/tree/master/pixiecore). -`pixiecore` acts as a server which offers net boot files over the network and it's automatically discovered on a network where a DHCP server is running and is compatible with [the pixiecore architecture](https://github.com/danderson/netboot/blob/master/pixiecore/README.booting.md). - - -Assuming the current directory has the `kernel`, `initrd` and `squashfs` artifacts, -`pixiecore` server can be started with `docker` like this: - - - -```bash -#!/bin/bash - -VERSION="v1.3.0" -wget "https://github.com/kairos-io/kairos/releases/download/${VERSION}/kairos-opensuse-${VERSION}-kernel" -wget "https://github.com/kairos-io/kairos/releases/download/${VERSION}/kairos-opensuse-${VERSION}-initrd" -wget "https://github.com/kairos-io/kairos/releases/download/${VERSION}/kairos-opensuse-${VERSION}.squashfs" - -cat << EOF > config.yaml -#cloud-config - -hostname: "hostname.domain.tld" -users: -- name: "kairos" - passwd: "kairos" -EOF - -# This will start the pixiecore server. -# Any machine that depends on DHCP to netboot will be send the specified files and the cmd boot line. -docker run \ - -d --name pixiecore --net=host -v $PWD:/files quay.io/pixiecore/pixiecore \ - boot /files/kairos-opensuse-${VERSION}-kernel /files/kairos-opensuse-${VERSION}-initrd --cmdline="rd.neednet=1 ip=dhcp rd.cos.disable root=live:{{ ID \"/files/kairos-opensuse-${VERSION}.squashfs\" }} netboot nodepair.enable config_url={{ ID \"/files/config.yaml\" }} console=tty1 console=ttyS0 console=tty0" -``` - -If your machine doesn't support netbooting, you can use our [generic image](https://github.com/kairos-io/ipxe-dhcp/releases), which is built using an ipxe script [from the pixiecore project](https://github.com/danderson/netboot/blob/master/pixiecore/boot.ipxe). The ISO will wait for a DHCP proxy response from pixiecore. - -If pixiecore is successfully reached, you should see an output similar to this in the `pixiecore` docker container: - -``` -$ docker logs pixiecore -[DHCP] Offering to boot 08:00:27:e5:22:8c -[DHCP] Offering to boot 08:00:27:e5:22:8c -[HTTP] Sending ipxe boot script to 192.168.1.49:4371 -[HTTP] Sent file "kernel" to 192.168.1.49:4371 -[HTTP] Sent file "initrd-0" to 192.168.1.49:4371 -``` diff --git a/docs/content/en/docs/Installation/p2p.md b/docs/content/en/docs/Installation/p2p.md deleted file mode 100644 index cb78b5842..000000000 --- a/docs/content/en/docs/Installation/p2p.md +++ /dev/null @@ -1,281 +0,0 @@ ---- -title: "P2P support" -linkTitle: "P2P support" -weight: 6 -date: 2022-11-13 -description: > - Install Kairos with p2p support ---- - -{{% alert title="Note" %}} - -This feature is crazy and experimental! Do not run in production servers. -Feedback and bug reports are welcome, as we are improving the p2p aspects of Kairos. - -{{% /alert %}} - -Deploying Kubernetes at the Edge can be a complex and time-consuming process, especially when it comes to setting up and managing multiple clusters. To make this process easier, Kairos leverages peer-to-peer technology to automatically coordinate and create Kubernetes clusters without the need of a control management interface. - -With this feature, users don't need to specify any network settings. They can just set the desired number of master nodes (in the case of an HA cluster) and the necessary configuration details, and Kairos will take care of the rest. The peer-to-peer technology allows the nodes in the cluster to communicate and coordinate with each other, ensuring that the clusters are set up correctly and efficiently with K3s. - -This makes it easier to deploy and manage Kubernetes clusters at the Edge, saving user's time and effort, allowing them to focus on running and scaling their applications. For more information about how does it work behind the scenes, [check out the architecture section](/docs/architecture/network). - -You can find full examples in our [examples section](/docs/examples): -- [Full end to end example to bootstrap a self-coordinated cluster with Kairos and AuroraBoot](/docs/examples/p2p_e2e/) -- [Self-coordinated K3s HA cluster with KubeVIP](/docs/examples/multi-node-p2p-ha-kubevip/) -- [Multi-node, single master setup](/docs/examples/multi-node-p2p/) -- [Multi-node, HA setup](/docs/examples/multi-node-p2p-ha/) -- [Single-node setup](/docs/examples/single-node-p2p/) - -This feature is currently experimental and can be optionally enabled by adding the following configuration to the node deployment file. If you are not familiar with the installation process, it is suggested to follow the [quickstart](/docs/getting-started): - -```yaml -p2p: - # Disabling DHT makes co-ordination to discover nodes only in the local network - disable_dht: true #Enabled by default - # Automatic cluster deployment configuration - auto: - ha: - # Enables HA control-plane - enable: true - # number of HA master node (beside the one used for init) for the control-plane - master_nodes: 2 - # network_token is the shared secret used by the nodes to co-ordinate with p2p. - # Setting a network token implies auto.enable = true. - # To disable, just set auto.enable = false - network_token: "YOUR_TOKEN_GOES_HERE" - - -``` - -To enable the automatic cluster deployment with peer-to-peer technology, specify a `p2p.network_token`. To enable HA, set `p2p.auto.ha.master_nodes` to the number of wanted HA/master nodes. Additionally, the p2p block can be used to configure the VPN and other settings as needed. - -With these settings used to deploy all the nodes, those will automatically communicate and coordinate with each other to deploy and manage the Kubernetes cluster without the need for a control management interface and user intervention. - -## Configuration - -A minimum configuration file, that bootstraps a cluster with a simple single-master topology, can look like the following: - -```yaml -#cloud-config -hostname: "kubevip-{{ trunc 4 .MachineID }}" - -users: -- name: "kairos" - passwd: "kairos" - ssh_authorized_keys: - - github:mudler -p2p: - network_token: "YOUR_TOKEN_GOES_HERE" -``` - -The `p2p` block is used to configure settings related to the mesh functionalities. The minimum required argument is the `network_token` and there is no need to configure `k3s` manually with the `k3s` block as it is already implied. - -{{% alert title="Note" %}} - -The `k3s` block can still be used to override other `k3s` settings, e.g. `args`. - -{{% /alert %}} - -The network token is a shared secret available to all the nodes of the cluster. It allows the node to co-ordinate and automatically assign roles. To generate a network token, see [documentation](/docs/installation/p2p/#network_token). - -Simply applying the same configuration file to all the nodes should eventually bring one master and all the other nodes as workers. Adding nodes can be done also in a later step, which will automatically setup the node without any further configuration. - -Full example: - - -```yaml -#cloud-config - -install: - auto: true - device: "auto" - reboot: true - -hostname: "kubevip-{{ trunc 4 .MachineID }}" -users: -- name: "kairos" - passwd: "kairos" - ssh_authorized_keys: - - github:mudler - -## Sets the Elastic IP used in KubeVIP -kubevip: - eip: "192.168.1.110" - # Specify a manifest URL for KubeVIP. Empty uses default - manifest_url: "" - # Enables KubeVIP - enable: true - # Specifies a KubeVIP Interface - interface: "ens18" - -p2p: - role: "" # Set an hardcoded role, optional - # Disabling DHT makes co-ordination to discover nodes only in the local network - disable_dht: true #Enabled by default - # Configures a VPN for the cluster nodes - vpn: - create: false # defaults to true - use: false # defaults to true - env: - ..... - # Automatic cluster deployment configuration - auto: - # Enables Automatic node configuration (self-coordination) - # for role assignment - enable: true - # HA enables automatic HA roles assignment. - # A master cluster init is always required, - # Any additional master_node is configured as part of the - # HA control plane. - # If auto is disabled, HA has no effect. - ha: - # Enables HA control-plane - enable: true - # Number of HA additional master nodes. - # A master node is always required for creating the cluster and is implied. - # The setting below adds 2 additional master nodes, for a total of 3. - master_nodes: 2 - # Use an External database for the HA control plane - external_db: "external-db-string" - # network_token is the shared secret used by the nodes to co-ordinate with p2p - network_token: "YOUR_TOKEN_GOES_HERE" -``` - -In the YAML configuration example, there are several important keywords that control the behavior of the automatic cluster deployment: - -| Keyword | Description | -| --- | --- | -| `p2p` | Configures the peer to peer networking of the cluster | -| `p2p.disable_dht` | Disables the distributed hash table for cluster discovery | -| `p2p.network_token` | The shared secret used by the nodes to coordinate with p2p | -| `p2p.network_id` | Optional, unique identifier for the kubernetes cluster. It allows bootstrapping of multiple cluster using the same network token | -| `p2p.role` | Force a specific role for the node of the cluster | -| `p2p.vpn` | Configures a VPN for the cluster nodes | -| `p2p.vpn.create` | Enables the creation of the VPN | -| `p2p.vpn.use` | Enables the use of the VPN for routing Kubernetes traffic of the cluster | -| `p2p.vpn.env` | Configures the environment variables used to start for the VPN | -| `p2p.vpn.auto` | Configures the automatic deployment of the cluster | -| `p2p.auto.enable` | Enables automatic node configuration for role assignment | -| `p2p.auto.ha` | Configures the high availability settings for the cluster | -| `p2p.auto.ha.enable` | Enables the high availability settings | -| `p2p.auto.ha.master_nodes` | The number of additional HA master nodes expected in the cluster. | -| `p2p.auto.ha.external_db` | The external database used for high availability | - -## Elastic IP - -If deploying a cluster in a Local network, it might be preferable to disable the VPN functionalities. - -We use KubeVIP to provide an elastic ip for the control plane that can be configured via a specific block: - -```yaml - -p2p: - network_token: ".." - vpn: - # Disable VPN, so traffic is not configured with a VPN - create: false - use: false - -## Sets the Elastic IP used in KubeVIP -kubevip: - eip: "192.168.1.110" - # Specify a manifest URL for KubeVIP. Empty uses default - manifest_url: "" - # Enables KubeVIP - enable: true - # Specifies a KubeVIP Interface - interface: "ens18" -``` - - -| Keyword | Description | -| --- | --- | -| `kubevip` | Block to configure KubeVIP for the cluster | -| `kubevip.eip` | The Elastic IP used for KubeVIP. Specifying one automatically enables KubeVIP. Choose a free IP that is not in a DHCP range of your network. | -| `kubevip.manifest_url` | The URL for the KubeVIP manifest | -| `kubevip.enable` | Enables KubeVIP for the cluster | -| `kubevip.interface` | The interface used for KubeVIP | - -A full example, with KubeVIP and HA: - -```yaml - -#cloud-config - -install: - auto: true - device: "auto" - reboot: true - -hostname: "kubevip-{{ trunc 4 .MachineID }}" -users: -- name: "kairos" - passwd: "kairos" - ssh_authorized_keys: - - github:mudler - -p2p: - network_token: "..." - ha: - master_nodes: 2 - vpn: - # Disable VPN, so traffic is not configured with a VPN - create: false - use: false - -kubevip: - eip: "192.168.1.110" -``` - -## `network_token` - -The `network_token` is a unique code that is shared among nodes and can be created with the Kairos CLI or `edgevpn`. This allows nodes to automatically connect to the same network and generates private/public key pairs for secure communication using end-to-end encryption. - -To generate a new token, run: -{{< tabpane text=true right=true >}} -{{% tab header="docker" %}} -```bash -docker run -ti --rm quay.io/mudler/edgevpn -b -g -``` -{{% /tab %}} -{{% tab header="CLI" %}} -```bash -kairos generate-token -``` -{{% /tab %}} -{{< /tabpane >}} - -## Join new nodes - -To add new nodes to the network, follow the same process as before and use the same configuration file for all machines. Unless you have specified roles for each node, no further changes to the configuration are necessary. The machines will automatically connect to each other, whether they are on a local or remote network. - -## Connect to the nodes - -To connect to the nodes, you can use the kairos-cli and provide the network_token to establish a tunnel to the nodes network. - -```bash -sudo kairos bridge --network-token -``` - -This command creates a TUN device on your machine and allows you to communicate with each node in the cluster. - -{{% alert title="Note" color="info" %}} -The command requires root permissions in order to create a TUN/TAP device on the host. -{{% /alert %}} - -An API will be also available at [localhost:8080](http://localhost:8080) for inspecting the network status. - -## Get kubeconfig - -To get the cluster `kubeconfig`, you can log in to the master node and retrieve it from the engine (e.g., it is located at `/etc/rancher/k3s/k3s.yaml` for K3s) or use the Kairos CLI. If using the CLI, you must be connected to the bridge or logged in from one of the nodes and run the following command in the console: - -```bash -kairos get-kubeconfig > kubeconfig -``` - -{{% alert title="Note" color="info" %}} - -Note that you must run kairos bridge in a separate window as act like `kubectl proxy` and access the Kubernetes cluster VPN. Keep the kairos bridge command running to operate the cluster. - -{{% /alert %}} diff --git a/docs/content/en/docs/Installation/qrcode.md b/docs/content/en/docs/Installation/qrcode.md deleted file mode 100644 index 9d7816649..000000000 --- a/docs/content/en/docs/Installation/qrcode.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: "QR Code" -linkTitle: "QR Code" -weight: 1 -date: 2022-11-13 -description: > - Use the QR code displayed at boot to drive the installation ---- - -{{% alert title="Warning" %}} -You will need a Standard Kairos OS image in order to use QR Code feature. -{{% /alert %}} - -By default Kairos will display a QR code after booting the ISO to install the machine: - -![livecd](https://user-images.githubusercontent.com/2420543/189219806-29b4deed-b4a1-4704-b558-7a60ae31caf2.gif) - - -To trigger the installation process via QR code, you need to use the Kairos CLI and provide a Cloud Config, as described in the [Getting started guide](/docs/getting-started). You can also see some Cloud Config examples in our [Examples section](/docs/examples). The CLI is currently available only for Linux and Windows. It can be downloaded from the release artifact: - -```bash -VERSION=$(wget -q -O- https://api.github.com/repos/kairos-io/provider-kairos/releases/latest | jq -r '.tag_name') -curl -L https://github.com/kairos-io/provider-kairos/releases/download/${VERSION}/kairos-cli-${VERSION}-Linux-x86_64.tar.gz -o - | tar -xvzf - -C . -``` - -The CLI allows to register a node with a screenshot, an image, or a token. During pairing, the configuration is sent over, and the node will continue the installation process. - -In a terminal window from your desktop/workstation, run: - -``` -kairos register --reboot --device /dev/sda --config config.yaml -``` - -- The `--reboot` flag will make the node reboot automatically after the installation is completed. -- The `--device` flag determines the specific drive where Kairos will be installed. Replace `/dev/sda` with your drive. Any existing data will be overwritten, so please be cautious. -- The `--config` flag is used to specify the config file used by the installation process. - -{{% alert title="Note" %}} -By default, the CLI will automatically take a screenshot to get the QR code. Make sure it fits into the screen. Alternatively, an image path or a token can be supplied via arguments (e.g. `kairos register /img/path` or `kairos register `). -{{% /alert %}} - -After a few minutes, the configuration is distributed to the node and the installation starts. At the end of the installation, the system is automatically rebooted. - diff --git a/docs/content/en/docs/Installation/raspberry.md b/docs/content/en/docs/Installation/raspberry.md deleted file mode 100644 index cd0bc37da..000000000 --- a/docs/content/en/docs/Installation/raspberry.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -title: "RaspberryPi" -linkTitle: "RaspberryPi" -weight: 4 -date: 2022-11-13 -description: > - Install Kairos on RaspberryPi 3 and 4 ---- - -Kairos supports Raspberry Pi model 3 and 4 with 64bit architecture. - -If you are not familiar with the process, it is suggested to follow the [quickstart](/docs/getting-started) first to see how Kairos works. - -## Prerequisites - -- An SD card which size is at least 16 GB -- Etcher or `dd` -- A Linux host where to flash the device - -## Download - -Extract the `img` file from a container image as described [in this page](/docs/reference/image_matrix) - -## Flash the image - -Plug the SD card to your system. To flash the image, you can either use Etcher or `dd`. Note it's compressed with "XZ", so we need to decompress it first: - -```bash -xzcat kairos-opensuse-leap-arm-rpi-v1.0.0-rc2-k3sv1.21.14+k3s1.img.xz | sudo dd of= oflag=sync status=progress bs=10MB -``` - -Once the image is flashed, there is no need to carry any other installation steps. We can boot the image, or apply our config. - -## Boot - -Use the SD Card to boot. The default username/password is `kairos`/`kairos`. -To configure your access or disable password change the `/usr/local/cloud-config/01_defaults.yaml` accordingly. - -## Configure your node - -To configure the device beforehand, be sure to have the SD plugged in your host. We need to copy a configuration file into `cloud-config` in the `COS_PERSISTENT` partition: - -``` -$ PERSISTENT=$(blkid -L COS_PERSISTENT) -$ mkdir /tmp/persistent -$ sudo mount $PERSISTENT /tmp/persistent -$ sudo mkdir /tmp/persistent/cloud-config -$ sudo cp cloud-config.yaml /tmp/persistent/cloud-config -$ sudo umount /tmp/persistent -``` - -You can push additional `cloud config` files. For a full reference check out the [docs](/docs/reference/configuration) and also [configuration after-installation](/docs/advanced/after-install) - -## Customizing the disk image - -The following shell script shows how to localy rebuild and customize the image with docker - -``` -IMAGE=quay.io/kairos/kairos-alpine-arm-rpi:v1.1.6-k3sv1.25.3-k3s1 -# Pull the image locally -docker pull $IMAGE -mkdir -p build -docker run -v $PWD:/HERE -v /var/run/docker.sock:/var/run/docker.sock --privileged -i --rm --entrypoint=/build-arm-image.sh quay.io/kairos/osbuilder-tools:v0.4.0 \ - --model rpi64 \ - --state-partition-size 6200 \ - --recovery-partition-size 4200 \ - --size 15200 \ - --images-size 2000 \ - --local \ - --config /HERE/cloud-config.yaml \ - --docker-image $IMAGE /HERE/build/out.img - -``` diff --git a/docs/content/en/docs/Installation/takeover.md b/docs/content/en/docs/Installation/takeover.md deleted file mode 100644 index 9cfb09d50..000000000 --- a/docs/content/en/docs/Installation/takeover.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: "Takeover" -linkTitle: "Takeover" -weight: 7 -date: 2022-11-13 -description: > ---- - -Kairos supports takeover installations. Here are a few summarized steps: - -- From the dedicated control panel (OVH, Hetzner, etc.), boot in *rescue* mode -- [Install docker](https://docs.docker.com/engine/install/debian/) and run for example: - -``` -export DEVICE=/dev/sda -export IMAGE=quay.io/kairos/core-opensuse-leap:v1.1.4 -cat <<'EOF' > config.yaml -#cloud-config -users: -- name: "kairos" - passwd: "kairos" - ssh_authorized_keys: - - github:mudler -EOF -export CONFIG_FILE=config.yaml -docker run --privileged -v $PWD:/data -v /dev:/dev -ti $IMAGE elemental install --cloud-init /data/$CONFIG_FILE --system.uri $IMAGE $DEVICE -``` - -- Switch back to *booting* from HD and reboot. diff --git a/docs/content/en/docs/Installation/webui.md b/docs/content/en/docs/Installation/webui.md deleted file mode 100644 index 53c81b07e..000000000 --- a/docs/content/en/docs/Installation/webui.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: "WebUI" -linkTitle: "WebUI" -weight: 1 -date: 2022-11-13 -description: > - Use the WebUI at boot to drive the installation ---- - -{{% alert title="Note" color="warning" %}} - -This feature will be available in Kairos version `1.5.0` and in all future releases. - -{{% /alert %}} - -By default when running the LiveCD, or during installation, Kairos will start a WebUI in the background, listening by default on the `8080` port: - -![WebUI](https://user-images.githubusercontent.com/2420543/214573939-31f887b8-890c-4cce-a02a-0100198ea7d9.png) - -The WebUI has an input form that accepts the `YAML` config file, features a syntax highlighter and a `YAML` syntax checker. You can find a [full example in our documentation](/docs/reference/configuration) or navigate to our [examples section](/docs/examples). diff --git a/docs/content/en/docs/Media/_index.md b/docs/content/en/docs/Media/_index.md deleted file mode 100644 index 73f11a253..000000000 --- a/docs/content/en/docs/Media/_index.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: "Media" -linkTitle: "Media" -weight: 9 -description: > - Presentation Slides, Videos and other media on Kairos ---- - -## Articles - - * [Livin’ Kubernetes on the (Immutable) Edge with Kairos Project](https://thenewstack.io/livin-kubernetes-on-the-immutable-edge-with-kairos-project/) on The New Stack - -## Slides - - * [Kairos and libp2p](https://github.com/kairos-io/kairos/files/10743709/Kairos_P2P.pdf) - -## Videos - -### Introduction to Kairos - -{{< youtube id="WzKf6WrL3nE" title="Introduction to Kairos" >}} - -### Meet Kairos, an OSS project building the immutable Kubernetes edge - -{{< youtube id="kiDQujibz2k" title="Meet Kairos, an OSS project building the immutable Kubernetes edge" >}} - -### How we build and maintain Kairos - -{{< youtube id="XD5nfMf59v4" title="How we build and maintain Kairos" >}} - -### How Kairos uses libp2p - -{{< youtube id="7Vym18wz9Uw" title="Kairos and libp2p" >}} diff --git a/docs/content/en/docs/Reference/_index.md b/docs/content/en/docs/Reference/_index.md deleted file mode 100644 index 1e3870bc7..000000000 --- a/docs/content/en/docs/Reference/_index.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Reference" -linkTitle: "Reference" -weight: 6 -description: > ---- diff --git a/docs/content/en/docs/Reference/architecture.md b/docs/content/en/docs/Reference/architecture.md deleted file mode 100644 index 000e35696..000000000 --- a/docs/content/en/docs/Reference/architecture.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: "Architecture" -linkTitle: "Architecture" -weight: 1 -date: 2022-11-13 -description: > - Kairos internal architecture ---- - -This section contains refrences to how Kairos works internally. - -## Setup process - -`kairos` node at first boot will start the `kairos-agent` service, you can always check what's happening by running `journalctl -fu kairos-agent`. - -This service will setup `k3s` and `edgevpn` dynamically on first-boot, once it configures the machine it does not run on boot anymore, unless `/usr/local/.kairos/deployed` is removed.. - -Those are the steps executed in sequence by the `kairos-agent` service: - -- Will create a `edgevpn@kairos` service and enabled on start. The configuration for the connection is stored in `/etc/systemd/system.conf.d/edgevpn-kairos.env` and depends on the cloud-init configuration file provided during installation time -- Automatic role negotiation starts, nodes will co-ordinate for an IP and a role -- Once roles are defined a node will either set the `k3s` or `k3s-agent` service. Configuration for each service is stored in `/etc/sysconfig/k3s` and `/etc/sysconfig/k3s-agent` respectively - -## Paths - -The following paths are relevant for Kairos: - -| Path | Description | -| :-------------------------- | :--------------------------------------------------------------------------------------------- | -| /usr/local/.kairos/deployed | Sentinel file written after bootstrapping is complete. Remove to retrigger automatic bootstrap | -| /usr/local/.kairos/lease | IP Lease of the node in the network. Delete to change IP address of the node | diff --git a/docs/content/en/docs/Reference/auroraboot.md b/docs/content/en/docs/Reference/auroraboot.md deleted file mode 100644 index 3afbcb9fa..000000000 --- a/docs/content/en/docs/Reference/auroraboot.md +++ /dev/null @@ -1,615 +0,0 @@ ---- -title: "AuroraBoot" -linkTitle: "AuroraBoot" -weight: 8 -date: 2023-02-08 -description: > - Automatically provision machines with Kairos and AuroraBoot. ---- - -**AuroraBoot** is a tool designed to make the process of bootstrapping Kairos machines quick, simple and efficient. It is specifically designed for the Kairos operating system and provides a comprehensive solution for downloading required artifacts and provisioning a machine, both from network or manually via flashing to USB stick. - -With AuroraBoot, you can prepare the environment for network-based bootstrapping, download the necessary release assets, and also customize the installation media for USB-based mass-installations. Whether you're looking to install Kairos on a single machine or multiple machines, AuroraBoot makes it easy and efficient. - -AuroraBoot can be useful to: - -- prepare multiple-nodes in a lab before shipment -- offer a simple, intuitive and streamlined way to deploy Kairos automatically and manually -- deploy Kairos nodes in a network segment where we can already send workload to (running AuroraBoot in an already-existing downstream cluster) - -![AuroraBoot](https://user-images.githubusercontent.com/2420543/217617696-f993a8e3-55ac-4d3e-98f0-c2317cb54cb9.png) - -## Scope - -**AuroraBoot** has the following scope: - -- **Download** release assets in order to provision one or more machines -- **Prepare** automatically the environment to boot from network -- **Provision** machines from network with a version of Kairos and cloud config -- **Customize** The installation media for installations from USB - -## Prerequisites - -- `docker` or a container engine of your choice -- Port `8090`, `8080` and `67` free on the host running AuroraBoot -- The machine running AuroraBoot have to be on the same network segment of the nodes to be bootstrapped -- The nodes need to be configured to boot over network, or be capable of booting via USB for offline mode -- `ProxyDHCP` supported by the `DHCP` network attempting to netboot (see also [pixiecore architecture](https://github.com/danderson/netboot/blob/master/pixiecore/README.booting.md#step-1-dhcpproxydhcp)). - There should be an already running `DHCP` server on your network. AuroraBoot doesn't take over the `DHCP` server, neither require you to do any specific configuration, however a `DHCP` server which is compliant to `ProxyDHCP` requests should be present in the same network running **AuroraBoot** and the machines to boot. - -## MacOS - -Unfortunately for macOS systems we cannot run the netboot through docker as it's run inside a VM, as it can't see the host network. -Building ISOs still works as long as you mount the container `/tmp` disk to a local dir so its exported there like so: - -```bash -docker run --rm -ti -v ${PWD}:/tmp quay.io/kairos/auroraboot \ - --set "artifact_version=v1.5.1" \ - --set "release_version=v1.5.1" \ - --set "flavor=opensuse-leap" \ - --set "repository=kairos-io/kairos" \ - --set "disable_http_server=true" \ - --set "disable_netboot=true" \ - --cloud-config /config.yaml -``` - -This will build the ISO and put the generated artifacts in the current dir under the `${PWD}/iso` dir. - -For netboot, we recommend that you run the AuroraBoot binary directly by grabbing it from the [releases page](https://github.com/kairos-io/AuroraBoot/releases). -This requires just one dependency that you can install via [brew](https://brew.sh/) with `brew install xorriso` - -## Windows - -Netboot in windows is not supported, only iso creation via the docker image. - - -## Overview - -To run AuroraBoot, simply use `docker` or the container engine of your choice (such as `podman`, ...). AuroraBoot images are published in [quay](https://quay.io/repository/kairos/auroraboot) and the source code is available [in GitHub](https://github.com/kairos-io/AuroraBoot). - -The basic usage of AuroraBoot involves passing it several parameters that define the installation environment, such as the version of Kairos you want to install, the cloud config you want to use, and other customizations you may need. You can pass these parameters either as command-line arguments, or as a full YAML configuration file. - -AuroraBoot will download the artifacts required for bootstrapping the nodes, and prepare the environment required for a zero-touch deployment. - -For example, to netboot a machine with the latest version of Kairos and Rocky Linux using a cloud config, you would run the following command: - -```bash -docker run --rm -ti --net host quay.io/kairos/auroraboot \ - --set "artifact_version=v1.5.0" \ - --set "release_version=v1.5.0" \ - --set "flavor=rockylinux" \ - --set repository="kairos-io/kairos" \ - --cloud-config https://... -``` - -This command will download the necessary artifacts and start the provisioning process. The machine will attempt to boot from network, and will be configured with the specified version of Kairos. - -### Network-based bootstrapping - -By default AuroraBoot will automatically attempt to bootstrap other machines, which are configured to boot from network, within the same network. No further configuration or settings necessary. - -There are only 3 steps involved in the process: - -1. Select the release of Kairos that you want to deploy and optionally a cloud config (see also our [examples](/docs/examples)) -1. Run AuroraBoot in your workstation with the appropriate CLI args -1. Boot up other nodes, already configured to boot from network - -#### 1. Selecting a release - -AuroraBoot can bootstrap container images or released assets from our GitHub release process. - -To use GitHub releases set a release version with `--set release_version` (the GitHub release), an artifact version with `--set artifact_version` (the artifact version) a flavor with `--set flavor` and a repository with `--set repository`. -Kairos has releases with and without k3s. The [release page at kairos](https://github.com/kairos-io/kairos/releases) are ["core" images that can be used as installer](/docs/examples/core/) while the [provider-kairos](https://github.com/kairos-io/provider-kairos/releases) images contains also `k3s`. - -To use a container image, you can use [the Kairos released images](/docs/reference/image_matrix/) or [customized](/docs/advanced/customizing) by specifying `--set container_image` instead with the container image of choice. - -#### 2. Run AuroraBoot - -Now we can run AuroraBoot with the version we selected, either from GitHub releases or directly from a container image. - -In the example below we selected `v1.5.1-k3sv1.21.14-k3s1`, `opensuse-leap` flavor, so we would run either one of the following: - -{{< tabpane text=true >}} -{{% tab header="Container image" %}} - -By indicating a `container_image`, AuroraBoot will pull the image locally and start to serve it for network booting. - -You can use [the Kairos released images](/docs/reference/image_matrix/) or [your own](/docs/advanced/customizing). - -```bash -docker run --rm -ti --net host quay.io/kairos/auroraboot \ - --set "container_image=quay.io/kairos/kairos-opensuse-leap:v1.5.1-k3sv1.21.14-k3s1" -``` - -{{% /tab %}} -{{% tab header="Container Image, with dockerd" %}} - -By indicating a `container_image` prefixed with `docker://`, AuroraBoot will pull the image from the local daemon and start to serve it for network booting. - -This implies that the host has a docker daemon, and we have to give access to its socket with `-v /var/run/docker.sock:/var/run/docker.sock`. - -```bash -docker pull quay.io/kairos/kairos-opensuse-leap:v1.5.1-k3sv1.21.14-k3s1 -# This will use the container image from the host's docker daemon -docker run --rm -ti -v /var/run/docker.sock:/var/run/docker.sock --net host quay.io/kairos/auroraboot \ - --set "container_image=docker://quay.io/kairos/kairos-opensuse-leap:v1.5.1-k3sv1.21.14-k3s1" -``` -{{% /tab %}} -{{% tab header="Github releases" %}} - -By indicating a `artifact_version`, a `release_version`, a `flavor` and a `repository`, AuroraBoot will use GitHub released assets. - -```bash -docker run --rm -ti --net host quay.io/kairos/auroraboot \ - --set "artifact_version=v1.5.1-k3sv1.21.14+k3s1" \ - --set "release_version=v1.5.1" \ - --set "flavor=opensuse-leap" \ - --set "repository=kairos-io/provider-kairos" -``` -{{% /tab %}} -{{< /tabpane >}} - -To specify a cloud config, you can set it with `--cloud-config`. See the sections below for further examples. - -#### 3. Start nodes - -Generic hardware based netbooting is out of scope for this document. - -Nodes need to be configured to boot over network, and after AuroraBoot is started should be ready to accept a connection, a typical output of a successfull run is: - -```bash -2023/02/08 14:27:30 DHCP: Offering to boot 08:00:27:54:1a:d1 -2023/02/08 14:27:30 TFTP: Sent "08:00:27:54:1a:d1/4" to 192.168.68.113:6489 -2023/02/08 14:27:36 DHCP: Offering to boot 08:00:27:54:1a:d1 -2023/02/08 14:27:36 HTTP: Sending ipxe boot script to 192.168.68.113:45435 -2023/02/08 14:27:36 HTTP: Sent file "kernel" to 192.168.68.113:45435 -2023/02/08 14:27:36 HTTP: Sent file "initrd-0" to 192.168.68.113:45435 -2023/02/08 14:27:49 HTTP: Sent file "other-0" to 192.168.68.113:43044 -``` - -If trying on a VM, for instance on VirtualBox or QEMU, a typical setup might be: - -- Set Netboot as first boot in the boot process order - -![Screenshot from 2023-02-08 10-37-59](https://user-images.githubusercontent.com/2420543/217587463-cd293842-575e-4484-aee5-de46c4f053fb.png) - -- Use bridge networking with the host (if running AuroraBoot and the VM in the same host) - -![Screenshot from 2023-02-08 10-38-05](https://user-images.githubusercontent.com/2420543/217587465-35486742-26a1-4971-bee0-3049d9ec329a.png) - -### USB-based bootstrapping - -AuroraBoot by default prepares an ISO with the custom cloud init prepared for being flashed to an USB stick either with `dd` or with [BalenaEtcher](https://www.balena.io/etcher). - -To disable netboot and provide only offline artifacts, run `auroraboot` with `--set disable_netboot=true`. - -#### 1. Node configuration - -Create a cloud config file, see [our documentation](/docs/examples) for ready-to use examples, but a minimal configuration that automatically installs, and allows us to login afterward can be the following: - -```yaml -#cloud-config - -install: - auto: true - device: "auto" - reboot: true - -# Define the user accounts on the node. -users: -- name: "kairos" # The username for the user. - passwd: "kairos" # The password for the user. - ssh_authorized_keys: # A list of SSH keys to add to the user's authorized keys. - # - github:mudler # A key from the user's GitHub account. - # - "ssh-rsa AAA..." # A raw SSH key. -``` - -Save the file locally or remotely, you can pass it by in the arguments with `--cloud-config` to AuroraBoot. Note that can also be a remote http(s) path. - -#### 2. Create an offline ISO - -Run AuroraBoot with a cloud-config to create an ISO with the embedded configuration: - -{{< tabpane text=true >}} -{{% tab header="Container image" %}} - -Check we have the cloud config file: -```bash -ls -# config.yaml -``` - -Build the ISO: -```bash -docker run -v "$PWD"/config.yaml:/config.yaml \ - -v "$PWD"/build:/tmp/auroraboot \ - --rm -ti quay.io/kairos/auroraboot \ - --set container_image=quay.io/kairos/core-rockylinux:v1.5.0 \ - --set "disable_http_server=true" \ - --set "disable_netboot=true" \ - --cloud-config /config.yaml \ - --set "state_dir=/tmp/auroraboot" -``` - -Results should be available under `build/` in the current directory: -```bash -sudo ls -liah build/iso -# -# total 778M -# 34648528 drwx------ 2 root root 4.0K Feb 8 16:39 . -# 34648526 drwxr-xr-x 5 root root 4.0K Feb 8 16:38 .. -# 34648529 -rw-r--r-- 1 root root 253 Feb 8 16:38 config.yaml -# 34649370 -rw-r--r-- 1 root root 389M Feb 8 16:38 kairos.iso -# 34649372 -rw-r--r-- 1 root root 389M Feb 8 16:39 kairos.iso.custom.iso -# 34649371 -rw-r--r-- 1 root root 76 Feb 8 16:39 kairos.iso.sha256 -``` -{{% /tab %}} -{{% tab header="Github releases" %}} - -Check we have the cloud config file: -```bash -ls -# config.yaml -``` - -Build the ISO: -```bash -docker run -v "$PWD"/build:/tmp/auroraboot -v /var/run/docker.sock:/var/run/docker.sock --rm -ti quay.io/kairos/auroraboot \ - --set "artifact_version=v1.5.1-k3sv1.21.14+k3s1" \ - --set "release_version=v1.5.1" \ - --set "flavor=opensuse-leap" \ - --set "repository=kairos-io/provider-kairos" \ - --set "disable_http_server=true" \ - --set "disable_netboot=true" \ - --cloud-config /config.yaml \ - --set "state_dir=/tmp/auroraboot" -``` - -Results should be available under `build/` in the current directory: - -```bash -sudo ls -liah build/iso -# -# total 778M -# 34648528 drwx------ 2 root root 4.0K Feb 8 16:39 . -# 34648526 drwxr-xr-x 5 root root 4.0K Feb 8 16:38 .. -# 34648529 -rw-r--r-- 1 root root 253 Feb 8 16:38 config.yaml -# 34649370 -rw-r--r-- 1 root root 389M Feb 8 16:38 kairos.iso -# 34649372 -rw-r--r-- 1 root root 389M Feb 8 16:39 kairos.iso.custom.iso -# 34649371 -rw-r--r-- 1 root root 76 Feb 8 16:39 kairos.iso.sha256 -``` - -{{% /tab %}} -{{< /tabpane >}} - - -The result process will write an iso `kairos.iso.custom.iso` under `build/iso`. That is the iso with our embedded cloud-config. - -#### 2. Run the image - -The iso now is ready to be written to USB stick with either `dd` or with [BalenaEtcher](https://www.balena.io/etcher), or attached to a VM. - -{{< tabpane text=true right=true >}} - {{% tab header="**Machine**:" disabled=true /%}} - {{% tab header="Bare-Metal" %}} - - When deploying on a bare metal server, directly flash the image into a USB stick. There are multiple ways to do this: - - **From the command line using the `dd` command** - - ```bash - dd if=build/kairos.iso.custom.iso of=/path/to/dev bs=4MB - ``` - - or with [BalenaEtcher](https://www.balena.io/etcher). - - {{% /tab %}} - {{< tab header="QEMU" >}} - {{% alert title="Warning" %}} - Make sure you have KVM enabled, this will improve the performance of your VM significantly! - {{% /alert %}} - - This would be the way to start it via the command line, but you can also use the GUI - - {{< highlight bash >}} - virt-install --name my-first-kairos-vm \ - --vcpus 1 \ - --memory 1024 \ - --cdrom build/kairos.iso.custom.iso \ - --disk size=30 \ - --os-variant opensuse-factory \ - --virt-type kvm - - {{< / highlight >}} - Immediately after open a viewer so you can interact with the boot menu: - {{< highlight bash >}} - virt-viewer my-first-kairos-vm - {{< / highlight >}} - - {{% /tab %}} -{{< /tabpane >}} - -## Configuration - -The AuroraBoot configuration file reference is the following: - -```yaml -# Corresponding artifact versions from the kairos release page (e.g. kubernetes version included) -artifact_version: "v..." -# Version of the release in github -release_version: "v1.5.0" - -# Flavor -flavor: "rockylinux" - -# Github repository -repository: "kairos-io/kairos" - -# Container image (takes over) -container_image: "..." - -# Disable netboot -disable_netboot: true - -# Disable http server for serving offline generated ISOs -disable_http_server: true - -# Specify a directory that will be used by auroraboot to download artifacts -# Reuse the same to cache artifacts. -state_dir: "/tmp/auroraboot" - -# Default http binding port for offline ISO generation -listen_addr: ":8080" - -# Cloud config to use when booting the machine. -cloud_config: | -``` - -| Option | Description | -| ------ | ----------- | -| `artifact_version` | Corresponding artifact versions from the Kairos release page (e.g. Kubernetes version included). | -| `release_version` | Version of the release in GitHub. | -| `flavor` | The Kairos flavor to use. See [the Kairos support matrix](/docs/reference/image_matrix/) for a list. | -| `repository` | Github repository to use. This can either be `kairos-io/kairos` or `kairos-io/provider-kairos` for images with `k3s`. | -| `container_image` | Container image. If prefixed with `docker://` it will try to pull from the local docker daemon. If a `container_image` is specified, `artifact_version`, `flavor` and `release_version` are ignored. | -| `disable_netboot` | Disable netboot. | -| `disable_http_server` | Disable http server for serving offline generated ISOs. | -| `netboot_http_port` | Specify a netboot HTTP port (defaults to `8090`). | -| `state_dir` | Specify a directory that will be used by auroraboot to download artifacts and reuse the same to cache artifacts. | -| `listen_addr` | Default http binding port for offline ISO generation. | -| `cloud_config` | Cloud config path to use for the machines. A URL can be specified, use `-` to pass-by the cloud-config from _STDIN_ | -| `iso.data` | Defines a path to be embedded into the resulting iso. When booting, the files will be accessible at `/run/initramfs/live` | -| `netboot.cmdlin` | Override the automatically generated cmdline with a custom one to use during netboot. `config_url` and `rootfs` are automatically constructed. A reasonable value can be `netboot.cmdline=rd.neednet=1 ip=dhcp rd.cos.disable netboot nodepair.enable console=tty0` | - -To use the configuration file with AuroraBoot, run AuroraBoot specifying the file or URL of the config as first argument: - -```bash -docker run --rm -ti -v "$PWD"/config.yaml:/config.yaml --net host quay.io/kairos/auroraboot /config.yaml -``` - -The CLI options can be used in place of specifying a file, and to set fields of it. Any field of the YAML file, excluding `cloud_config` can be configured with the `--set` for instance, to disable netboot we can run AuroraBoot with: - -```bash -docker run --rm -ti --net host quay.io/kairos/auroraboot .... --set "disable_netboot=true" -``` - -To specify a cloud config file instead, use `--cloud-config` (can be also url): - -```bash -docker run --rm -ti -v "$PWD"/config.yaml:/config.yaml --net host quay.io/kairos/auroraboot .... --cloud-config /config.yaml -``` - -Both the config file and the cloud-config file can be a URL. - -### Cloud config - -A custom cloud configuration file can be passed either with the `--cloud-config` flag, or in the AuroraBoot configuration file under the `cloud_config` key. - -It is possible to apply templating to a cloud config. Indeed any value passed to `--set` is accessible as a template in the cloud config file with the `[[` and `]]` delimiter, for instance consider the following cloud config file, which allows to set a password for the `kairos` user and a GitHub handle allowed to login to the machine: - -```yaml -#cloud-config - -install: - auto: true - device: "auto" - reboot: true - -# Define the user accounts on the node. -users: -- name: "kairos" # The username for the user. - passwd: "[[.kairos.password]]" # The password for the user. - ssh_authorized_keys: # A list of SSH keys to add to the user's authorized keys. - - github:[[.github.user]] -``` - -We would then set the user to `mudler` and the password to `foobar` when running AuroraBoot like the following: - -```bash -docker run --rm -ti -v "$PWD"/config.yaml:/config.yaml --net host \ - quay.io/kairos/auroraboot \ - --cloud-config /config.yaml \ - --set "github.user=mudler" \ - --set "kairos.password=foobar" -``` - -Config files can be also hosted remotely, and given as URLs to AuroraBoot. - -We can indeed use the template in the example folder with the command above: - -```bash -docker run --rm -ti --net host \ - quay.io/kairos/auroraboot \ - --cloud-config https://raw.githubusercontent.com/kairos-io/kairos/master/examples/auroraboot/master-template.yaml \ - --set "github.user=mudler" \ - --set "kairos.password=foobar" -``` - -To pass-by a cloud-config via pipes, set `--cloud-config -`, for example: - -```yaml -cat < -``` - -Build the custom ISO with the cloud config: - -```bash -docker run -v "$PWD"/config.yaml:/config.yaml \ - -v "$PWD"/build:/tmp/auroraboot \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --rm -ti quay.io/kairos/auroraboot \ - --set container_image=docker:// \ - --set "disable_http_server=true" \ - --set "disable_netboot=true" \ - --cloud-config /config.yaml \ - --set "state_dir=/tmp/auroraboot" -``` - -### Offline ISO build from container images - -Build the custom ISO with the cloud config: - -```bash -docker run -v "$PWD"/config.yaml:/config.yaml \ - -v "$PWD"/build:/tmp/auroraboot \ - --rm -ti quay.io/kairos/auroraboot \ - --set container_image=quay.io/kairos/core-rockylinux:v1.5.0 \ - --set "disable_http_server=true" \ - --set "disable_netboot=true" \ - --cloud-config /config.yaml \ - --set "state_dir=/tmp/auroraboot" -``` - -### Override GRUB config file - -It is possible to override the default GRUB config file of the ISO by creating a directory that -contains the files that we want to add or replace in it. - -For example, to override the GRUB config file: - -```bash -mkdir -p data/boot/grub2 -# You can replace this step with your own grub config. This GRUB configuration is the boot menu of the ISO -wget https://raw.githubusercontent.com/kairos-io/kairos/master/overlay/files-iso/boot/grub2/grub.cfg -O data/boot/grub2/grub.cfg - -docker run -v "$PWD"/config.yaml:/config.yaml \ - -v "$PWD"/data:/tmp/data \ - -v "$PWD"/build:/tmp/auroraboot \ - --rm -ti quay.io/kairos/auroraboot \ - --set container_image=quay.io/kairos/core-rockylinux:v1.5.0 \ - --set "disable_http_server=true" \ - --set "disable_netboot=true" \ - --cloud-config /config.yaml \ - --set "state_dir=/tmp/auroraboot" \ - --set "iso.data=/tmp/data" -``` - -### Prepare ISO for Airgap installations - -See the [Airgap example](/docs/examples/airgap) in the [examples section](/docs/examples). - -### Netboot with core images from Github releases - -```bash -docker run -v "$PWD"/config.yaml:/config.yaml --rm -ti --net host quay.io/kairos/auroraboot \ - --set "artifact_version=v1.5.0" \ - --set "release_version=v1.5.0" \ - --set "flavor=rockylinux" \ - --set repository="kairos-io/kairos" \ - --cloud-config /config.yaml -``` - -### Netboot with k3s images from Github releases - -```bash -docker run -v "$PWD"/config.yaml:/config.yaml --rm -ti --net host quay.io/kairos/auroraboot \ - --set "artifact_version=v1.5.1-k3sv1.21.14+k3s1" \ - --set "release_version=v1.5.1" \ - --set "flavor=opensuse-leap" \ - --set "repository=kairos-io/provider-kairos" \ - --cloud-config /config.yaml -``` - -### Netboot from container images - -```bash -docker run -v "$PWD"/config.yaml:/config.yaml --rm -ti --net host quay.io/kairos/auroraboot \ - --set container_image=quay.io/kairos/core-rockylinux:v1.5.0 - --cloud-config /config.yaml -``` - -### Use the config file - -Write down an aurora config file as `aurora.yaml`: -```yaml -container_image: "quay.io/kairos/core-rockylinux:v1.5.0" - -cloud_config: | - #cloud-config - - install: - auto: true - device: "auto" - reboot: true - - # Define the user accounts on the node. - users: - - name: "kairos" # The username for the user. - passwd: "kairos" # The password for the user. - ssh_authorized_keys: # A list of SSH keys to add to the user's authorized keys. - # - github:mudler # A key from the user's GitHub account. - # - "ssh-rsa AAA..." # A raw SSH key. -``` - -And then run: - -```bash -docker run -v "$PWD"/aurora.yaml:/aurora.yaml --rm -ti --net host quay.io/kairos/auroraboot /aurora.yaml -``` diff --git a/docs/content/en/docs/Reference/build-from-scratch.md b/docs/content/en/docs/Reference/build-from-scratch.md deleted file mode 100644 index 43ceabaf5..000000000 --- a/docs/content/en/docs/Reference/build-from-scratch.md +++ /dev/null @@ -1,181 +0,0 @@ ---- -title: "Build Kairos from scratch" -linkTitle: "Build Kairos from scratch" -weight: 5 -description: > - This article shows how to bring your own image with Kairos, and build a Kairos derivative from scratch using base container images from popular distributions such as Ubuntu, Fedora, openSUSE, etc. ---- - -{{% alert title="Note" %}} -By default, Core and Standard Kairos images are pre-configured, optimized and maintained by the Kairos team, meeting most use cases. However, if you're an advanced user interested in creating your own derivative or building new flavors for Kairos core images, this section is reserved just for you. - -While the process of building these images is still a work in progress, it's already usable for general consumption. You can follow our development efforts in the [factory epic](https://github.com/kairos-io/kairos/issues/116). For instance, we are currently working on adding features like [conformance tests](https://github.com/kairos-io/kairos/issues/958) to enable users to test images built with this process, ensuring their correctness before attempting to boot the system. -{{% /alert %}} - -Kairos enables the creation of a distribution based on any base OS image that satisfies the Kairos model and contract. Essentially, every OS is treated solely as a collection of packages, and upgrades and operations are managed by Kairos components, which abstract the management model. - -In practical terms, upgrades are not carried out by the package manager of the OS. Instead, the `kairos-agent` handles upgrades through container images. All installation and upgrades are delivered exclusively through container images. These images are overlayed at boot time, which means there is no additional runtime overhead, as no container engine is required for booting the OS. - -The Kairos framework is an abstract layer between the OS and the management interface. It follows an atomic A/B approach, which can be controlled through Kubernetes, the CLI, or a declarative model. - -The Kairos contract is straightforward: the OS container image must include everything required for booting, from the kernel to the init system. - -The contract has several advantages: - -- Delegation of package maintenance, CVE, and security fixes to the OS layer -- Easy issuance of upgrades to container images by chaining Dockerfiles or manually committing changes to the image. See also [Customizing](/docs/advanced/customizing). -- Clear separation of concerns: the OS provides the booting bits and packages necessary for the OS to function, while Kairos provides the operational framework for handling the node's lifecycle and immutability interface. -- Support for long-term maintenance: each framework image allows conversion of any OS to the given Kairos framework version, potentially enabling maintenance for as long as the base OS support model allows. - -This document outlines the steps for making any base image fully bootable with the Kairos framework. The steps include: - -- Building a container image - - Selecting a base image from the supported OS family (although it should work with any distro) - - Installing the required packages from the package manager of the chosen OS - - Building the initramfs -- Building an offline bootable ISO or netbooting the container image. - -## Prerequisites - -To follow the steps below, you'll need to have Docker or a container engine installed on your local machine. Additionally, note that the steps have been tested on Linux but should also work in other environments. If you encounter any issues, please feel free to open up issues and help us improve the Documentation! - -## Build a container image - -To build the container image, follow these steps: - -1. Create a new directory for the image and write a Dockerfile inside it. The Dockerfile will contain the instructions for building the image: - -```Dockerfile -FROM fedora:36 - -# Install any package wanted here -# Note we need to install _at least_ the minimum required packages for Kairos to work: -# - An init system (systemd) -# - Grub -# - kernel/initramfs -RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf - -RUN dnf install -y \ - audit \ - coreutils \ - curl \ - device-mapper \ - dosfstools \ - dracut \ - dracut-live \ - dracut-network \ - dracut-squash \ - e2fsprogs \ - efibootmgr \ - gawk \ - gdisk \ - grub2 \ - grub2-efi-x64 \ - grub2-efi-x64-modules \ - grub2-pc \ - haveged \ - kernel \ - kernel-modules \ - kernel-modules-extra \ - livecd-tools \ - nano \ - NetworkManager \ - openssh-server \ - parted \ - polkit \ - rsync \ - shim-x64 \ - squashfs-tools \ - sudo \ - systemd \ - systemd-networkd \ - systemd-resolved \ - tar \ - which \ - && dnf clean all - -RUN mkdir -p /run/lock -RUN touch /usr/libexec/.keep - -# Copy the Kairos framework files. We use master builds here for fedora. See https://quay.io/repository/kairos/framework?tab=tags for a list -COPY --from=quay.io/kairos/framework:master_fedora / / - -# Activate Kairos services -RUN systemctl enable cos-setup-reconcile.timer && \ - systemctl enable cos-setup-fs.service && \ - systemctl enable cos-setup-boot.service && \ - systemctl enable cos-setup-network.service - -## Generate initrd -RUN kernel=$(ls /boot/vmlinuz-* | head -n1) && \ - ln -sf "${kernel#/boot/}" /boot/vmlinuz -RUN kernel=$(ls /lib/modules | head -n1) && \ - dracut -v -N -f "/boot/initrd-${kernel}" "${kernel}" && \ - ln -sf "initrd-${kernel}" /boot/initrd && depmod -a "${kernel}" -RUN rm -rf /boot/initramfs-* -``` - -In the Dockerfile, note the following: - -- The base image we're using is fedora. However, you could also base your image on other distributions. See [the Kairos official images](https://github.com/kairos-io/kairos/tree/master/images) for an example. -- We're installing a set of packages, including `rsync`, `grub`, `systemd`, `kernel`, and we're generating the initramfs inside the image. -- We're copying the Kairos framework image file to the root of the container. Choose the framework image that closely matches your setup. You can find the framework images published here: https://quay.io/repository/kairos/framework?tab=tags - -3. Now build the image with: - -```bash -docker build -t test-byoi . -``` - -## Build bootable assets - -Once the container image is built, we can proceed directly to creating an ISO or netboot it using [AuroraBoot](/docs/reference/auroraboot). We can use AuroraBoot to handle the ISO build process and even attach a default cloud config if desired. Here's an example for both scenarios: - -{{< tabpane text=true >}} -{{% tab header="ISO" %}} - -We can use [AuroraBoot](/docs/reference/auroraboot) to handle the the ISO build process and optionally attach it a default cloud config, for example: - -```bash -docker run -v "$PWD"/build:/tmp/auroraboot \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --rm -ti quay.io/kairos/auroraboot:v0.2.2 \ - --set container_image=docker://test-byoi \ - --set "disable_http_server=true" \ - --set "disable_netboot=true" \ - --set "state_dir=/tmp/auroraboot" -# 2:45PM INF Pulling container image 'test-byoi' to '/tmp/auroraboot/temp-rootfs' (local: true) -# 2:45PM INF Generating iso 'kairos' from '/tmp/auroraboot/temp-rootfs' to '/tmp/auroraboot/iso' -# $ sudo ls -liah build/iso -# total 449M -# 35142520 drwx------ 2 root root 4.0K Mar 7 15:46 . -# 35142517 drwxr-xr-x 5 root root 4.0K Mar 7 15:42 .. -# 35142521 -rw-r--r-- 1 root root 0 Mar 7 15:45 config.yaml -# 35138094 -rw-r--r-- 1 root root 449M Mar 7 15:46 kairos.iso -``` - -This will generate an ISO named kairos.iso which will be located at `build/iso/`. You can use either `BalenaEtcher` or `dd` to flash this ISO to a USB stick. Additionally, QEMU can be used to test the ISO: - -```bash -qemu-system-x86_64 -m 2048 -drive if=virtio,media=disk,file=build/iso/kairos.iso -``` - -{{% /tab %}} - -{{% tab header="Netboot" %}} - -To netboot, we can also use [AuroraBoot](/docs/reference/auroraboot) to handle the process, or refer to [Netboot](/docs/installation/netboot). Here's an example: - -```bash -docker run -v --net host \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --rm -ti quay.io/kairos/auroraboot:v0.2.2 \ - --set container_image=docker://test-byoi \ - --set "disable_http_server=true" \ - --set "netboot.cmdline=rd.neednet=1 ip=dhcp rd.cos.disable netboot nodepair.enable console=tty0 selinux=0" -``` - -{{% /tab %}} -{{< /tabpane >}} - -This example is available in the `examples/byoi/fedora` directory of the [Kairos repository](https://github.com/kairos-io/kairos/tree/master/examples/byoi/fedora), where you can run `build.sh` to reproduce it. \ No newline at end of file diff --git a/docs/content/en/docs/Reference/cli.md b/docs/content/en/docs/Reference/cli.md deleted file mode 100644 index ab84e683c..000000000 --- a/docs/content/en/docs/Reference/cli.md +++ /dev/null @@ -1,151 +0,0 @@ ---- -title: "CLI" -linkTitle: "CLI" -weight: 3 -date: 2022-11-13 -description: > ---- - -A CLI is provided as part of releases associated to each Kairos version. - -The CLI can be used from an external machine to generate network tokens and pair nodes on first-boot. - -``` -./kairos --help -NAME: - kairos - kairos (register|install) - -USAGE: - [global options] command [command options] [arguments...] - -VERSION: - 0.1 - -DESCRIPTION: - kairos registers and installs kairos boxes - -AUTHOR: - Ettore Di Giacinto - -COMMANDS: - register - create-config, c - generate-token, g - setup, s - get-kubeconfig - install, i - help, h Shows a list of commands or help for one command -``` - -## `create-config` - -Generates a new Kairos configuration file which can be used as `cloud-init`, with a new unique network token: - -``` -$ ./kairos create-config -kairos: - network_token: b3RwOgogIGRodDoKICAgIGludGVydmFsOiA5MjIzMzcyMDM2ODU0Nzc1ODA3CiAgICBrZXk6IEVCMzJJMlNXTjJCNFBHNEtCWTNBUVBBS0FWRTY0Q0VLVUlDTktTUFVWVU5BWTM0QklEQ0EKICAgIGxlbmd0aDogMzIKICBjcnlwdG86CiAgICBpbnRlcnZhbDogOTIyMzM3MjAzNjg1NDc3NTgwNwogICAga2V5OiBDMk1RRk5DWEFVRElPWjVHM1pZUUIzVEVHTzVXVEdQR1pZSEVQQkY3SFEyVUROUlZCTkxRCiAgICBsZW5ndGg6IDMyCnJvb206IGp6Q29kQVVOWUZSUklQU3JISmx4d1BVUnVxTGJQQnh4CnJlbmRlenZvdXM6IG5NckRCbllyVVBMdnFPV0Z2dWZvTktXek1adEJIRmpzCm1kbnM6IGpQUUhIbVZza2x6V29xbWNkeVlnbVhMSVFjTE1HUFN6Cm1heF9tZXNzYWdlX3NpemU6IDIwOTcxNTIwCg== - offline: false - reboot: false - device: "" - poweroff: false -``` - -Now you can use this in your configuration file to create new Kairos nodes: - -```yaml -kairos: - network_token: b3RwOgogIGRodDoKICAgIGludGVydmFsOiA5MjIzMzcyMDM2ODU0Nzc1ODA3CiAgICBrZXk6IEVCMzJJMlNXTjJCNFBHNEtCWTNBUVBBS0FWRTY0Q0VLVUlDTktTUFVWVU5BWTM0QklEQ0EKICAgIGxlbmd0aDogMzIKICBjcnlwdG86CiAgICBpbnRlcnZhbDogOTIyMzM3MjAzNjg1NDc3NTgwNwogICAga2V5OiBDMk1RRk5DWEFVRElPWjVHM1pZUUIzVEVHTzVXVEdQR1pZSEVQQkY3SFEyVUROUlZCTkxRCiAgICBsZW5ndGg6IDMyCnJvb206IGp6Q29kQVVOWUZSUklQU3JISmx4d1BVUnVxTGJQQnh4CnJlbmRlenZvdXM6IG5NckRCbllyVVBMdnFPV0Z2dWZvTktXek1adEJIRmpzCm1kbnM6IGpQUUhIbVZza2x6V29xbWNkeVlnbVhMSVFjTE1HUFN6Cm1heF9tZXNzYWdlX3NpemU6IDIwOTcxNTIwCg== - offline: false - reboot: false - device: "" - poweroff: false - -stages: - network: - - name: "Setup users" - authorized_keys: - kairos: - - github:yourhandle! -``` - -## `generate-token` - -Generates a new network token which can be used in a configuration file: - -``` -$ ./kairos generate-token -b3RwOgogIGRodDoKICAgIGludGVydmFsOiA5MjIzMzcyMDM2ODU0Nzc1ODA3CiAgICBrZXk6IFhMMjRYUk1MTlFOQ1pJQTU0SVFLQ1laMk83SENQWEFBU1ZKN0tZSTQ3MzVaUkpKSktRSEEKICAgIGxlbmd0aDogMzIKICBjcnlwdG86CiAgICBpbnRlcnZhbDogOTIyMzM3MjAzNjg1NDc3NTgwNwogICAga2V5OiBMR1dMWFBTUllaU0ZERDdOT0pBNzdKV0ZWQjRHVkZBMjJIWlZPWU1VT0lNSFVYNFZXUURRCiAgICBsZW5ndGg6IDMyCnJvb206IFRtcUt5VnFHQ1ZZam9TRm9CTEVNRGVEdmJzelBkVEdoCnJlbmRlenZvdXM6IGttb3J4Q21sY2NjVVppWmdkSW5xTERvTGJtS3ZGdm9mCm1kbnM6IEZkWVdQc2R4aHdvWHZlb0VzSXNnVHRXbEJUbE9IVHJmCm1heF9tZXNzYWdlX3NpemU6IDIwOTcxNTIwCg== -``` - -And now: - -```yaml -kairos: - network_token: b3RwOgogIGRodDoKICAgIGludGVydmFsOiA5MjIzMzcyMDM2ODU0Nzc1ODA3CiAgICBrZXk6IFhMMjRYUk1MTlFOQ1pJQTU0SVFLQ1laMk83SENQWEFBU1ZKN0tZSTQ3MzVaUkpKSktRSEEKICAgIGxlbmd0aDogMzIKICBjcnlwdG86CiAgICBpbnRlcnZhbDogOTIyMzM3MjAzNjg1NDc3NTgwNwogICAga2V5OiBMR1dMWFBTUllaU0ZERDdOT0pBNzdKV0ZWQjRHVkZBMjJIWlZPWU1VT0lNSFVYNFZXUURRCiAgICBsZW5ndGg6IDMyCnJvb206IFRtcUt5VnFHQ1ZZam9TRm9CTEVNRGVEdmJzelBkVEdoCnJlbmRlenZvdXM6IGttb3J4Q21sY2NjVVppWmdkSW5xTERvTGJtS3ZGdm9mCm1kbnM6IEZkWVdQc2R4aHdvWHZlb0VzSXNnVHRXbEJUbE9IVHJmCm1heF9tZXNzYWdlX3NpemU6IDIwOTcxNTIwCg== - offline: false - reboot: false - device: "" - poweroff: false - -stages: - network: - - name: "Setup users" - authorized_keys: - kairos: - - github:yourhandle! -``` - -## `register` - -The **register** command can be used to register and drive installation of nodes via QR code with a `cloud-init` config file (with `--config`). - -``` -NAME: - register - - -USAGE: - register [command options] [arguments...] - -OPTIONS: - --config value - --device value - --reboot - --poweroff -``` - -When booting Kairos via ISO, the boot process ends up in displaying a QR code which can be parsed by `kairos register` from another machine. - -### Taking a screenshot - -`register` by default takes a screenshot and tries to find a QR code in it: - -``` -kairos register -``` - -### Providing a QR code image/screenshot manually - -It can be also be specified an image: - -``` -kairos register -``` - -After the pairing is done, the node will start installation with the provided options. - -A `--device` and a `--config` file are required in order to have a functional installation. - -## `bridge` - -Connect to the nodes in the VPN P2P network by creating a tun device on the host. - -It needs a `--network-token`(`$NETWORK_TOKEN`) argument and exposes an API endpoint available at [localhost:8080](http://localhost:8080) to monitor the network status. - -## `install` - -Is called by Kairos nodes on boot and not meant to be used manually. It kicks in the installation and the QR pairing process. - -## `setup` - -Is called by Kairos nodes on boot and not meant to be used manually. It prepares `edgevpn` and K3s bootstrapping the node and the VPN. diff --git a/docs/content/en/docs/Reference/configuration.md b/docs/content/en/docs/Reference/configuration.md deleted file mode 100644 index 64bb74cda..000000000 --- a/docs/content/en/docs/Reference/configuration.md +++ /dev/null @@ -1,1101 +0,0 @@ ---- -title: "Configuration" -linkTitle: "Configuration" -weight: 2 -date: 2022-11-13 -description: > ---- - -Welcome to the Kairos configuration reference page. This page provides details on the fields available in the YAML file used for installing Kairos, a Linux distribution focused on running Kubernetes. This file, written in cloud-config format, allows you to enable Kairos features, configure k3s, and set various other options. - -The structure of the configuration file is as follows: - -```yaml -#cloud-config - -# Additional system users -users: -- name: "kairos" - passwd: "kairos" - lock_passwd: true - groups: "admin" - ssh_authorized_keys: - # - github:mudler - -# The install block is to drive automatic installations without user interaction. -install: - # Device for automated installs - device: "/dev/sda" - # Reboot after installation - reboot: true - # Power off after installation - poweroff: true - # Set to true when installing without Pairing - auto: true - # Use a different container image for the installation - image: "docker:.." - # Add bundles in runtime - bundles: - - ... - # Set grub options - grub_options: - # additional Kernel option cmdline to apply - extra_cmdline: "config_url=http://" - # Same, just for active - extra_active_cmdline: "" - # Same, just for passive - extra_passive_cmdline: "" - # Change GRUB menu entry - default_menu_entry: "" - # Environmental variable to set to the installer calls - env: - - foo=bar - # custom user mounts - # bind mounts, can be read and modified, changes persist reboots - bind_mounts: - - /mnt/bind1 - - /mnt/bind2 - # ephemeral mounts, can be read and modified, changed are discarded at reboot - ephemeral_mounts: - -k3s: - # Additional env/args for k3s server instances - env: - K3S_RESOLV_CONF: "" - K3S_DATASTORE_ENDPOINT: "mysql://username:password@tcp(hostname:3306)/database-name" - args: - - --node-label "" - - --data-dir "" - # Enabling below it replaces args/env entirely - # replace_env: true - # replace_args: true - -k3s-agent: - # Additional env/args for k3s agent instances - env: - K3S_NODE_NAME: "foo" - args: - - --private-registry "..." - # Enabling below it replaces args/env entirely - # replace_env: true - # replace_args: true - -# The p2p block enables the p2p full-mesh functionalities. -# To disable, don't specify one. -p2p: - # Manually set node role. Available: master, worker. Defaults auto (none). This is available - role: "master" - # User defined network-id. Can be used to have multiple clusters in the same network - network_id: "dev" - # Enable embedded DNS See also: https://mudler.github.io/edgevpn/docs/concepts/overview/dns/ - dns: true - # Disabling DHT makes co-ordination to discover nodes only in the local network - disable_dht: true #Enabled by default - # Configures a VPN for the cluster nodes - vpn: - create: false # defaults to true - use: false # defaults to true - env: - # EdgeVPN environment options - DHCP: "true" - # Disable DHT (for airgap) - EDGEVPNDHT: "false" - EDGEVPNMAXCONNS: "200" - # If DHCP is false, it's required to be given a specific node IP. Can be arbitrary - ADDRESS: "10.2.0.30/24" - # See all EDGEVPN options: - # - https://github.com/mudler/edgevpn/blob/master/cmd/util.go#L33 - # - https://github.com/mudler/edgevpn/blob/master/cmd/main.go#L48 - # Automatic cluster deployment configuration - auto: - # Enables Automatic node configuration (self-coordination) - # for role assignment - enable: true - # HA enables automatic HA roles assignment. - # A master cluster init is always required, - # Any additional master_node is configured as part of the - # HA control plane. - # If auto is disabled, HA has no effect. - ha: - # Enables HA control-plane - enable: true - # Number of HA additional master nodes. - # A master node is always required for creating the cluster and is implied. - # The setting below adds 2 additional master nodes, for a total of 3. - master_nodes: 2 - # Use an External database for the HA control plane - external_db: "external-db-string" - # network_token is the shared secret used by the nodes to co-ordinate with p2p - network_token: "YOUR_TOKEN_GOES_HERE" - -## Sets the Elastic IP used in KubeVIP. Only valid with p2p -kubevip: - eip: "192.168.1.110" - # Specify a manifest URL for KubeVIP. Empty uses default - manifest_url: "" - # Enables KubeVIP - enable: true - # Specifies a KubeVIP Interface - interface: "ens18" - -# Additional cloud init syntax can be used here. -# See `stages` below. -stages: - network: - - name: "Setup users" - authorized_keys: - kairos: - - github:mudler - -# Various options. -# Those could apply to install, or other phases as well -options: - # Specify an alternative image to use during installation - system.uri: "" - # Specify an alternative recovery image to use during installation - recovery-system.uri: "" - # Just set it to eject the cd after install - eject-cd: "" - -# Standard cloud-init syntax, see: https://github.com/mudler/yip/tree/e688612df3b6f24dba8102f63a76e48db49606b2#compatibility-with-cloud-init-format -growpart: - devices: ['/'] - - -runcmd: -- foo -hostname: "bar" -write_files: -- encoding: b64 - content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4 - path: /foo/bar - permissions: "0644" - owner: "bar" -``` - -The `p2p` block is used to enable the p2p full-mesh functionalities of Kairos. If you do not want to use these functionalities, simply don't specify a kairos block in your configuration file. - -Inside the `p2p` block, you can specify the network_token field, which is used to establish the p2p full meshed network. If you do not want to use the full-mesh functionalities, don't specify a network_token value. - -The role field allows you to manually set the node role for your Kairos installation. The available options are `master` and `worker`, and the default value is auto (which means no role is set). - -The `network_id` field allows you to set a user-defined network ID, which can be used to have multiple Kairos clusters on the same network. - -Finally, the `dns` field allows you to enable embedded DNS for Kairos. For more information on DNS in Kairos, see the link provided in the YAML code above. - -That's a brief overview of the structure and fields available in the Kairos configuration file. For more detailed information on how to use these fields, see the examples and explanations provided in the sections below. - - -## Syntax - -Kairos supports a portion of the standard [cloud-init](https://cloud-init.io/) syntax, and the extended syntax which is based on [yip](https://github.com/mudler/yip). - -Examples using the extended notation for running K3s as agent or server can be found in the [examples](https://github.com/kairos-io/kairos/tree/master/examples) directory of the Kairos repository. - -Here's an example that shows how to set up DNS at the [boot stage](/docs/architecture/cloud-init) using the extended syntax: - -```yaml -#cloud-config - -stages: - boot: - - name: "DNS settings" - dns: - path: /etc/resolv.conf - nameservers: - - 8.8.8.8 -``` - -{{% alert title="Note" %}} - -Kairos does not use [cloud-init](https://cloud-init.io/). [yip](https://github.com/mudler/yip) was created with the goal of being distro agnostic, and does not use Bash at all (with the exception of systemd configurations, which are assumed to be available). This makes it possible to run yip on minimal Linux distros that have been built from scratch. - -The rationale behind using yip instead of cloud-init is that it allows Kairos to have very minimal requirements. The cloud-init implementation has dependencies, while yip does not, which keeps the dependency tree small. There is also a CoreOS implementation of cloud-init, but it makes assumptions about the layout of the system that are not always applicable to Kairos, making it less portable. - -{{% /alert %}} - - -The extended syntax can also be used to pass commands through Kernel boot parameters. See the examples below for more details. - -### Test your cloud configs - -Writing YAML files can be a tedious process, and it's easy to make syntax or indentation errors. To make sure your configuration is correct, you can use the cloud-init commands to test your YAML files locally in a container. - -Here's an example of how to test your configuration using a Docker container: - -```bash -# List the YAML files in your current directory -$ ls -liah -total 32K -38548066 drwxr-xr-x 2 mudler mudler 4.0K Nov 12 19:21 . -38548063 drwxr-xr-x 3 mudler mudler 4.0K Nov 12 19:21 .. -38548158 -rw-r--r-- 1 mudler mudler 1.4K Nov 12 19:21 00_rootfs.yaml -38548159 -rw-r--r-- 1 mudler mudler 1.1K Nov 12 19:21 06_recovery.yaml -38552350 -rw-r--r-- 1 mudler mudler 608 Nov 12 19:21 07_live.yaml -38552420 -rw-r--r-- 1 mudler mudler 5.3K Nov 12 19:21 08_boot_assessment.yaml -38553235 -rw-r--r-- 1 mudler mudler 380 Nov 12 19:21 09_services.yaml - -# Run the cloud-init command on your YAML files in a Docker container -$ docker run -ti -v $PWD:/test --entrypoint /usr/bin/elemental --rm {{< registryURL >}}/core-{{< flavor >}} cloud-init /test - -# Output from the cloud-init command -INFO[2022-11-18T08:51:33Z] Starting elemental version ... -INFO[2022-11-18T08:51:33Z] Running stage: default -INFO[2022-11-18T08:51:33Z] Executing /test/00_rootfs.yaml -INFO[2022-11-18T08:51:33Z] Executing /test/06_recovery.yaml -INFO[2022-11-18T08:51:33Z] Executing /test/07_live.yaml -INFO[2022-11-18T08:51:33Z] Executing /test/08_boot_assessment.yaml -INFO[2022-11-18T08:51:33Z] Executing /test/09_services.yaml -INFO[2022-11-18T08:51:33Z] Done executing stage 'default' -``` - -By default, the cloud-init command runs the `default` stage, which doesn't actually map to any specific stage in your YAML files. To test a different stage, you can use the `--stage` (`-s`) option, like this: - -```bash -# Run the cloud-init command on your YAML files in a Docker container, and specify the "initramfs" stage -$ docker run -ti -v $PWD:/test --entrypoint /usr/bin/elemental --rm {{< registryURL >}}/core-{{< flavor >}} cloud-init -s initramfs /test -``` - -You can also test individual YAML files by piping them to the cloud-init command, like this: -```bash -cat <}}/core-{{< flavor >}} cloud-init -s test - -#cloud-config - -stages: - test: - - commands: - - echo "test" -EOF - -# INFO[2022-11-18T08:53:45Z] Starting elemental version v0.0.1 -# INFO[2022-11-18T08:53:45Z] Running stage: test -# INFO[2022-11-18T08:53:45Z] Executing stages: -# test: -# - commands: -# - echo "test" -# INFO[2022-11-18T08:53:45Z] Applying '' for stage 'test'. Total stages: 1 -# INFO[2022-11-18T08:53:45Z] Processing stage step ''. ( commands: 1, files: 0, ... ) -# INFO[2022-11-18T08:53:45Z] Command output: test -# INFO[2022-11-18T08:53:45Z] Stage 'test'. Defined stages: 1. Errors: false -# INFO[2022-11-18T08:53:45Z] Done executing stage 'test' -``` - -### Validate Your Cloud Config - -{{% alert title="Note" %}} - -Validation of configuration is available on Kairos [v1.6.0-rc1](https://github.com/kairos-io/kairos/releases/tag/v1.6.0-rc1) and later. If you're interested in the validation rules or want to build a tool based on it, you can access them online via `https://kairos.io/RELEASE/cloud-config.json` e.g. [v1.6.0 cloud-config.json](https://kairos.io/v1.6.0/cloud-config.json) - -{{% /alert %}} - -You have two options to validate your Cloud Config, one is with the Kairos command line, and the other with the Web UI. - -#### Configuration Validation via the Kairos Command Line - -To validate a configuration using the command line, we have introduced the `validate` command. As an argument you need to pass a URL or local file to be validated, e.g.: - -If you had the following `cloud-config.yaml` in the current working directory - -```yaml -#cloud-config -users: - - name: 007 -``` - -You could validate it as follows - -```sh -kairos validate ./cloud-config.yaml -jsonschema: '/users/0/name' does not validate with file:///home/mauro/workspace/kairos/schema.json#/properties/users/items/$ref/properties/name/type: expected string, but got number -``` - -#### Configuration Validation via Web UI - -The validation in the Web UI is automatic, all you need to do is copy/paste or type your configuration on the input. - -![Schema Validation Preview](/images/schema-validation-preview.gif) - - -### Using templates - -Fields in the Kairos cloud-init configuration can be templated, which allows for dynamic configuration. Node information is retrieved using the [sysinfo](https://github.com/zcalusic/sysinfo#sample-output) library, and can be templated in the `commands`, `file`, and `entity` fields. - -Here's an example of how you can use templating in your Kairos configuration: - -```yaml -#cloud-config - -stages: - foo: - - name: "echo" - commands: - - echo "{{.Values.node.hostname}}" -``` -In addition to standard templating, [sprig functions](http://masterminds.github.io/sprig/) are also available for use in your Kairos configuration. - -#### Automatic Hostname at scale - -You can also use templating to automatically generate hostnames for a set of machines. For example, if you have a single `cloud-init` file that you want to use for multiple machines, you can use the machine ID (which is generated for each host) to automatically set the hostname for each machine. - -Here's an example of how you can do this: - -```yaml -#cloud-config - -stages: - initramfs: - - name: "Setup hostname" - hostname: "node-{{ trunc 4 .MachineID }}" -``` - -This will set the hostname for each machine based on the first 4 characters of the machine ID. For example, if the machine ID for a particular machine is `abcdef123456`, the hostname for that machine will be set to `node-abcd`. - -### K3s settings - -The `k3s` and `k3s-agent` blocks in the Kairos configuration file allow you to customize the environment and argument settings for K3s. - -Here's an example of how to use these blocks in your Kairos configuration: - -{{< tabpane text=true right=true >}} -{{% tab header="server" %}} -```yaml -k3s: - enabled: true - # Additional env/args for k3s server instances - env: - K3S_RESOLV_CONF: "" - K3S_DATASTORE_ENDPOINT: "" - args: - - ... -``` -{{% /tab %}} -{{% tab header="agent" %}} -```yaml -k3s-agent: - enabled: true - # Additional env/args for k3s server instances - env: - K3S_RESOLV_CONF: "" - K3S_DATASTORE_ENDPOINT: "" - args: - - -``` -{{% /tab %}} -{{< /tabpane >}} - -For more examples of how to configure K3s manually, see the [examples](/docs/examples) section or [HA](/docs/advanced/ha). - -### Grub options - -The `install.grub_options` field in the Kairos configuration file allows you to set key/value pairs for GRUB options that will be set in the GRUB environment after installation. - -Here's an example of how you can use this field to set the `panic=0` boot argument: - -```yaml -#cloud-config - -install: - grub_options: - extra_cmdline: "panic=0" -``` - -The table below lists all the available options for the `install.grub_options` field: - -| Variable | Description | -| ---------------------- | ------------------------------------------------------- | -| next_entry | Set the next reboot entry | -| saved_entry | Set the default boot entry | -| default_menu_entry | Set the name entries on the GRUB menu | -| extra_active_cmdline | Set additional boot commands when booting into active | -| extra_passive_cmdline | Set additional boot commands when booting into passive | -| extra_recovery_cmdline | Set additional boot commands when booting into recovery | -| extra_cmdline | Set additional boot commands for all entries | -| default_fallback | Sets default fallback logic | - -## Kubernetes manifests - -The `k3s` distribution of Kubernetes allows you to automatically deploy Helm charts or Kubernetes resources after deployment. - -Here's an example of how you can use the `k3s` configuration file to deploy Fleet out of the box: - -```yaml -name: "Deploy fleet out of the box" -stages: - boot: - - name: "Copy fleet deployment files" - files: - - path: /var/lib/rancher/k3s/server/manifests/fleet-config.yaml - content: | - apiVersion: v1 - kind: Namespace - metadata: - name: cattle-system - --- - apiVersion: helm.cattle.io/v1 - kind: HelmChart - metadata: - name: fleet-crd - namespace: cattle-system - spec: - chart: https://github.com/rancher/fleet/releases/download/v0.3.8/fleet-crd-0.3.8.tgz - --- - apiVersion: helm.cattle.io/v1 - kind: HelmChart - metadata: - name: fleet - namespace: cattle-system - spec: - chart: https://github.com/rancher/fleet/releases/download/v0.3.8/fleet-0.3.8.tgz -``` - -This configuration will automatically deploy the Fleet Helm chart in the cattle-system namespace after the deployment of `k3s` using the extended syntax. - -## Kernel boot parameters - -All the configurations can be issued via Kernel boot parameters, for instance, consider to add an user from the boot menu: - -`stages.boot[0].authorized_keys.root[0]=github:mudler` - -Or to either load a config url from network: - -`config_url=http://...` - -Usually secret gists are used to share such config files. - -## Additional users - -Kairos comes with the `kairos` user pre-configured, however, it is possible to configure additional users to the system via the cloud-init config mechanism - -### Add a user during first-install - -Consider the following example cloud-config, which adds the `testuser` user to the system with admin access: - -```yaml -#cloud-config -install: - device: /dev/sda -k3s: - enabled: true - -users: -- name: "kairos" - passwd: "kairos" - ssh_authorized_keys: - - github:mudler -- name: "testuser" - passwd: "testuser" - ssh_authorized_keys: - - github:mudler - groups: - - "admin" -``` - -### Add a user to an existing install - -To add an user to an existing installation you can simply add a `/oem` file for the new user. For instance, consider the following: -```yaml -stages: - initramfs: - - name: "Set user and password" - users: - testuser: - groups: - - "admin" - passwd: "mypassword" - shell: /bin/bash - homedir: "/home/testuser" -``` - -This configuration can be either manually copied over, or can be propagated also via Kubernetes using the system upgrade controller. See [the after-install](/docs/advanced/after-install) section for an example. - -```bash -❯ ssh testuser@192.168.1.238 -testuser@192.168.1.238's password: -Welcome to kairos! - -Refer to https://kairos.io for documentation. -localhost:~$ sudo su - -localhost:~# whoami -root -localhost:~# exit -localhost:~$ whoami -testuser -localhost:~$ -``` - -## P2P configuration - -P2P functionalities are experimental Kairos features and disabled by default. In order to enable them, just use the `p2p` configuration block. - -### `p2p.network_token` - -This defines the network token used by peers to join the p2p virtual private network. You can generate it with the Kairos CLI with `kairos generate-token`. Check out [the P2P section](/docs/installation/p2p) for more instructions. - -### `p2p.role` - -Define a role for the node. Accepted: `worker`, `master`. Currently only one master is supported. - -### `p2p.network_id` - -Define a custom ID for the Kubernetes cluster. This can be used to create multiple clusters in the same network segment by specifying the same id across nodes with the same network token. Accepted: any string. - -### `p2p.dns` - -When the `p2p.dns` is set to `true` the embedded DNS is configured on the node. This allows to propagate custom records to the nodes by using the blockchain DNS server. For example, this is assuming `kairos bridge` is running in a separate terminal: - -```bash -curl -X POST http://localhost:8080/api/dns --header "Content-Type: application/json" -d '{ "Regex": "foo.bar", "Records": { "A": "2.2.2.2" } }' -``` - -It will add the `foo.bar` domain with `2.2.2.2` as `A` response. - -Every node with DNS enabled will be able to resolve the domain after the domain is correctly announced. - -You can check out the DNS in the [DNS page in the API](http://localhost:8080/dns.html), see also the [EdgeVPN docs](https://mudler.github.io/edgevpn/docs/concepts/overview/dns/). - -Furthermore, it is possible to tweak the DNS server which are used to forward requests for domain listed outside, and as well, it's possible to lock down resolving only to nodes in the blockchain, by customizing the configuration file: - -```yaml -#cloud-config -p2p: - network_token: "...." - # Enable embedded DNS See also: https://mudler.github.io/edgevpn/docs/concepts/overview/dns/ - dns: true - vpn: - env: - # Disable DNS forwarding - DNSFORWARD: "false" - # Set cache size - DNSCACHESIZE: "200" - # Set DNS forward server - DNSFORWARDSERVER: "8.8.8.8:53" -``` - -## Stages - -The `stages` key is a map that allows to execute blocks of cloud-init directives during the lifecycle of the node [stages](/docs/architecture/cloud-init). - -A full example of a stage is the following: - - -```yaml -#cloud-config - -stages: - # "boot" is the stage - boot: - - systemd_firstboot: - keymap: us - - files: - - path: /tmp/bar - content: | - test - permissions: 0777 - owner: 1000 - group: 100 - if: "[ ! -e /tmp/bar ]" - - files: - - path: /tmp/foo - content: | - test - permissions: 0777 - owner: 1000 - group: 100 - commands: - - echo "test" - modules: - - nvidia - environment: - FOO: "bar" - systctl: - debug.exception-trace: "0" - hostname: "foo" - systemctl: - enable: - - foo - disable: - - bar - start: - - baz - mask: - - foobar - authorized_keys: - user: - - "github:mudler" - - "ssh-rsa ...." - dns: - path: /etc/resolv.conf - nameservers: - - 8.8.8.8 - ensure_entities: - - path: /etc/passwd - entity: | - kind: "user" - username: "foo" - password: "pass" - uid: 0 - gid: 0 - info: "Foo!" - homedir: "/home/foo" - shell: "/bin/bash" - delete_entities: - - path: /etc/passwd - entity: | - kind: "user" - username: "foo" - password: "pass" - uid: 0 - gid: 0 - info: "Foo!" - homedir: "/home/foo" - shell: "/bin/bash" - datasource: - providers: - - "digitalocean" - - "aws" - - "gcp" - path: "/usr/local/etc" -``` - -Note multiple stages can be specified, to execute blocks into different stages, consider: - -```yaml -#cloud-config - -stages: - boot: - - commands: - - echo "hello from the boot stage" - initramfs: - - commands: - - echo "hello from the boot stage" - - commands: - - echo "so much wow, /foo/bar bar exists!" - if: "[ -e /foo/bar ]" -``` - -Below you can find a list of all the supported fields. Mind to replace with the appropriate stage you want to hook into. - -### Filtering stages by node hostname - -Stages can be filtered using the `node` key with a hostname value: - - -```yaml -#cloud-config - -stages: - foo: - - name: "echo" - commands: - - echo hello - node: "the_node_hostname_here" # Node hostname - -``` - -### Filtering stages with if statement - -Stages can be skipped based on if statements: - -```yaml -#cloud-config - -stages: - foo: - - name: "echo" - commands: - - echo hello - if: "cat /proc/cmdline | grep debug" - -name: "Test yip!" -``` - -The expression inside the `if` will be evaluated in bash and, if specified, the stage gets executed only if the condition returns successfully (exit 0). - - -### `name` - -A description of the stage step. Used only when printing output to console. - -### `commands` - -A list of arbitrary commands to run after file writes and directory creation. - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup something" - commands: - - echo 1 > /bar -``` - -### `files` - -A list of files to write to disk. - -```yaml -#cloud-config - -stages: - boot: - - files: - - path: /tmp/bar - content: | - #!/bin/sh - echo "test" - permissions: 0777 - owner: 1000 - group: 100 -``` - -### `directories` - -A list of directories to be created on disk. Runs before `files`. - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup folders" - directories: - - path: "/etc/foo" - permissions: 0600 - owner: 0 - group: 0 -``` - -### `dns` - -A way to configure the `/etc/resolv.conf` file. - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup dns" - dns: - nameservers: - - 8.8.8.8 - - 1.1.1.1 - search: - - foo.bar - options: - - .. - path: "/etc/resolv.conf.bak" -``` -### `hostname` - -A string representing the machine hostname. It sets it in the running system, updates `/etc/hostname` and adds the new hostname to `/etc/hosts`. -Templates can be used to allow dynamic configuration. For example in mass-install scenario it could be needed (and easier) to specify hostnames for multiple machines from a single cloud-init config file. - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup hostname" - hostname: "node-{{ trunc 4 .MachineID }}" -``` -### `sysctl` - -Kernel configuration. It sets `/proc/sys/` accordingly, similarly to `sysctl`. - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup exception trace" - systctl: - debug.exception-trace: "0" -``` - -### `authorized_keys` - -A list of SSH authorized keys that should be added for each user. -SSH keys can be obtained from GitHub user accounts by using the format github:${USERNAME}, similarly for GitLab with gitlab:${USERNAME}. - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup exception trace" - authorized_keys: - mudler: - - "github:mudler" - - "ssh-rsa: ..." -``` - -### `node` - -If defined, the node hostname where this stage has to run, otherwise it skips the execution. The node can also be a regexp in the [Golang format](https://pkg.go.dev/regexp/syntax). - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup logging" - node: "bastion" -``` - -### `users` - -A map of users and user info to set. Passwords can also be encrypted. - -The `users` parameter adds or modifies the specified list of users. Each user is an object which consists of the following fields. Each field is optional and of type string unless otherwise noted. -In case the user already exists, only the `password` and `ssh-authorized-keys` are evaluated. The rest of the fields are ignored. - -- **name**: Required. Login name of user -- **gecos**: GECOS comment of user -- **passwd**: Hash of the password to use for this user. Unencrypted strings are supported too. -- **homedir**: User's home directory. Defaults to /home/*name* -- **no-create-home**: Boolean. Skip home directory creation. -- **primary-group**: Default group for the user. Defaults to a new group created named after the user. -- **groups**: Add user to these additional groups -- **no-user-group**: Boolean. Skip default group creation. -- **ssh-authorized-keys**: List of public SSH keys to authorize for this user -- **system**: Create the user as a system user. No home directory will be created. -- **no-log-init**: Boolean. Skip initialization of lastlog and faillog databases. -- **shell**: User's login shell. - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup users" - users: - bastion: - passwd: "strongpassword" - homedir: "/home/foo -``` - -### `ensure_entities` - -A `user` or a `group` in the [entity](https://github.com/mudler/entities) format to be configured in the system - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup users" - ensure_entities: - - path: /etc/passwd - entity: | - kind: "user" - username: "foo" - password: "x" - uid: 0 - gid: 0 - info: "Foo!" - homedir: "/home/foo" - shell: "/bin/bash" -``` -### `delete_entities` - -A `user` or a `group` in the [entity](https://github.com/mudler/entities) format to be pruned from the system - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup users" - delete_entities: - - path: /etc/passwd - entity: | - kind: "user" - username: "foo" - password: "x" - uid: 0 - gid: 0 - info: "Foo!" - homedir: "/home/foo" - shell: "/bin/bash" -``` -### `modules` - -A list of kernel modules to load. - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup users" - modules: - - nvidia -``` -### `systemctl` - -A list of systemd services to `enable`, `disable`, `mask` or `start`. - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup users" - systemctl: - enable: - - systemd-timesyncd - - cronie - mask: - - purge-kernels - disable: - - crond - start: - - cronie -``` -### `environment` - -A map of variables to write in `/etc/environment`, or otherwise specified in `environment_file` - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup users" - environment: - FOO: "bar" -``` -### `environment_file` - -A string to specify where to set the environment file - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup users" - environment_file: "/home/user/.envrc" - environment: - FOO: "bar" -``` -### `timesyncd` - -Sets the `systemd-timesyncd` daemon file (`/etc/system/timesyncd.conf`) file accordingly. The documentation for `timesyncd` and all the options can be found [here](https://www.freedesktop.org/software/systemd/man/timesyncd.conf.html). - -```yaml -#cloud-config - -stages: - boot: - - name: "Setup NTP" - systemctl: - enable: - - systemd-timesyncd - timesyncd: - NTP: "0.pool.org foo.pool.org" - FallbackNTP: "" - ... -``` - -### `datasource` - -Sets to fetch user data from the specified cloud providers. It populates -provider specific data into `/run/config` folder and the custom user data -is stored into the provided path. - -```yaml -#cloud-config - -stages: - boot: - - name: "Fetch cloud provider's user data" - datasource: - providers: - - "aws" - - "digitalocean" - path: "/etc/cloud-data" -``` - -### `layout` - -Sets additional partitions on disk free space, if any, and/or expands the last -partition. All sizes are expressed in MiB only and default value of `size: 0` -means all available free space in disk. This plugin is useful to be used in -oem images where the default partitions might not suit the actual disk geometry. - -```yaml -#cloud-config - -stages: - boot: - - name: "Repart disk" - layout: - device: - # It will partition a device including the given filesystem label - # or partition label (filesystem label matches first) or the device - # provided in 'path'. The label check has precedence over path when - # both are provided. - label: "COS_RECOVERY" - path: "/dev/sda" - # Only last partition can be expanded and it happens after all the other - # partitions are created. size: 0 means all available free space - expand_partition: - size: 4096 - add_partitions: - - fsLabel: "COS_STATE" - size: 8192 - # No partition label is applied if omitted - pLabel: "state" - - fsLabel: "COS_PERSISTENT" - # default filesystem is ext2 if omitted - filesystem: "ext4" -``` - -### `git` - -Pull git repositories, using golang native git (no need of git in the host). - -```yaml -#cloud-config - -stages: - boot: - - git: - url: "git@gitlab.com:.....git" - path: "/oem/cloud-config-files" - branch: "main" - auth: - insecure: true - private_key: | - -----BEGIN RSA PRIVATE KEY----- - -----END RSA PRIVATE KEY----- -``` - -### `downloads` - -Download files to specified locations - -```yaml -#cloud-config - -stages: - boot: - - downloads: - - path: /tmp/out - url: "https://www...." - permissions: 0700 - owner: 0 - group: 0 - timeout: 0 - owner_string: "root" - - path: /tmp/out - url: "https://www...." - permissions: 0700 - owner: 0 - group: 0 - timeout: 0 - owner_string: "root" -``` diff --git a/docs/content/en/docs/Reference/entangle.md b/docs/content/en/docs/Reference/entangle.md deleted file mode 100644 index 5e672cddb..000000000 --- a/docs/content/en/docs/Reference/entangle.md +++ /dev/null @@ -1,402 +0,0 @@ ---- -title: "Entangle CRDs" -linkTitle: "Entangle" -weight: 8 -date: 2022-11-13 -description: > - Inter-connecting Kubernetes clusters without the need of exposing any service to the public via E2E P2P encrypted networks. - ---- - -{{% alert title="Note" %}} - -This feature is crazy and experimental! Do not run in production servers. -Feedback and bug reports are welcome, as we are improving the p2p aspects of Kairos. - -{{% /alert %}} - -Kairos has two Kubernetes Native extensions ( [entangle](https://github.com/kairos-io/entangle) and [entangle-proxy](https://github.com/kairos-io/entangle-proxy) ) that allows to interconnect services between different clusters via P2P with a shared secret. - -The clusters won't need to do any specific setting in order to establish a connection, as it uses [libp2p](https://github.com/libp2p/go-libp2p) to establish a connection between the nodes. - -Entangle can be used to connect services running on different clusters or can be used with `entangle-proxy` to control another cluster remotely via P2P. - -## Prerequisites - -To `entangle` two or more clusters you need one or more Kubernetes cluster; `entangle` depends on `cert-manager`: - -```bash -kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml -kubectl wait --for=condition=Available deployment --timeout=2m -n cert-manager --all -``` - -- `entangle` needs to run on all the clusters that you wish to interconnect. It provides capabilities to interconnect services between clusters -- `entangle-proxy` only on the cluster that you wish to use as control cluster - -### Install the CRD and `entangle` - -First, add the kairos helm repository: - -```bash -helm repo add kairos https://kairos-io.github.io/helm-charts -helm repo update -``` - -Install the CRDs with: - -```bash -helm install kairos-crd kairos/kairos-crds -``` - -Install `entangle`: - -```bash -helm install kairos-entangle kairos/entangle -## To use a different image: -## helm install kairos-entangle kairos/entangle --set image.serviceTag=v0.18.0 --set image.tag=latest -``` - -### Install `entangle-proxy` - -Now install `entangle-proxy` only on the cluster which is used to control, and which dispatches manifests to downstream clusters. - - -```bash -helm install kairos-entangle-proxy kairos/entangle-proxy -``` - -## Controlling a remote cluster - -![control](https://user-images.githubusercontent.com/2420543/205872002-894f24aa-ac1c-4f70-bb46-aaad89392a25.png) - -To control a remote cluster, you need a cluster where to issue and apply manifest from (the control cluster, where `entangle-proxy` is installed) and a cluster running `entangle` which proxies `kubectl` with a `ServiceAccount`/`Role` associated with it. - -They both need to agree on a secret, which is the `network_token` to be able to communicate, otherwise it won't work. There is no other configuration needed in order for the two cluster to talk to each other. - -### Generating a network token - -Generating a network token is described in [the p2p section](/docs/installation/p2p) - -### Managed cluster - -The cluster which is the target of our manifests, as specified needs to run a deployment which _entangles_ `kubectl`: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: mysecret - namespace: default -type: Opaque -stringData: - network_token: YOUR_NETWORK_TOKEN_GOES_HERE ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: entangle - namespace: default ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: entangle -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - update - - watch - -- apiGroups: - - "" - resources: - - events - verbs: - - create ---- -apiVersion: v1 -kind: List -items: - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: entangle - subjects: - - kind: ServiceAccount - name: entangle - namespace: default - roleRef: - kind: ClusterRole - name: entangle - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: agent-proxy - name: agent-proxy - namespace: default -spec: - selector: - matchLabels: - app: agent-proxy - replicas: 1 - template: - metadata: - labels: - app: agent-proxy - entanglement.kairos.io/name: "mysecret" - entanglement.kairos.io/service: "foo" - entanglement.kairos.io/target_port: "8001" - entanglement.kairos.io/direction: "entangle" - spec: - serviceAccountName: entangle - containers: - - name: proxy - image: "quay.io/kairos/kubectl" - imagePullPolicy: Always - command: ["/usr/bin/kubectl"] - args: - - "proxy" -``` - -Note: replace *YOUR_NETWORK_TOKEN_GOES_HERE* with the token generated with the `kairos-cli`. - -### Control - -To control, from the cluster that has `entangle-proxy` installed we can apply: -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: mysecret - namespace: default -type: Opaque -stringData: - network_token: YOUR_NETWORK_TOKEN_GOES_HERE ---- -apiVersion: entangle-proxy.kairos.io/v1alpha1 -kind: Manifests -metadata: - name: hello - namespace: default - labels: - entanglement.kairos.io/name: "mysecret" - entanglement.kairos.io/service: "foo" - entanglement.kairos.io/target_port: "9090" -spec: - serviceUUID: "foo" - secretRef: "mysecret" - manifests: - - | - apiVersion: v1 - kind: Pod - metadata: - name: test - namespace: default - spec: - containers: - - name: hello - image: busybox:1.28 - command: ['sh', '-c', 'echo "Hello, ssaa!" && sleep 3600'] - restartPolicy: OnFailure -``` - -Note: replace *YOUR_NETWORK_TOKEN_GOES_HERE* with the token generated with the `kairos-cli` and used in the step above. - -## Expose services - -The `entangle` CRD can be used to interconnect services of clusters, or create tunnels to cluster services. - -- Can inject a sidecar container to access a remote services exposed -- Can create a deployment which exposes a remote service from another cluster - -### Deployment - - -`entangle` can be used to tunnel a connection or a service available from one cluster to another. - -![entangle-A](https://user-images.githubusercontent.com/2420543/205871973-d913680d-355f-4322-8cbb-6a94f8505ccb.png) -In the image above, we can see how entangle can create a tunnel for a service running on Cluster A and mirror it to to Cluster B. - - -It can also expose services that are reachable from the host Network: -![entangle-B](https://user-images.githubusercontent.com/2420543/205871999-17abcde8-1b78-4a71-bc3e-ed77664c5551.png) - - -Consider the following example that tunnels a cluster `192.168.1.1:80` to another one using an `Entanglement`: - -{{< tabpane text=true right=true >}} -{{% tab header="Cluster A (where `192.168.1.1:80` is accessible)" %}} -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: mysecret - namespace: default -type: Opaque -stringData: - network_token: _YOUR_SECRET_GOES_HERE_ ---- -apiVersion: entangle.kairos.io/v1alpha1 -kind: Entanglement -metadata: - name: test2 - namespace: default -spec: - serviceUUID: "foo2" - secretRef: "mysecret" - host: "192.168.1.1" - port: "80" - hostNetwork: true -``` -{{% /tab %}} -{{% tab header="Cluster B (which will have a `ClusterIP` available on the Kubernetes service network)" %}} -```yaml ---- -apiVersion: v1 -kind: Secret -metadata: - name: mysecret - namespace: default -type: Opaque -stringData: - network_token: _YOUR_SECRET_GOES_HERE_ ---- -apiVersion: entangle.kairos.io/v1alpha1 -kind: Entanglement -metadata: - name: test3 - namespace: default -spec: - serviceUUID: "foo2" - secretRef: "mysecret" - host: "127.0.0.1" - port: "8080" - inbound: true - serviceSpec: - ports: - - port: 8080 - protocol: TCP - type: ClusterIP -``` -{{% /tab %}} -{{< /tabpane >}} - -### Sidecar injection - -The controller can inject a container which exposes a connection (in both directions): - - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: mysecret - namespace: default -type: Opaque -stringData: - network_token: _YOUR_SECRET_GOES_HERE_ ---- -apiVersion: v1 -kind: Pod -metadata: - name: hello - namespace: default - labels: - # Here we use the labels to refer to the service on the network, and the secret which contains our network_token - entanglement.kairos.io/name: "mysecret" - entanglement.kairos.io/service: "foo" - entanglement.kairos.io/target_port: "9090" -spec: - containers: - - name: hello - image: busybox:1.28 - command: ['sh', '-c', 'echo "Hello, Kubernetes!" && sleep 3600'] - restartPolicy: OnFailure -``` - - -Or we can combine them together: - -{{< tabpane text=true right=true >}} -{{% tab header="Cluster A" %}} -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: mysecret - namespace: default -type: Opaque -stringData: - network_token: _YOUR_SECRET_GOES_HERE_ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: entangle-proxy - name: entangle-proxy - namespace: default -spec: - selector: - matchLabels: - app: entangle-proxy - replicas: 1 - template: - metadata: - labels: - app: entangle-proxy - entanglement.kairos.io/name: "mysecret" - entanglement.kairos.io/service: "foo" - entanglement.kairos.io/target_port: "8001" - entanglement.kairos.io/direction: "entangle" - name: entangle-proxy - spec: - containers: - - name: proxy - image: "quay.io/mudler/k8s-resource-scheduler:latest" - imagePullPolicy: Always - command: ["/usr/bin/kubectl"] - args: - - "proxy" -``` -{{% /tab %}} -{{% tab header="Cluster B" %}} -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: mysecret - namespace: default -type: Opaque -stringData: - network_token: _YOUR_SECRET_GOES_HERE_ ---- -apiVersion: entangle.kairos.io/v1alpha1 -kind: Entanglement -metadata: - name: test - namespace: default -spec: - serviceUUID: "foo" - secretRef: "mysecret" - host: "127.0.0.1" - port: "8080" - inbound: true - serviceSpec: - ports: - - port: 8080 - protocol: TCP - type: ClusterIP -``` -{{% /tab %}} -{{< /tabpane >}} diff --git a/docs/content/en/docs/Reference/faq.md b/docs/content/en/docs/Reference/faq.md deleted file mode 100644 index 1aa286d7c..000000000 --- a/docs/content/en/docs/Reference/faq.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: "FAQ" -linkTitle: "Frequently asked questions" -weight: 9 -date: 2022-11-13 -description: > ---- - -## What is the difference between Kairos compared to Talos/Sidero Metal and Flatcar? - -Kairos is distro-agnostic by design. Currently, you can pick among a list from the [supported matrix](/docs/reference/image_matrix/#image-flavors), but we are working on CRDs to let assemble OSes from other bases in a Kubernetes native way. - -The key difference, is that the OS is distributed as a standard container, similar to how apps are distributed with container registries. You can also use `docker run` locally and inspect the OS, and similarly, push customizations by pointing nodes to a new image. - -Also, Kairos is easy to setup. The P2P capabilities allow nodes to self-coordinate, simplifying the setting up of a multi-node cluster. - -## What would be the difference between Kairos and Fedora Coreos? - -Kairos is distribution agnostic. It supports all the distributions in the [supported matrix](/docs/reference/image_matrix/#image-flavors). In addition, we plan to have K3s automatically deploy Kubernetes (even by self-coordinating nodes). - -Additionally, Kairos is OCI-based, and the system is based from a container image. This makes it possible to also run it locally with `docker run` to inspect it, as well to customize and upgrade your nodes by just pointing at it. Think of it like containers apps, but bootable. - -## If the OS is a container, what is running the container runtime beneath? - -There is no real container runtime. The container is used to construct an image internally, that is then used to boot the system in an A/B fashion, so there is no overhead at all. The system being booted is actually a snapshot of the container. - -## Does this let the OS "containers" install extra kernel extensions/drivers? - -Every container/OS ships its own kernels and drivers within a single image, so you can customize that down the road quite easily. Since every release is a standard container, you can customize it just by writing your own Dockerfile and point your nodes at it. You can also use the CRDs, that allow you to do that natively inside Kubernetes to automate the process even further. - -Kairos also supports live overlaying, but that doesn't apply to kernel modules. However, that is somewhat discouraged, as it introduces snowflakes in your clusters unless you have a management cluster. - -## How is the P2P mesh formed? Is there an external service for discovery? - -The P2P mesh is optional and internally uses libp2p. You can use your own discovery bootstrap server or use the default already baked in the library. Furthermore you can limit and scope that only to local networks. For machines behinds a NAT, nodes operate automatically as relay servers (hops) when they are detected to be capable of it. You can limit that to specific nodes, or let automatic discovery handle it. diff --git a/docs/content/en/docs/Reference/image_matrix.md b/docs/content/en/docs/Reference/image_matrix.md deleted file mode 100644 index 939798436..000000000 --- a/docs/content/en/docs/Reference/image_matrix.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -title: "Image support matrix" -linkTitle: "Image support matrix" -weight: 5 -date: 2022-11-13 -description: > ---- - -Kairos offers several pre-built images for user convenience based on popular Linux distributions such as openSUSE, Alpine Linux, and Ubuntu. The Kairos core team does its best to test these images, but those that are based on systemd (e.g. openSUSE, Ubuntu) are more thoroughly tested due to their homogenous settings. Support for other non-systemd based flavors (e.g. Alpine) may be limited due to team bandwidth. However, as Kairos is an open source community-driven project, we welcome any contributions, bug reports, and bug fixes. Check out our [Contribution guidelines](https://github.com/kairos-io/kairos/contribute) for more information. - -In addition, tighter integration with systemd allows for several features that are only available with it, such as live layering. - -These images are pushed to quay.io and are available for installation and upgrading. The installable mediums included in the releases are generated using the methods described in the [automated installation reference](/docs/installation/automated/#iso-remastering), and the images can be used for upgrades as well. - -## Image flavors - -Kairos release processes generates images based on official container images from popular Linux distributions. If you don't see your preferred distribution, check if [we are already planning](https://github.com/kairos-io/kairos/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Fflavor) support for it or create a new issue. - -Below is a list of the available images and their locations on the quay.io registry: - -- The **Core** images do not include any Kubernetes engine and can be used as a base for customizations. -- The **Standard** images include `k3s` and the [kairos provider](https://github.com/kairos-io/provider-kairos), which enables Kubernetes deployments and optionally enables [p2p](/docs/installation/p2p). - -Base images are tagged with specific upstream versions (e.g. Ubuntu 20 LTS is pinned to Ubuntu 20:04, openSUSE to openSUSE leap 15.4, etc.). - -| **Flavor/Variant** | amd64 | arm64 | -|------------------------------------------|--------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Alpine Linux based (openSUSE kernel)** | [core][c-alpine-opensuse-leap], [standard][k-alpine-opensuse-leap] | [core][c-alpine-arm-rpi], [core-img][c-alpine-arm-rpi-img], [standard][k-alpine-arm-rpi], [standard-img][k-alpine-arm-rpi-img] | -| **Alpine Linux based (Ubuntu kernel)** | [core][c-alpine-ubuntu], [standard][k-alpine-ubuntu] | | -| **Debian based** | [core][c-debian], [standard][k-debian] | | -| **Fedora based** | [core][c-fedora], [standard][k-fedora] | | -| **openSUSE Leap based** | [core][c-opensuse-leap], [standard][k-opensuse-leap] | [core][c-opensuse-leap-arm-rpi], [core-img][c-opensuse-leap-arm-rpi-img], [standard][k-opensuse-leap-arm-rpi], [standard-img][k-opensuse-leap-arm-rpi-img] | -| **openSUSE Tumbleweed based** | [core][c-opensuse-tumbleweed], [standard][k-opensuse-tumbleweed] | [core][c-opensuse-tumbleweed-arm-rpi], [standard][k-opensuse-tumbleweed-arm-rpi] | -| **Ubuntu based (rolling)** ** | [core][c-ubuntu], [standard][k-ubuntu] | [core][c-ubuntu-rpi], [standard][k-ubuntu-rpi] | -| **Ubuntu based (22 LTS)** ** | [core][c-ubuntu-22-lts], [standard][k-ubuntu-22-lts] | | -| **Ubuntu based (20 LTS)** ** | [core][c-ubuntu-20-lts], [standard][k-ubuntu-20-lts] | | -| **Rocky Linux based** | [core][c-rockylinux], [standard][k-rockylinux] | | - - -[c-alpine-opensuse-leap]: https://quay.io/repository/kairos/core-alpine-opensuse-leap -[c-alpine-ubuntu]: https://quay.io/repository/kairos/core-alpine-ubuntu -[c-alpine-arm-rpi]: https://quay.io/repository/kairos/core-alpine-arm-rpi -[c-alpine-arm-rpi-img]: https://quay.io/repository/kairos/core-alpine-arm-rpi-img -[c-debian]: https://quay.io/repository/kairos/core-debian -[c-fedora]: https://quay.io/repository/kairos/core-fedora -[c-opensuse-leap]: https://quay.io/repository/kairos/core-opensuse-leap -[c-opensuse-leap-arm-rpi]: https://quay.io/repository/kairos/core-opensuse-leap-arm-rpi -[c-opensuse-leap-arm-rpi-img]: https://quay.io/repository/kairos/core-opensuse-leap-arm-rpi-img -[c-opensuse-tumbleweed]: https://quay.io/repository/kairos/core-opensuse-tumbleweed -[c-opensuse-tumbleweed-arm-rpi]: https://quay.io/repository/kairos/core-opensuse-tumbleweed-arm-rpi -[c-opensuse-tumbleweed-arm-rpi-img]: https://quay.io/repository/kairos/core-opensuse-tumbleweed-arm-rpi-img -[c-ubuntu]: https://quay.io/repository/kairos/core-ubuntu -[c-ubuntu-22-lts]: https://quay.io/repository/kairos/core-ubuntu-22-lts -[c-ubuntu-20-lts]: https://quay.io/repository/kairos/core-ubuntu-20-lts -[c-ubuntu-rpi]: https://quay.io/repository/kairos/core-ubuntu-arm-rpi -[c-rockylinux]: https://quay.io/repository/kairos/core-rockylinux - -[k-alpine-opensuse-leap]: https://quay.io/repository/kairos/kairos-alpine-opensuse-leap -[k-alpine-ubuntu]: https://quay.io/repository/kairos/kairos-alpine-ubuntu -[k-alpine-arm-rpi]: https://quay.io/repository/kairos/kairos-alpine-arm-rpi -[k-alpine-arm-rpi-img]: https://quay.io/repository/kairos/kairos-alpine-arm-rpi-img -[k-debian]: https://quay.io/repository/kairos/kairos-debian -[k-fedora]: https://quay.io/repository/kairos/kairos-fedora -[k-opensuse-leap]: https://quay.io/repository/kairos/kairos-opensuse-leap -[k-opensuse-leap-arm-rpi]: https://quay.io/repository/kairos/kairos-opensuse-leap-arm-rpi -[k-opensuse-leap-arm-rpi-img]: https://quay.io/repository/kairos/kairos-opensuse-leap-arm-rpi-img -[k-opensuse-tumbleweed]: https://quay.io/repository/kairos/kairos-opensuse-tumbleweed -[k-opensuse-tumbleweed-arm-rpi]: https://quay.io/repository/kairos/kairos-opensuse-tumbleweed-arm-rpi -[k-opensuse-tumbleweed-arm-rpi-img]: https://quay.io/repository/kairos/kairos-opensuse-tumbleweed-arm-rpi-img -[k-ubuntu]: https://quay.io/repository/kairos/kairos-ubuntu -[k-ubuntu-22-lts]: https://quay.io/repository/kairos/kairos-ubuntu-22-lts -[k-ubuntu-20-lts]: https://quay.io/repository/kairos/kairos-ubuntu-20-lts -[k-ubuntu-rpi]: https://quay.io/repository/kairos/kairos-ubuntu-arm-rpi -[k-rockylinux]: https://quay.io/repository/kairos/kairos-rockylinux - -{{% alert title="Note" color="info" %}} - -** The `ubuntu` flavor tracks the latest available Ubuntu release (at the time of writing 22.10). The LTS flavors, on the other hand, track the latest LTS available on DockerHub. For example, ubuntu-22-lts uses 22.04 as the base image. -{{% /alert %}} - -{{% alert title="Note" color="info" %}} -The pipelines do not publish `img` artifacts for the arm architecture because the files are too large for GitHub Actions (they exceed the artifact size limit). These artifacts can be extracted from the published docker images using the following command: - -```bash -export IMAGE={{< registryURL >}}/core-{{< armFlavor >}}-arm-rpi-img:{{< kairosVersion >}} -docker run -ti --rm -v $PWD:/image quay.io/luet/base util unpack "$IMAGE" /image -``` - -(replace `$IMAGE` with the proper image) - -The artifacts can be found in the `build` directory. - -{{% /alert %}} - -### Framework images - -Kairos releases contains also the __framework__ assets that can be used to [build Kairos images from Scratch](/docs/reference/build-from-scratch). - -Framework images can be found in quay at: https://quay.io/repository/kairos/framework. - -Each tag follow the convention: `_`. - -## Versioning policy - -Kairos follows [Semantic Versioning](https://semver.org/) and our releases signal changes to Kairos components, rather than changes to the underlying OS and package versions. Flavors are pinned to specific upstream OS branches (e.g. `opensuse` to `leap 15.4`) and major version bumps will be reflected through new flavors in our build matrix or through specific releases to follow upstream with regard to minor version bumps (e.g. `leap 15.3` and `leap 15.4`). - -Here are some key points to note: -- We only support the latest release branch with patch releases. -- Patch releases (e.g. _1.1.x_) follow a weekly release cadence, unless there are exceptions for highly impactful bugs in Kairos itself or at the OS layer (e.g. high-severity CVEs). -- Minor releases follow a monthly cadence and are expected to bring enhancements through planned releases. -- Major releases signal new advanced features or significant changes to the codebase. In-place upgrades from old to new major release branches are not always guaranteed, but we strive for compatibility across versions. - -{{% alert title="Note" color="info" %}} -In order to give users more control over the chosen base image (e.g. `openSUSE`, `Ubuntu`, etc.) and reduce reliance on our CI infrastructure, we are actively working on streamlining the creation of Kairos-based distributions directly from upstream base images. You can track the development progress [here](https://github.com/kairos-io/kairos/issues/116). - -If you need to further customize images, including changes to the base image, package updates, and CVE hotfixes, check out the [customization docs](/docs/advanced/customizing). -{{% /alert %}} - - -## Release changelog - -Our changelog is published as part of the release process and contains all the changes, highlights, and release notes that are relevant to the release. We strongly recommend checking the changelog for each release before upgrading or building a customized version of Kairos. - -Release changelogs are available for Kairos core and for each component. Below is a list of the components that are part of a Kairos release and their respective release pages with changelogs. - -| **Project** | **Release page** | -|----------------------------------------------------- |--------------------------------------------------------- | -| **Kairos core** | https://github.com/kairos-io/kairos/releases | -| **Kairos provider (k3s support)** | https://github.com/kairos-io/provider-kairos/releases | diff --git a/docs/content/en/docs/Reference/recovery_mode.md b/docs/content/en/docs/Reference/recovery_mode.md deleted file mode 100644 index e66232ad1..000000000 --- a/docs/content/en/docs/Reference/recovery_mode.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: "Recovery mode" -linkTitle: "Recovery mode" -weight: 7 -date: 2022-11-13 -description: > ---- - -The Kairos recovery mode can be used to recover a damaged system or to regain access remotely (with assistance) to a machine which has been lost access to. The recovery mode is accessible only from the GRUB menu, from both the LiveCD, and an installed system. - -{{% alert title="Note" %}} -On installed system, there are two recovery modes available during boot. Below describes only how the Kairos remote recovery works. It can be used to reset the A/B partitions (with the user/pass used during setup) and perform any other operation without remote access. -{{% /alert %}} - -## Boot into recovery mode - -Kairos recovery mode can be accessed either via ISO or from an installed system. - -A GRUB menu will be displayed: -![Screenshot from 2022-04-28 17-48-06](https://user-images.githubusercontent.com/2420543/165800177-3e4cccd8-f67c-43a2-bd88-329478539400.png) - -Select the last entry `kairos (remote recovery mode)` and press enter. - -At this point the boot process starts, and you should be welcomed by the Kairos screen: - -![Screenshot from 2022-04-28 17-48-32](https://user-images.githubusercontent.com/2420543/165800182-9aa29c90-09e9-4c53-b3c7-c8ced262e3ac.png) - -After few seconds, the recovery process starts, and right after a QR code will be printed out of the screen along with a password which can be used later to SSH into the machine: - -![Screenshot from 2022-04-28 17-48-43](https://user-images.githubusercontent.com/2420543/165800187-4d2fe04e-c501-4ad8-a29f-32a0110eaa72.png) - -At this stage, take a screenshot or a photo and save the image with the QR code. - -## Connect to the machine - -In the another machine that you are using to connect to your server, (your workstation, a jumpbox, or other) use the Kairos CLI to connect over the remote machine: - -``` -$ ./kairos bridge --qr-code-image /path/to/image.png - INFO Connecting to service kAIsuqiwKR - INFO SSH access password is yTXlkak - INFO SSH server reachable at 127.0.0.1:2200 - INFO To connect, keep this terminal open and run in another terminal 'ssh 127.0.0.1 -p 2200' the password is yTXlkak - INFO Note: the connection might not be available instantly and first attempts will likely fail. - INFO Few attempts might be required before establishing a tunnel to the host. - INFO Starting EdgeVPN network - INFO Node ID: 12D3KooWSTRBCTNGZ61wzK5tgYvFi8rQVxkXJCDUYngBWGDSyoBK - INFO Node Addresses: [/ip4/192.168.1.233/tcp/36071 /ip4/127.0.0.1/tcp/36071 /ip6/::1/tcp/37661] - INFO Bootstrapping DHT -``` - -At this point, the bridge should start, and you should be able to see connection messages in the terminal. You can connect to the remote machine by using `ssh` and pointing it locally at `127.0.0.1:2200`. The username is not relevant, the password is print from the CLI. - -The bridge operates in the foreground, so you have to shut it down by using CTRL-C. diff --git a/docs/content/en/docs/Reference/reset.md b/docs/content/en/docs/Reference/reset.md deleted file mode 100644 index 94a058360..000000000 --- a/docs/content/en/docs/Reference/reset.md +++ /dev/null @@ -1,130 +0,0 @@ ---- -title: "Reset a node" -linkTitle: "Reset" -weight: 4 -date: 2022-11-13 -description: > ---- - -Kairos has a recovery mechanism built-in which can be leveraged to restore the system to a known point. At installation time, the recovery partition is created from the installation medium and can be used to restore the system from scratch, leaving configuration intact and cleaning any persistent data accumulated by usage in the host (e.g. Kubernetes images, persistent volumes, etc. ). - -The reset action will regenerate the bootloader configuration and the images in the state partition (labeled `COS_STATE`) by using the recovery image generated at install time, cleaning up the host. - -The configuration files in `/oem` are kept intact, the node on the next reboot after a reset will perform the same boot sequence (again) of a first-boot installation. - -# How to - -{{% alert title="Note" %}} - -By following the steps below you will _reset_ entirely a node and the persistent data will be lost. This includes _every_ user-data stored on the machine. - -{{% /alert %}} - -The reset action can be accessed via the Boot menu, remotely, triggered via Kubernetes or manually. In each scenario the machine will reboot into reset mode, perform the cleanup, and reboot automatically afterwards. - -## From the boot menu - -It is possible to reset the state of a node by either booting into the "Reset" mode into the boot menu, which automatically will reset the node: - -![reset](https://user-images.githubusercontent.com/2420543/191941281-573e2bed-f66c-48db-8c46-e8034417539e.gif?classes=border,shadow) - -## Remotely, via command line - -On a Kairos booted system, logged as root: - -```bash -$ grub2-editenv /oem/grubenv set next_entry=statereset -$ reboot -``` - -## From Kubernetes - -`system-upgrade-controller` can be used to apply a plan to the nodes to use Kubernetes to schedule the reset on the nodes itself, similarly on how upgrades are applied. - -Consider the following example which resets a machine by changing the config file used during installation: -```yaml ---- -apiVersion: v1 -kind: Secret -metadata: - name: custom-script - namespace: system-upgrade -type: Opaque -stringData: - config.yaml: | - #cloud-config - hostname: testcluster-{{ trunc 4 .MachineID }} - k3s: - enabled: true - users: - - name: kairos - passwd: kairos - ssh_authorized_keys: - - github:mudler - add-config-file.sh: | - #!/bin/sh - set -e - if diff /host/run/system-upgrade/secrets/custom-script/config.yaml /host/oem/90_custom.yaml >/dev/null; then - echo config present - exit 0 - fi - # we can't cp, that's a symlink! - cat /host/run/system-upgrade/secrets/custom-script/config.yaml > /host/oem/90_custom.yaml - grub2-editenv /host/oem/grubenv set next_entry=statereset - sync - - mount --rbind /host/dev /dev - mount --rbind /host/run /run - nsenter -i -m -t 1 -- reboot - exit 1 ---- -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: reset-and-reconfig - namespace: system-upgrade -spec: - concurrency: 2 - # This is the version (tag) of the image. - # The version is refered to the kairos version plus the k3s version. - version: "v1.0.0-rc2-k3sv1.23.9-k3s1" - nodeSelector: - matchExpressions: - - { key: kubernetes.io/hostname, operator: Exists } - serviceAccountName: system-upgrade - cordon: false - upgrade: - # Here goes the image which is tied to the flavor being used. - # Currently can pick between opensuse and alpine - image: quay.io/kairos/kairos-opensuse - command: - - "/bin/bash" - - "-c" - args: - - bash /host/run/system-upgrade/secrets/custom-script/add-config-file.sh - secrets: - - name: custom-script - path: /host/run/system-upgrade/secrets/custom-script -``` - -## Manual reset - -It is possible to trigger the reset manually by logging into the recovery from the boot menu and running `kairos reset` from the console. - -To optionally change the behavior of the reset process (such as cleaning up also configurations), run `elemental reset` instead which supports options via arg: - -| Option | Description | -| ------------------- | --------------------------- | -| --reset-persistent | Clear persistent partitions | -| --reset-oem | Clear OEM partitions | -| --system.uri string | Reset with the given image | - -- **Note**: `--reset-oem` resets the system pruning all the configurations. -- `system.uri` allows to reset using another image or a directory. - `string` can be among the following: `dir:/path/to/dir`, `oci:`, `docker:`, `channel:` or `file:/path/to/file`. - -### Cleaning up state directories - -An alternative way and manual of resetting your system is possible by deleting the state paths. You can achieve this by deleting the contents of the `/usr/local` directory. It's recommended that you do this while in recovery mode with all services turned off. - -Please note that within `/usr/local`, there are two important folders to keep in mind. The first is `/usr/local/.kairos`, which contains sentinel files that will trigger a complete deployment from scratch when deleted. However, your data will be preserved. The second folder is `/usr/local/`.state`, which contains the bind-mounted data for the system. By deleting these two folders, you can achieve a pristine environment while leaving all other contents of `/usr/local` untouched. \ No newline at end of file diff --git a/docs/content/en/docs/Reference/troubleshooting.md b/docs/content/en/docs/Reference/troubleshooting.md deleted file mode 100644 index f308aaf74..000000000 --- a/docs/content/en/docs/Reference/troubleshooting.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: "Troubleshooting" -linkTitle: "Troubleshooting" -weight: 6 -date: 2022-11-13 -description: > ---- - -Things can go wrong. This section tries to give guidelines in helping out identify potential issues. - -It is important first to check out if your issue was already submitted [in the issue tracker](https://github.com/kairos-io/kairos/issues). - -## Gathering logs - -To gather useful logs and help developers spot right away issues, it's suggested to boot with `console=tty0 rd.debug` enabled for example: - -![debug](https://user-images.githubusercontent.com/2420543/191934926-7d4ac908-9a4c-4ef4-9891-75820e6b8fe6.gif) - -To edit the boot commands, type 'e' in the boot menu. To boot with the changes press 'CTRL+X'. - -In case logs can't be acquired, taking screenshot or videos while opening up issues it's strongly reccomended! - -## Initramfs breakpoints - -Initramfs can be instructed to drop a shell in various phases of the boot process. For instance: - -- `rd.break=pre-mount rd.shell`: Drops a shell before setting up mount points. -- `rd.break=pre-pivot rd.shell`: Drops a shell before switch-root - -## Disable immutability - -It is possible to disable immutability by adding `rd.cos.debugrw` to the kernel boot commands. - -## Root permission - -By default, there is no root user set. A default user (`kairos`) is created and can use `sudo` without password authentication during LiveCD bootup. - -## Get back the kubeconfig - -On all nodes, which are deployed with the P2P full-mesh feature of the cluster, it's possible to invoke `kairos get-kubeconfig` to recover the kubeconfig file. - -## See also - -- [Dracut debug docs](https://fedoraproject.org/wiki/How_to_debug_Dracut_problems) diff --git a/docs/content/en/docs/Upgrade/_index.md b/docs/content/en/docs/Upgrade/_index.md deleted file mode 100644 index 2b6c8fc77..000000000 --- a/docs/content/en/docs/Upgrade/_index.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Upgrade" -linkTitle: "Upgrade" -weight: 3 -description: > ---- diff --git a/docs/content/en/docs/Upgrade/kubernetes.md b/docs/content/en/docs/Upgrade/kubernetes.md deleted file mode 100644 index 17d80eb2b..000000000 --- a/docs/content/en/docs/Upgrade/kubernetes.md +++ /dev/null @@ -1,248 +0,0 @@ ---- -title: "Upgrading from Kubernetes" -linkTitle: "Upgrading from Kubernetes" -weight: 1 -date: 2022-11-13 -description: > ---- - -Kairos upgrades can be performed either manually or via Kubernetes if the cluster is composed of Kairos nodes. In order to trigger upgrades, it is required to apply a `Plan` spec to the target cluster for the upgrade. - -## Prerequisites - -- It is necessary [system-upgrade-controller](https://github.com/rancher/system-upgrade-controller) to be deployed in the target cluster. - -To install it, use kubectl: - -```bash -kubectl apply -f https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/system-upgrade-controller.yaml -``` - -### Upgrading from version X to version Y with Kubernetes - -To trigger an upgrade, create a plan for `system-upgrade-controller` which refers to the image version that we want to upgrade. - -```bash -cat <<'EOF' | kubectl apply -f - ---- -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: os-upgrade - namespace: system-upgrade - labels: - k3s-upgrade: server -spec: - concurrency: 1 - # This is the version (tag) of the image. - # The version is refered to the kairos version plus the k3s version. - version: "v1.0.0-k3sv1.24.3-k3s1" - nodeSelector: - matchExpressions: - - {key: kubernetes.io/hostname, operator: Exists} - serviceAccountName: system-upgrade - cordon: false - drain: - force: false - disableEviction: true - upgrade: - # Here goes the image which is tied to the flavor being used. - # Currently can pick between opensuse and alpine - image: quay.io/kairos/kairos-opensuse-leap - command: - - "/usr/sbin/suc-upgrade" -EOF -``` - -To check all the available versions, see the [images](https://quay.io/repository/kairos/kairos-opensuse-leap?tab=tags) available on the container registry, corresponding to the flavor/version selected. - -{{% alert title="Note" %}} - -Several upgrade strategies can be used with `system-upgrade-controller` which are not illustrated here in this example. For instance, it can be specified in the number of hosts which are running the upgrades, filtering by labels, and more. [Refer to the project documentation](https://github.com/rancher/system-upgrade-controller) on how to create efficient strategies to roll upgrades on the nodes. In the example above, the upgrades are applied to every host of the cluster, one-by-one in sequence. - -{{% /alert %}} - -A pod should appear right after which carries on the upgrade and automatically reboots the node: - -``` -$ kubectl get pods -A -... -system-upgrade apply-os-upgrade-on-kairos-with-1a1a24bcf897bd275730bdd8548-h7ffd 0/1 Creating 0 40s -``` - -Done! We should have all the basics to get our first cluster rolling, but there is much more we can do. - -## Verify images attestation during upgrades - -Container images can be signed during the build phase of a CI/CD pipeline using [Cosign](https://github.com/sigstore/cosign), Kairos signs every artifact as part of the release process. - -To ensure that the images used during upgrades match the expected signatures, [Kyverno](https://kyverno.io/) can be used to set up policies. This is done by checking if the signature is present in the OCI registry and if the image was signed using the specified key. The policy rule check fails if either of these conditions is not met. - -To learn more about this specific Kyverno feature, you can refer to the [documentation](https://kyverno.io/docs/writing-policies/verify-images/). This allows for the verification of image authenticity directly at the node level prior to upgrading. - -A Kyverno policy for `provider-kairos` images might look like the following: - -```yaml -apiVersion: kyverno.io/v1 -kind: ClusterPolicy -metadata: - name: check-image -spec: - validationFailureAction: Enforce - background: false - webhookTimeoutSeconds: 30 - failurePolicy: Fail - rules: - - name: check-image - match: - any: - - resources: - kinds: - - Pod - verifyImages: - - imageReferences: - - "quay.io/kairos/kairos-*" - attestors: - - entries: - # See: https://kyverno.io/docs/writing-policies/verify-images/#keyless-signing-and-verification - - keyless: - subject: "https://github.com/kairos-io/provider-kairos/.github/workflows/release.yaml@refs/tags/*" - issuer: "https://token.actions.githubusercontent.com" - rekor: - url: https://rekor.sigstore.dev -``` - -To install Kyverno in a Kairos cluster, you can simply use the community [bundles](/docs/advanced/bundles). For example, you can use the following installation cloud config file: - -```yaml -#cloud-config - -hostname: kyverno-{{ trunc 4 .MachineID }} - -# Specify the bundle to use -bundles: -- targets: - - run://quay.io/kairos/community-bundles:system-upgrade-controller_latest - - run://quay.io/kairos/community-bundles:cert-manager_latest - - run://quay.io/kairos/community-bundles:kyverno_latest - -users: -- name: kairos - passwd: kairos - -k3s: - enabled: true -``` - -This configuration file prepare the system with the `cert-manager`, `system-upgrade-controller` and the `kyverno` bundle, enabling `k3s`. - -## Customize the upgrade plan - -It is possible to run additional commands before the upgrade takes place into the node, consider the following example: - -```yaml ---- -apiVersion: v1 -kind: Secret -metadata: - name: custom-script - namespace: system-upgrade -type: Opaque -stringData: - upgrade.sh: | - #!/bin/sh - set -e - - # custom command, for example, that injects or modifies a configuration option - sed -i 's/something/to/g' /host/oem/99_custom.yaml - # run the upgrade script - /usr/sbin/suc-upgrade ---- -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: custom-os-upgrade - namespace: system-upgrade -spec: - concurrency: 1 - # This is the version (tag) of the image. - # The version is refered to the kairos version plus the k3s version. - version: "v1.0.0-rc2-k3sv1.23.9-k3s1" - nodeSelector: - matchExpressions: - - { key: kubernetes.io/hostname, operator: Exists } - serviceAccountName: system-upgrade - cordon: false - drain: - force: false - disableEviction: true - upgrade: - # Here goes the image which is tied to the flavor being used. - # Currently can pick between opensuse and alpine - image: quay.io/kairos/kairos-opensuse-leap - command: - - "/bin/bash" - - "-c" - args: - - bash /host/run/system-upgrade/secrets/custom-script/upgrade.sh - secrets: - - name: custom-script - path: /host/run/system-upgrade/secrets/custom-script -``` - -## Upgrade from c3os to Kairos - -If you already have a `c3os` deployment, upgrading to Kairos requires changing every instance of `c3os` to `kairos` in the configuration file. This can be either done manually or with Kubernetes before rolling the upgrade. Consider customizing the upgrade plan, for instance: - -```yaml ---- -apiVersion: v1 -kind: Secret -metadata: - name: custom-script - namespace: system-upgrade -type: Opaque -stringData: - upgrade.sh: | - #!/bin/sh - set -e - sed -i 's/c3os/kairos/g' /host/oem/99_custom.yaml - /usr/sbin/suc-upgrade ---- -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: custom-os-upgrade - namespace: system-upgrade -spec: - concurrency: 1 - # This is the version (tag) of the image. - # The version is refered to the kairos version plus the k3s version. - version: "v1.0.0-rc2-k3sv1.23.9-k3s1" - nodeSelector: - matchExpressions: - - { key: kubernetes.io/hostname, operator: Exists } - serviceAccountName: system-upgrade - cordon: false - drain: - force: false - disableEviction: true - upgrade: - # Here goes the image which is tied to the flavor being used. - # Currently can pick between opensuse and alpine - image: quay.io/kairos/kairos-opensuse-leap - command: - - "/bin/bash" - - "-c" - args: - - bash /host/run/system-upgrade/secrets/custom-script/upgrade.sh - secrets: - - name: custom-script - path: /host/run/system-upgrade/secrets/custom-script -``` - -## What's next? - -- [Upgrade nodes manually](/docs/upgrade/manual) -- [Immutable architecture](/docs/architecture/immutable) -- [Create decentralized clusters](/docs/installation/p2p) diff --git a/docs/content/en/docs/Upgrade/manual.md b/docs/content/en/docs/Upgrade/manual.md deleted file mode 100644 index 50826165f..000000000 --- a/docs/content/en/docs/Upgrade/manual.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: "Manual" -linkTitle: "Manual" -weight: 2 -date: 2022-11-13 -description: > ---- - -Upgrades can be run manually from the terminal. - -Kairos images are released on [quay.io](https://quay.io/organization/kairos). - -## List available versions - -To see all the available versions: - -```bash -$ sudo kairos-agent upgrade list-releases -v0.57.0 -v0.57.0-rc2 -v0.57.0-rc1 -v0.57.0-alpha2 -v0.57.0-alpha1 -``` - -## Upgrade - -To upgrade to the latest available version, run from a shell of a cluster node the following: - -```bash -sudo kairos-agent upgrade -``` - -To specify a version, run: - -```bash -sudo kairos-agent upgrade -``` - -Use `--force` to force upgrading to avoid checking versions. - -To specify a specific image, use the `--image` flag: - -```bash -sudo kairos-agent upgrade --image -``` - - -To upgrade with a container image behind a registry with authentication, the upgrade command provides the following flags: - -| Flag | Description | -|-------------------------|------------------------------------------------------------------------------------------| -| `--auth-username` | User to authenticate with | -| `--auth-password` | Password to authenticate with | -| `--auth-server-address` | Server address to authenticate to, defaults to docker | -| `--auth-registry-token` | IdentityToken is used to authenticate the user and get an access token for the registry. | -| `--auth-identity-token` | RegistryToken is a bearer token to be sent to a registry | - -For instance: - -```bash -sudo kairos-agent upgrade --image private/myimage:latest --auth-username MYNAME --auth-password MYPASSWORD -``` \ No newline at end of file diff --git a/docs/content/en/docs/_index.md b/docs/content/en/docs/_index.md deleted file mode 100755 index 7566fe932..000000000 --- a/docs/content/en/docs/_index.md +++ /dev/null @@ -1,143 +0,0 @@ - ---- -title: "Welcome" -linkTitle: "Documentation" -weight: 20 -menu: - main: - weight: 20 ---- - -Welcome to the Kairos Documentation - -Kairos is the open-source project that simplifies Edge, cloud, and bare metal OS lifecycle management. With a unified Cloud Native API, Kairos is community-driven, open source, and distro agnostic. - -Our key features include: - -- [Immutability](/docs/architecture/immutable): ensure your infrastructure stays consistent with atomic upgrades -- Security: protect your cluster from vulnerabilities and attacks with a read-only system -- [Container-based](/docs/architecture/container): manage your nodes as apps in containers for maximum flexibility and portability -- [P2P Mesh](/docs/architecture/network): self-coordinated, automated, no interaction Kubernetes deployments with P2P -- [Meta-Distribution](/docs/architecture/meta), distro agnostic - - -In this documentation, you will find everything you need to know about Kairos, from installation and configuration, to examples and advanced features. - -To get started with Kairos, follow the instructions in the [quickstart](/docs/getting-started) guide. Then, check out the [examples](/docs/examples) to see how Kairos can be used in real-world scenarios. - -For more information, please refer to this documentation. If you have any questions or feedback, feel free to [open an issue](https://github.com/kairos-io/kairos/issues/new) or [join our community forum](https://github.com/kairos-io/kairos/discussions). - -{{% alert title="Note" %}} -You can also find some good resources on the [Media Section]({{< ref "docs/media" >}} "Media") -{{% /alert %}} - -## What is Kairos ? - -Kairos is a cloud-native meta-Linux distribution that runs on Kubernetes and brings the power of the public cloud to your on-premises environment. With Kairos, you can build your own cloud with complete control and no vendor lock-in. - -Here are a few reasons why you should try Kairos: - -- Build your own cloud on-premises with complete control and no vendor lock-in -- Provision nodes with your own image or use Kairos releases for added flexibility -- Use Kairos for a wide range of use cases, from Kubernetes applications to appliances and more -- Simple and streamlined day-2 operations (e.g. node upgrades) - -## What I can do with it ? - -With Kairos, you can easily spin up a Kubernetes cluster with the Linux distribution of your choice, and manage the entire cluster lifecycle with Kubernetes. Try Kairos today and experience the benefits of a unified, cloud-native approach to OS management. - -With Kairos, you can: - -- Spin up a Kubernetes cluster with any Linux distribution in just a few clicks -- Create an immutable infrastructure that stays consistent and free of drift with atomic upgrades -- Manage your cluster's entire lifecycle with Kubernetes, from building to upgrading -- Automatically create multi-node, single clusters that spans across regions for maximum flexibility and scalability - -Try Kairos today and experience the benefits of a unified, cloud-native approach to OS management. Say goodbye to the hassle of managing multiple systems, and hello to a more streamlined and efficient way of working. - -## Features - -- Easily create multi-node Kubernetes clusters with [K3s](https://k3s.io), and enjoy all of [K3s](https://k3s.io)'s features -- Upgrade manually via CLI or with Kubernetes, and use container registries for distribution upgrades -- Enjoy the benefits of an immutable distribution that stays configured to your needs -- Configure nodes with a single cloud-init config file for added simplicity -- Upgrade even in airgap environments with in-cluster container registries -- Extend your image at runtime or build time with Kubernetes Native APIs -- Coming soon: CAPI support with full device lifecycle management and more -- Create private virtual network segments with a full-mesh P2P hybrid VPN network that can stretch up to 10000 km - -## More than a Linux distribution - -Kairos is more than just an ISO, qcow2, or Netboot artifact. It allows you to turn any Linux distribution into a uniform and compliant distro with an immutable design. This means that any distro "converted" with Kairos will share the same common feature set and can be managed in the same way using Kubernetes Native API components. Kairos treats all OSes homogeneously and upgrades are distributed via container registries. Installations mediums and other assets required for booting bare metal or edge devices are built dynamically by Kairos' Kubernetes Native API components. - -![livecd](https://user-images.githubusercontent.com/2420543/189219806-29b4deed-b4a1-4704-b558-7a60ae31caf2.gif) - -## Goals - -The Kairos ultimate goal is to bridge the gap between Cloud and Edge by creating a smooth user experience. There are several areas in the ecosystem that can be improved for edge deployments to make it in pair with the cloud. - -The Kairos project encompasses all the tools and architectural pieces needed to fill those gaps. This spans between providing Kubernetes Native API components to assemble OSes, deliver upgrades, and control nodes after deployment. - -Kairos is distro-agnostic, and embraces openness: The user can provide their own underlying base image, and Kairos onboards it and takes it over to make it Cloud Native, immutable that plugs into an already rich ecosystem by leveraging containers as distribution medium. - -## Contribute - -Kairos is an open source project, and any contribution is more than welcome! The project is big and narrows to various degrees of complexity and problem space. Feel free to join our chat, discuss in our forums and join us in the Office hours. Check out the [contribution guidelines](https://github.com/kairos-io/kairos/contribute) to see how to get started and our [governance](https://github.com/kairos-io/kairos/blob/master/GOVERNANCE.md). - -We have an open roadmap, so you can always have a look on what's going on, and actively contribute to it. - -Useful links: - -- [Upcoming releases](https://github.com/kairos-io/kairos/issues?q=is%3Aissue+is%3Aopen+label%3Arelease) - - -## Community - -You can find us at: - -- [#Kairos-io at matrix.org](https://matrix.to/#/#kairos-io:matrix.org) -- [IRC #kairos in libera.chat](https://web.libera.chat/#kairos) -- [GitHub Discussions](https://github.com/kairos-io/kairos/discussions) - -### Project Office Hours - -Project Office Hours is an opportunity for attendees to meet the maintainers of the project, learn more about the project, ask questions, learn about new features and upcoming updates. - -Office hours are happening weekly on Wednesday - 5:30 – 6:00pm CEST. [Meeting link](https://meet.google.com/aus-mhta-azb) - -Besides, we have monthly meetup to participate actively into the roadmap planning and presentation which takes part during the office hours: - -#### Roadmap planning - -We will discuss on agenda items and groom issues, where we plan where they fall into the release timeline. - -Occurring: Monthly on the first Wednesday - 5:30 – 6:30pm CEST. - -#### Roadmap presentation - -We will discuss the items of the roadmaps and the expected features on the next releases - -Occurring: Monthly on the second Wednesday - 5:30pm CEST. - -## Alternatives - -There are other projects that are similar to Kairos which are great and worth to mention, and actually Kairos took to some degree inspiration from. -However, Kairos have different goals and takes completely unique approaches to the underlying system, upgrade, and node lifecycle management. - -- [k3os](https://github.com/rancher/k3os) -- [Talos](https://github.com/siderolabs/talos) -- [FlatCar](https://flatcar-linux.org/) -- [CoreOS](https://getfedora.org/it/coreos?stream=stable) - -## Development - -### Building Kairos - -Requirements: Needs only Docker. - -Run `./earthly.sh +all --FLAVOR=opensuse`, should produce a Docker image along with a working ISO. - - -## What's next? - -See the [quickstart](/docs/getting-started) to install Kairos on a VM and create a Kubernetes cluster! diff --git a/docs/content/en/search-index.md b/docs/content/en/search-index.md deleted file mode 100644 index 62e6329ab..000000000 --- a/docs/content/en/search-index.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -type: "search-index" -url: "index.json" ---- diff --git a/docs/content/en/search.md b/docs/content/en/search.md deleted file mode 100644 index e3690fd5a..000000000 --- a/docs/content/en/search.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Search Results -layout: search - ---- - diff --git a/docs/layouts/404.html b/docs/layouts/404.html deleted file mode 100644 index 378b73675..000000000 --- a/docs/layouts/404.html +++ /dev/null @@ -1,10 +0,0 @@ -{{ define "main"}} -
-
-

Not found

-

Oops! This page doesn't exist. Try going back to our home page.

- -

You can learn how to make a 404 page like this in Custom 404 Pages.

-
-
-{{ end }} diff --git a/docs/layouts/index.html b/docs/layouts/index.html deleted file mode 100644 index b4912d810..000000000 --- a/docs/layouts/index.html +++ /dev/null @@ -1,75 +0,0 @@ - - - - {{ partial "head.html" . }} - - - - - - - - - -
- -
-
-
- -
-

- Edge, but better -

- The immutable Linux meta-distribution
for edge Kubernetes -
- - Documentation - - Download -
-
-
-

- With Kairos you can build immutable, bootable Kubernetes - and OS images for your edge devices as easily as writing a Dockerfile. - Optional P2P mesh with distributed ledger automates node bootstrapping - and coordination. Updating nodes is as easy as CI/CD: push a new image - to your container registry and let secure, risk-free A/B atomic upgrades - do the rest. -

-
-
-
-
-
-
What makes Kairos different?
-
section logo
The Immutable
edge factory

Let's get meta:

We call Kairos a meta-Linux Distribution. Why meta? Because it sits as a container layer, turning any Linux distro into an immutable system distributed via container registries. With Kairos, the OS is the container image, which is used for new installations and upgrades.

Bring your own OS:

The Kairos 'factory' enables you to build custom bootable-OS images for your edge devices, from your choice of OS (including openSUSE, Alpine and Ubuntu), and your choice of edge Kubernetes distribution—Kairos is totally agnostic.

Repeatable, immutable:

Each node boots from the same image, so no more snowflakes in your clusters, and each system is immutable—it boots in a restricted, permissionless mode, where certain paths are not writeable. For instance, after an installation it's not possible to install additional packages in the system, and any configuration change is discarded after a reboot. This dramatically reduces the attack surface and the impact of malicious actors gaining access to the device.

section logo
Welcome to the
self-driving edge

Zero touch provisioning:

Keeping simplicity while providing complex solutions is a key factor of Kairos. Onboarding of nodes can be done via QR code, manually, remotely via SSH, interactively, or completely automated with Kubernetes, with zero touch provisioning.

Secure peer to peer mesh with VPN:

Kairos optionally supports P2P full-mesh out of the box. New devices wake up with a shared secret and distributed ledger of other nodes and clusters to look for—they form a unified overlay network that’s E2E encrypted to discover other devices, even spanning multiple networks, to bootstrap the cluster.

section logo
Containerize your
lifecycle management

Treat your OS just like any app:

Each Kairos OS is created as easily as writing a Dockerfile—no custom recipes or arcane languages here. You can run and customize the container images locally with Docker, Podman, or your container engine of choice exactly how you do for apps already.

Run your pipeline to the edge:

Your built OS is a container-based, single image that is distributed via container registries, so it plugs neatly into your existing CI/CD pipelines. It makes edge scale as repeatable and portable as driving containers. Customizing, mirroring of images, scanning vulnerabilities, gating upgrades, patching CVEs are some of the endless possibilities. Updating nodes is just as easy as selecting a new version via Kubernetes. Each node will pull the update from your repo, installing on A/B partitions for zero-risk upgrades with failover.

Run K8s with K8s:

Use Kubernetes management principles to manage and provision your clusters. Kairos supports automatic node provisioning via CRDs; upgrade management via Kubernetes; node repurposing and machine auto scaling capabilities; and complete configuration management via cloud-init.

section logo
A community soul

On the shoulders of giants:

Kairos draws on the strength of the cloud-native ecosystem, not just for principles and approaches, but components. Cluster API is optionally supported as well, and can be used to manage Kubernetes clusters using native Kubernetes APIs with zero touch provisioning.

Extensively tested:

We move fast, but we try not to break stuff—particularly your nodes. Every change in the Kairos codebase runs through highly engineered automated testing before release to catch bugs earlier.

Designed for everyone:

While Kairos has been engineered for large-scale use by DevOps and IT Engineering teams working in cloud, bare metal, edge and embedded systems environments, we welcome makers, hobbyists, and anyone in the community to participate in driving forward our vision of the immutable, decentralized, and containerized edge.

Backed by Spectro Cloud:

Kairos is a vibrant, active project with time and financial backing from Spectro Cloud, a Kubernetes management platform provider with a strong commitment to the open source community. It is a silver member of the CNCF and LF Edge, a Certified Kubernetes Service Provider, and a contributor to projects such as Cluster API. Find more about Spectro Cloud here.

section logo

See how to get up and running with Kairos, in less than 15 minutes!

section logo

Get inside Kairos, from the factory to P2P mesh capabilities.

section logo

Stretch your wings with best practices of common tasks after installing Kairos.

Join our community
section logo
Contribute on GitHub

section logo
Documentation

-
-
-
- footer logo -
- - {{ partialCached "scripts.html" . }} - - diff --git a/docs/layouts/shortcodes/armFlavor.html b/docs/layouts/shortcodes/armFlavor.html deleted file mode 100644 index ba680d36b..000000000 --- a/docs/layouts/shortcodes/armFlavor.html +++ /dev/null @@ -1 +0,0 @@ -{{$.Page.Site.Params.softwareVersions.armFlavor}} \ No newline at end of file diff --git a/docs/layouts/shortcodes/flavor.html b/docs/layouts/shortcodes/flavor.html deleted file mode 100644 index b51bf67cb..000000000 --- a/docs/layouts/shortcodes/flavor.html +++ /dev/null @@ -1 +0,0 @@ -{{$.Page.Site.Params.softwareVersions.flavor}} \ No newline at end of file diff --git a/docs/layouts/shortcodes/githubembed.html b/docs/layouts/shortcodes/githubembed.html deleted file mode 100644 index 0c819ba18..000000000 --- a/docs/layouts/shortcodes/githubembed.html +++ /dev/null @@ -1,9 +0,0 @@ -{{/* https://github.com/haideralipunjabi/hugo-shortcodes/tree/master/github */}} - -{{ $dataJ := getJSON "https://api.github.com/repos/" (.Get "repo") "/contents/" (.Get "file") }} -{{ $con := base64Decode $dataJ.content }} - -{{ highlight $con (.Get "lang") (.Get "options") }} - - Complete source code: {{ print "https://github.com/" ( .Get "repo" ) "/blob/master/" (.Get "file" ) }} -
diff --git a/docs/layouts/shortcodes/k3sVersion.html b/docs/layouts/shortcodes/k3sVersion.html deleted file mode 100644 index 168167c81..000000000 --- a/docs/layouts/shortcodes/k3sVersion.html +++ /dev/null @@ -1 +0,0 @@ -{{$.Page.Site.Params.softwareVersions.k3s}} \ No newline at end of file diff --git a/docs/layouts/shortcodes/kairosVersion.html b/docs/layouts/shortcodes/kairosVersion.html deleted file mode 100644 index 9af76c92d..000000000 --- a/docs/layouts/shortcodes/kairosVersion.html +++ /dev/null @@ -1 +0,0 @@ -{{$.Page.Site.Params.softwareVersions.kairos}} \ No newline at end of file diff --git a/docs/layouts/shortcodes/registryURL.html b/docs/layouts/shortcodes/registryURL.html deleted file mode 100644 index ef1454539..000000000 --- a/docs/layouts/shortcodes/registryURL.html +++ /dev/null @@ -1 +0,0 @@ -{{$.Page.Site.Params.softwareVersions.registryURL}} \ No newline at end of file diff --git a/docs/netlify.toml b/docs/netlify.toml deleted file mode 100644 index 64edbae90..000000000 --- a/docs/netlify.toml +++ /dev/null @@ -1,10 +0,0 @@ -# Hugo build configuration for Netlify -# (https://gohugo.io/hosting-and-deployment/hosting-on-netlify/#configure-hugo-version-in-netlify) - -[[redirects]] -from = "/docs/contribution-guidelines/" -to = "https://github.com/kairos-io/kairos/blob/master/CONTRIBUTING.md" -status = 200 -force = true # COMMENT: ensure that we always redirect -headers = {X-From = "Netlify"} -signed = "API_SIGNATURE_TOKEN" diff --git a/docs/package-lock.json b/docs/package-lock.json deleted file mode 100644 index d852b477f..000000000 --- a/docs/package-lock.json +++ /dev/null @@ -1,1599 +0,0 @@ -{ - "name": "docs", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "devDependencies": { - "autoprefixer": "^10.4.13", - "postcss": "^8.4.21", - "postcss-cli": "^10.1.0" - } - }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/anymatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", - "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "dev": true, - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/autoprefixer": { - "version": "10.4.14", - "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.14.tgz", - "integrity": "sha512-FQzyfOsTlwVzjHxKEqRIAdJx9niO6VCBCoEwax/VLSoQF29ggECcPuBqUMZ+u8jCZOPSy8b8/8KnuFbp0SaFZQ==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/autoprefixer" - } - ], - "dependencies": { - "browserslist": "^4.21.5", - "caniuse-lite": "^1.0.30001464", - "fraction.js": "^4.2.0", - "normalize-range": "^0.1.2", - "picocolors": "^1.0.0", - "postcss-value-parser": "^4.2.0" - }, - "bin": { - "autoprefixer": "bin/autoprefixer" - }, - "engines": { - "node": "^10 || ^12 || >=14" - }, - "peerDependencies": { - "postcss": "^8.1.0" - } - }, - "node_modules/binary-extensions": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, - "dependencies": { - "fill-range": "^7.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/browserslist": { - "version": "4.21.5", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz", - "integrity": "sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - } - ], - "dependencies": { - "caniuse-lite": "^1.0.30001449", - "electron-to-chromium": "^1.4.284", - "node-releases": "^2.0.8", - "update-browserslist-db": "^1.0.10" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001472", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001472.tgz", - "integrity": "sha512-xWC/0+hHHQgj3/vrKYY0AAzeIUgr7L9wlELIcAvZdDUHlhL/kNxMdnQLOSOQfP8R51ZzPhmHdyMkI0MMpmxCfg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ] - }, - "node_modules/chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, - "node_modules/dependency-graph": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz", - "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==", - "dev": true, - "engines": { - "node": ">= 0.6.0" - } - }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "dependencies": { - "path-type": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/electron-to-chromium": { - "version": "1.4.284", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", - "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", - "dev": true - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true - }, - "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/fast-glob": { - "version": "3.2.12", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", - "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", - "dev": true, - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fastq": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", - "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", - "dev": true, - "dependencies": { - "reusify": "^1.0.4" - } - }, - "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/fraction.js": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", - "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==", - "dev": true, - "engines": { - "node": "*" - }, - "funding": { - "type": "patreon", - "url": "https://www.patreon.com/infusion" - } - }, - "node_modules/fs-extra": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.0.tgz", - "integrity": "sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw==", - "dev": true, - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=14.14" - } - }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-stdin": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-9.0.0.tgz", - "integrity": "sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/globby": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", - "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", - "dev": true, - "dependencies": { - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.11", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", - "dev": true - }, - "node_modules/ignore": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", - "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", - "dev": true, - "engines": { - "node": ">= 4" - } - }, - "node_modules/is-binary-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, - "dependencies": { - "binary-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/jsonfile": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", - "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - "dev": true, - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/lilconfig": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.0.6.tgz", - "integrity": "sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg==", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "dev": true, - "dependencies": { - "braces": "^3.0.2", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/nanoid": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/node-releases": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz", - "integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==", - "dev": true - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/normalize-range": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", - "integrity": "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", - "dev": true - }, - "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postcss": { - "version": "8.4.23", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.23.tgz", - "integrity": "sha512-bQ3qMcpF6A/YjR55xtoTr0jGOlnPOKAIMdOWiv0EIT6HVPEaJiJB4NLljSbiHoC2RX7DN5Uvjtpbg1NPdwv1oA==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "dependencies": { - "nanoid": "^3.3.6", - "picocolors": "^1.0.0", - "source-map-js": "^1.0.2" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, - "node_modules/postcss-cli": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/postcss-cli/-/postcss-cli-10.1.0.tgz", - "integrity": "sha512-Zu7PLORkE9YwNdvOeOVKPmWghprOtjFQU3srMUGbdz3pHJiFh7yZ4geiZFMkjMfB0mtTFR3h8RemR62rPkbOPA==", - "dev": true, - "dependencies": { - "chokidar": "^3.3.0", - "dependency-graph": "^0.11.0", - "fs-extra": "^11.0.0", - "get-stdin": "^9.0.0", - "globby": "^13.0.0", - "picocolors": "^1.0.0", - "postcss-load-config": "^4.0.0", - "postcss-reporter": "^7.0.0", - "pretty-hrtime": "^1.0.3", - "read-cache": "^1.0.0", - "slash": "^5.0.0", - "yargs": "^17.0.0" - }, - "bin": { - "postcss": "index.js" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "postcss": "^8.0.0" - } - }, - "node_modules/postcss-cli/node_modules/slash": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-5.0.0.tgz", - "integrity": "sha512-n6KkmvKS0623igEVj3FF0OZs1gYYJ0o0Hj939yc1fyxl2xt+xYpLnzJB6xBSqOfV9ZFLEWodBBN/heZJahuIJQ==", - "dev": true, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/postcss-load-config": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.1.tgz", - "integrity": "sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA==", - "dev": true, - "dependencies": { - "lilconfig": "^2.0.5", - "yaml": "^2.1.1" - }, - "engines": { - "node": ">= 14" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - "peerDependencies": { - "postcss": ">=8.0.9", - "ts-node": ">=9.0.0" - }, - "peerDependenciesMeta": { - "postcss": { - "optional": true - }, - "ts-node": { - "optional": true - } - } - }, - "node_modules/postcss-reporter": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/postcss-reporter/-/postcss-reporter-7.0.5.tgz", - "integrity": "sha512-glWg7VZBilooZGOFPhN9msJ3FQs19Hie7l5a/eE6WglzYqVeH3ong3ShFcp9kDWJT1g2Y/wd59cocf9XxBtkWA==", - "dev": true, - "dependencies": { - "picocolors": "^1.0.0", - "thenby": "^1.3.4" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - "peerDependencies": { - "postcss": "^8.1.0" - } - }, - "node_modules/postcss-value-parser": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", - "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", - "dev": true - }, - "node_modules/pretty-hrtime": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz", - "integrity": "sha512-66hKPCr+72mlfiSjlEB1+45IjXSqvVAIy6mocupoww4tBFE9R9IhwwUGoI4G++Tc9Aq+2rxOt0RFU6gPcrte0A==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/read-cache": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", - "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", - "dev": true, - "dependencies": { - "pify": "^2.3.0" - } - }, - "node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true, - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, - "node_modules/slash": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", - "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/source-map-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/thenby": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/thenby/-/thenby-1.3.4.tgz", - "integrity": "sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==", - "dev": true - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/universalify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", - "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", - "dev": true, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/update-browserslist-db": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", - "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - } - ], - "dependencies": { - "escalade": "^3.1.1", - "picocolors": "^1.0.0" - }, - "bin": { - "browserslist-lint": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/yaml": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.3.tgz", - "integrity": "sha512-AacA8nRULjKMX2DvWvOAdBZMOfQlypSFkjcOcu9FalllIDJ1kvlREzcdIZmidQUqqeMv7jorHjq2HlLv/+c2lg==", - "dev": true, - "engines": { - "node": ">= 14" - } - }, - "node_modules/yargs": { - "version": "17.6.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", - "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", - "dev": true, - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, - "engines": { - "node": ">=12" - } - } - }, - "dependencies": { - "@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "requires": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - } - }, - "@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true - }, - "@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "requires": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - } - }, - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "requires": { - "color-convert": "^2.0.1" - } - }, - "anymatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", - "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "dev": true, - "requires": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - } - }, - "autoprefixer": { - "version": "10.4.14", - "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.14.tgz", - "integrity": "sha512-FQzyfOsTlwVzjHxKEqRIAdJx9niO6VCBCoEwax/VLSoQF29ggECcPuBqUMZ+u8jCZOPSy8b8/8KnuFbp0SaFZQ==", - "dev": true, - "requires": { - "browserslist": "^4.21.5", - "caniuse-lite": "^1.0.30001464", - "fraction.js": "^4.2.0", - "normalize-range": "^0.1.2", - "picocolors": "^1.0.0", - "postcss-value-parser": "^4.2.0" - } - }, - "binary-extensions": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "dev": true - }, - "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, - "requires": { - "fill-range": "^7.0.1" - } - }, - "browserslist": { - "version": "4.21.5", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz", - "integrity": "sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==", - "dev": true, - "requires": { - "caniuse-lite": "^1.0.30001449", - "electron-to-chromium": "^1.4.284", - "node-releases": "^2.0.8", - "update-browserslist-db": "^1.0.10" - } - }, - "caniuse-lite": { - "version": "1.0.30001472", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001472.tgz", - "integrity": "sha512-xWC/0+hHHQgj3/vrKYY0AAzeIUgr7L9wlELIcAvZdDUHlhL/kNxMdnQLOSOQfP8R51ZzPhmHdyMkI0MMpmxCfg==", - "dev": true - }, - "chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, - "requires": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "fsevents": "~2.3.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - } - }, - "cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, - "dependency-graph": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz", - "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==", - "dev": true - }, - "dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "requires": { - "path-type": "^4.0.0" - } - }, - "electron-to-chromium": { - "version": "1.4.284", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", - "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", - "dev": true - }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true - }, - "escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "dev": true - }, - "fast-glob": { - "version": "3.2.12", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", - "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", - "dev": true, - "requires": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - } - }, - "fastq": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", - "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", - "dev": true, - "requires": { - "reusify": "^1.0.4" - } - }, - "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, - "requires": { - "to-regex-range": "^5.0.1" - } - }, - "fraction.js": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", - "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==", - "dev": true - }, - "fs-extra": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.0.tgz", - "integrity": "sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw==", - "dev": true, - "requires": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - } - }, - "fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "optional": true - }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true - }, - "get-stdin": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-9.0.0.tgz", - "integrity": "sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA==", - "dev": true - }, - "glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "requires": { - "is-glob": "^4.0.1" - } - }, - "globby": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", - "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", - "dev": true, - "requires": { - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.11", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^4.0.0" - } - }, - "graceful-fs": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", - "dev": true - }, - "ignore": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", - "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", - "dev": true - }, - "is-binary-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, - "requires": { - "binary-extensions": "^2.0.0" - } - }, - "is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true - }, - "is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "requires": { - "is-extglob": "^2.1.1" - } - }, - "is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true - }, - "jsonfile": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", - "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - "dev": true, - "requires": { - "graceful-fs": "^4.1.6", - "universalify": "^2.0.0" - } - }, - "lilconfig": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.0.6.tgz", - "integrity": "sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg==", - "dev": true - }, - "merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true - }, - "micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "dev": true, - "requires": { - "braces": "^3.0.2", - "picomatch": "^2.3.1" - } - }, - "nanoid": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", - "dev": true - }, - "node-releases": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz", - "integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==", - "dev": true - }, - "normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true - }, - "normalize-range": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", - "integrity": "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==", - "dev": true - }, - "path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true - }, - "picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", - "dev": true - }, - "picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true - }, - "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - "dev": true - }, - "postcss": { - "version": "8.4.23", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.23.tgz", - "integrity": "sha512-bQ3qMcpF6A/YjR55xtoTr0jGOlnPOKAIMdOWiv0EIT6HVPEaJiJB4NLljSbiHoC2RX7DN5Uvjtpbg1NPdwv1oA==", - "dev": true, - "requires": { - "nanoid": "^3.3.6", - "picocolors": "^1.0.0", - "source-map-js": "^1.0.2" - } - }, - "postcss-cli": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/postcss-cli/-/postcss-cli-10.1.0.tgz", - "integrity": "sha512-Zu7PLORkE9YwNdvOeOVKPmWghprOtjFQU3srMUGbdz3pHJiFh7yZ4geiZFMkjMfB0mtTFR3h8RemR62rPkbOPA==", - "dev": true, - "requires": { - "chokidar": "^3.3.0", - "dependency-graph": "^0.11.0", - "fs-extra": "^11.0.0", - "get-stdin": "^9.0.0", - "globby": "^13.0.0", - "picocolors": "^1.0.0", - "postcss-load-config": "^4.0.0", - "postcss-reporter": "^7.0.0", - "pretty-hrtime": "^1.0.3", - "read-cache": "^1.0.0", - "slash": "^5.0.0", - "yargs": "^17.0.0" - }, - "dependencies": { - "slash": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-5.0.0.tgz", - "integrity": "sha512-n6KkmvKS0623igEVj3FF0OZs1gYYJ0o0Hj939yc1fyxl2xt+xYpLnzJB6xBSqOfV9ZFLEWodBBN/heZJahuIJQ==", - "dev": true - } - } - }, - "postcss-load-config": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.1.tgz", - "integrity": "sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA==", - "dev": true, - "requires": { - "lilconfig": "^2.0.5", - "yaml": "^2.1.1" - } - }, - "postcss-reporter": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/postcss-reporter/-/postcss-reporter-7.0.5.tgz", - "integrity": "sha512-glWg7VZBilooZGOFPhN9msJ3FQs19Hie7l5a/eE6WglzYqVeH3ong3ShFcp9kDWJT1g2Y/wd59cocf9XxBtkWA==", - "dev": true, - "requires": { - "picocolors": "^1.0.0", - "thenby": "^1.3.4" - } - }, - "postcss-value-parser": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", - "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", - "dev": true - }, - "pretty-hrtime": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz", - "integrity": "sha512-66hKPCr+72mlfiSjlEB1+45IjXSqvVAIy6mocupoww4tBFE9R9IhwwUGoI4G++Tc9Aq+2rxOt0RFU6gPcrte0A==", - "dev": true - }, - "queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true - }, - "read-cache": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", - "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", - "dev": true, - "requires": { - "pify": "^2.3.0" - } - }, - "readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "requires": { - "picomatch": "^2.2.1" - } - }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true - }, - "reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true - }, - "run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "requires": { - "queue-microtask": "^1.2.2" - } - }, - "slash": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", - "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", - "dev": true - }, - "source-map-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", - "dev": true - }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "requires": { - "ansi-regex": "^5.0.1" - } - }, - "thenby": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/thenby/-/thenby-1.3.4.tgz", - "integrity": "sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==", - "dev": true - }, - "to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "requires": { - "is-number": "^7.0.0" - } - }, - "universalify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", - "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", - "dev": true - }, - "update-browserslist-db": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", - "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", - "dev": true, - "requires": { - "escalade": "^3.1.1", - "picocolors": "^1.0.0" - } - }, - "wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true - }, - "yaml": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.3.tgz", - "integrity": "sha512-AacA8nRULjKMX2DvWvOAdBZMOfQlypSFkjcOcu9FalllIDJ1kvlREzcdIZmidQUqqeMv7jorHjq2HlLv/+c2lg==", - "dev": true - }, - "yargs": { - "version": "17.6.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", - "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", - "dev": true, - "requires": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - } - }, - "yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true - } - } -} diff --git a/docs/package.json b/docs/package.json deleted file mode 100644 index 5a37fe0e4..000000000 --- a/docs/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "devDependencies": { - "autoprefixer": "^10.4.13", - "postcss": "^8.4.21", - "postcss-cli": "^10.1.0" - }, - "scripts": { - "get:submodule": "git submodule update --init --depth 1", - "_prepare:docsy": "cd themes/docsy && npm install", - "prepare": "npm run get:submodule && npm run _prepare:docsy" - } -} diff --git a/docs/scripts/build.sh b/docs/scripts/build.sh deleted file mode 100755 index 50344bad7..000000000 --- a/docs/scripts/build.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -e - -BASE_URL="${BASE_URL:-https://kairos.io}" - -binpath="${ROOT_DIR}/bin" -publicpath="${ROOT_DIR}/public" -export PATH=$PATH:$binpath - -if [ -z "$(type -P hugo)" ]; -then - [[ ! -d "${binpath}" ]] && mkdir -p "${binpath}" - wget https://github.com/gohugoio/hugo/releases/download/v"${HUGO_VERSION}"/hugo_extended_"${HUGO_VERSION}"_"${HUGO_PLATFORM}".tar.gz -O "$binpath"/hugo.tar.gz - tar -xvf "$binpath"/hugo.tar.gz -C "${binpath}" - rm -rf "$binpath"/hugo.tar.gz - chmod +x "$binpath"/hugo -fi - -rm -rf "${publicpath}" || true -[[ ! -d "${publicpath}" ]] && mkdir -p "${publicpath}" - -npm install --save-dev autoprefixer postcss-cli postcss - -HUGO_ENV="production" hugo --buildFuture --gc -b "${BASE_URL}" -d "${publicpath}" - -cp -rf CNAME "${publicpath}" diff --git a/docs/scripts/publish.sh b/docs/scripts/publish.sh deleted file mode 100755 index ba50c82fb..000000000 --- a/docs/scripts/publish.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -e - -"${ROOT_DIR}"/scripts/build.sh - -git branch -D gh-pages || true - -git checkout --orphan gh-pages - -git rm -rf . - -cp -rfv public/* ./ -rm -rf public/ - diff --git a/docs/scripts/serve.sh b/docs/scripts/serve.sh deleted file mode 100755 index 7d0c3c200..000000000 --- a/docs/scripts/serve.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -binpath="${ROOT_DIR}/bin" -export PATH=$PATH:$binpath - -if [ -z "$(type -P hugo)" ]; -then - [[ ! -d "${binpath}" ]] && mkdir -p "${binpath}" - wget https://github.com/gohugoio/hugo/releases/download/v"${HUGO_VERSION}"/hugo_extended_"${HUGO_VERSION}"_"${HUGO_PLATFORM}".tar.gz -O "$binpath"/hugo.tar.gz - tar -xvf "$binpath"/hugo.tar.gz -C "${binpath}" - rm -rf "$binpath"/hugo.tar.gz - chmod +x "$binpath"/hugo -fi - -hugo --baseURL="$BASE_URL" -s "$ROOT_DIR" serve diff --git a/docs/static/favicons/android-chrome-192x192.png b/docs/static/favicons/android-chrome-192x192.png deleted file mode 100644 index c7693d000ee831245ec293e88f7e1d9219fd09e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14851 zcmZ`=WmFqYxDFw>yK8YT?(P(dyM*HIZUKsG(c)6HxEHq;w-$FOTHH0j&G+x#ANTCp zos-#}dFSlByO~EeMqNz-1C zOjt7D8Ow3UrBsyamwr#F(A4{Qd$Z)Z>N-^B^nG%F9Qx(41j}NU-^AqI{2T)Ta}2Fq zEN*Ja*HO!}=dd88Pn&Bm>yOym$O_az2WnbulHm!I6#aqo>wpcB`<~--pi}d2d9wd~ zxcm$u4T7J9v~30l%nL#vh9&mpH?>w5p0;A>B-`<-qq@(iKc>{I_T$MxxNQm>>-NA0 zi{1O=usV@`JShn3l={C$bBRC$=)Kq;`D%R&bcLsN0m!Zj@yeAFt%BM4Oh{>Rb4AQU&h5zk>l2#rFr-ONiAFIZ7Q4x^p$2iKvs|96w1c!AUPq{6g z+`e4=YF2-ZBv3r(Gjd4Cc-O~}E5&X~3OG%tY;%#CAK6cWL7G- zX_w{HnDf5?Cb|Rn)>nU7!-ob65UOtkOo}Lz+K#c&i^TF%o;FRnp}l^bM1E`Wt!Mm( zVE_Va>GHD{8z~CNcun$w(aTZH7mnLrG>4lGh8d&Rd*YFD)gIxj42b}r0qC#pA1%>i zQyStjXB{4nzjTRm@)xxhBR?KczA>?BZE7zcA@ckyGWG4m4E>CXV)>`qxS;+fh_&vDqT@uC>x&pFYOm64s@U5e1c~abG@U z1>VN_9lt?6H@bxKzPxu#L2X4A4m_|+pbN@x9lg#1)l%gxN?;Hb+2hB^L1M8kvzCa%C zP17SQFS=C+)AVzv$9!Dm_ogMj^4AWdtC`n$oWxph6xW=UyW@#*S~<{&OOLKUu=<@(+o#*&(v}nEf@#Z1fx;3( z`RAVl3r*wtw6)|uTQFDpc7cgObMVV+^Cdu%5P^2in8#&)p*|g@>q`f80LS8~COO^T z1k#<3VNd|L`)H-VR*+U3^1wvpe~&gOq0`(Vs$bg)s^+^xhvQEMsGU}I5?u9clv9S( zYCTGo8wIKm9oD7SOpKI&=(*!!emgs`7;-7&=8iq;-)ZH&)^$YKdi6hA8TD9xe&eT` zJ@DdRZ1eH}0RZ}i+NgiWzV5wdhKzZ?`VahZaCZ3i`v$Z`J%8}wx_u4RFLJQ>{y1o* z0Mq{;Qb!@Y>`dEj|8Y3GJZ=7g#9Fjr$n)>`$I^G=T=kO=VAfWkYu>mxao~H8`=o%$ z!-CEw!H(kV#qR4#yuL_;SoX%JSF6GM)XhDaw*AtKK6~1e<(Kd>`aWH+SDN0#hpEE5 zTlnBh_+IG)P`^&p>Td2&Pa5(1jczWR^{nYJ5PxMBD_E;Ff8^s`S91Gv3ysBVRB%>M zeqTXCZQv=9`n~)1ssSNMHN#83N)A6g+=9~WGG@bHbPA5EUZKKn zSKbP!(oea~rI@Krj@ZloZ!!39yWL7Vd1UV$2y^U;d&uDEB#g_80XLT0n*yp27hi10 zEPZP+y)_CKzs#gePju$1X&(ot;+RZbD(Gd_6aCFLx!FRv|q@i z(QWxRs1GX5xB}CJ-m(p_A>*aV_W|K6Sz0)IQ8;-$MW~*DG|J( z>GPo)gfvC4NVrp5ZM|o2R)_DMtn<~gd6vJ7RaI;284xbFmBHZAUb-XrPDA|RRWEWg zF(NmBey>4YMyc=w`q7aJb9o0Hdcs>^L~tRtSxWhE{O;+^lePlNDV1kR2s-zcsKcR_ zJPdo178VQOt`_Cn)ruc&HvW*qTB%9|Xq$>W&stlWxlb*4>D+l7ePgHQ;nZ25SfRmh zqskTl(l%U6=vr6B{g9xY__AYvcbGh4O{F{?n5Y*tMQYRQ`R|YO*Y@@X+a*SLViFR! zDJZLuvufY7ak=n=&2!6n&)m1y^Sq9SM3$Kt2K|OsG$~g3H6649o9Yb5bHtnZfJe|{#j0b1xP0l z-pSIk#GKCdtBg>|$H$5DM=HeXOtm|NSDW5gy|_oH*zto-pFzxJ!`@ne)?&|B!DP}P zKH5}HX115kx6w|<(C;f_f6sW2L+;(gZ?4fE`{K>QF7gqd}^ppPVR zZ!}-Idziz`2RJ4*6A)7KjY&zf+n*)wO3tkaSdSfUC{AfdRIh#pkG<4QSBzG|I-*@5 z#KisXf_cV5Dh)w>3>LZYQtFe*(pH=(i%wA*QAAog{qkkbV z?Ih7;5K5M{`;bQ=ClgjbD(yN4#;&DfnNNNMEKoKU$%7HIK6vMr7hg=AVtfP`iTte6 zwe>_-?|CfcE_9>+wC{m^-L$2ROO7ss7zX(>L$?r_9Aian(PpkD_W8pwX^=<)q5*Yh zm&~uDZ-%6;sJz(#$p}@h+E#8Q6y={%S-ebnpW{8l*7F&%RZ*xRP0YlpSYWla2!b4> z635~IM{>H6jcp&{5SicTAi&sPI8zsJG+-Pkmsv!Ynp}h}_y=X-SmUKD9|4#%b(jVmoa!Kh0i3W&Phy52!Lm;183co;`en&QU6^3L0S}XA}N(GfQUPjLs-E< zNPdaO=1Xu61G|ha?NG2qNsi6m)^l+$}>r zi7<-(?_NeWQ}U8{#FrZ$$`5BqijEU4NhXTubn%1QrT&JUw(m6bWN!A`?XPIs~Pu``o5+;s_Fl?Olf9x<&8Fsq@bsuGW#QfNbfx;J+ibH~^lu zZK!FxVbSW1Ly&Sw)OB1-;4`MNaFW6@G9~4Yhml}C0>8r16nfrBk%?lqydf)c=9i2Z zrSW!PE-zzTJxGqn6?&ptk|wmeF3vDWqDejssy}1Nh+N=*);e$@k_8uFom@gvV#3&z z%k8z)mn?1Zz*ynh6i0nHwlah2(k77Ng$1PurXeJLoD4$_aa%;7s9$?q9)zUC9-AU) z21^{HG@!oX+V9@lb`l}+FxiyK)~?}VLoRNl*qU0BEGM+)dT?DGDe=GIx%(+^uC8No zLTC37?{CAyXw*GPXi1Rn{=DZSm|^J42ZVRQ*@h1nVUUL-%g@wDq#~bOhp$t*{@IXF zH0NQ==CuI*W{~6wCn@PHx}*5q`UXl%>{&U6o=R)_WKnIpnNEkKFqTSV9V71Um9&IA;+ah^%F9Sr%Jn%xGT<3le}^$}bZmD0 z#H<&*P!(9TaS*;Rxy=9jc~P8NZ+X@(!U>{{anv(-5#I&3coC;gE0)kxnkS`!YDMsj zNYm-xpN$D%^V7i+Jn%5{GBg%bsz1i92-|fN?r9>)$CSJtk54Ko6=?O-3Ia>%|JgK0 zOe*k#C2Yi1g!SyBNa#4i@;n9zxMll6gJ@gIx%92U6l(ADwathD1pvKUb%9@cseRvh zWKH1zrt2Ok0H}*gm|0%?mReU{*W12HKgFC~tJ!@+gJUTU|E{I^b|uS+w(KFpxg*MX z{WN4dzs)B(kYlo39qms(vN~V#boo;drAwy5C_RR0G)eA|{jk}s(cR*gFF5h>@iWf1 z*sBCcM&&!5>&Sy=QOAf|V)+H+fqX&NAD<@o1iHBcn0}m>lsyXfL3RgNSvplfw8W!w zCHrRtrm@R%yGoPd3;t(8+d{#6Wj!Ivs1eU=yCDrmLf*o~qVvZ%@LunN;lYYO54f)e z_N`UjmUx{%^3F5l&BHeZ4g zzg&#v(J&nuU0*1!oX`Wh&FHc4F<8W6ZP*@nsj;G~!F%!RLD`J$NIxnj3437=ddChU5Z74jk(su{m4ZH{mOyeN_c_9_(3 zNc9p6ZkG4>vN5T_Fl8$@jys`z`RrAJM2^7bA5+#DF95yW zM7{mMgIR%m;dHV9RyMp%9fL@LXTN|10PcV?^7LP^pv{)eC$*4{LeM_Ru4iDt-88rb z+CaWzXEQXWR4)yxQ!cU*N!#J{s^UDmkNL*V9tYsDxH^4Cc{F?FaxwYQWjR%dFQSto z*LYKv!#Gez?v5wC8SE$KF{+9ALY_KOq_pjraU9b>BnoYZK2&; zEJ#2HD=wcuk2243iR)(qtP-)({$#5Sn=Q`>J5G4NY8*Dsw=v!T}+duEDlU&P>?RqYMUAYYP@NgQOFJyt^uUI$ucHNuN9<={&Z7Jkt zWX%A*;pPw`&M9jL*7kE&?8sv6=-y}B(>(>q*MCd z+nKOKeGIdPCX54{6q6ar^h~Efgplbck_hDw1_XHdp}$cccER^Ab~^=UzxEs(WkCUf zd7)R4vidv3WFvVxSPaxWJ$irIB0ugkzBbRs_Et8`hm(SACVv3A6d58!{Eeu7Kt;|g zr-|nJx;`tyR)ap)e*FGi>G=xtF^Gx0%wy488ca8z+OFmn{paeu+J8Y*`b>J^syV-Y z`Dhe)?WXyyEe!dPaXkat}rx8#n+@yTN z?3!yK^o57t{A9C@sybDh0WfPxoRsRmEkD;$hiw%|`q+i~7-Pt3Bny%wnp@n)oyLet zgGYLio+|TWwrdP3@uAN_0m#eCZ&kAzKK5gvF6si;I9PF@JG;N?sJQlD$5^k+5iBP& z*Q^cTK4``p&_fR!pGT;AJgIuJ2y2;?d3sJyrT{(){mZ(k7)+zZ=|S*-U>#~#-olfR zS`|NQ!IE5*<2jw){%R3H&i7&k>AgR=Y+i1@k?i2GNW|$(_fx*56M$j3ECtg(+sM6bhUYtqw?0Blko1?$@&zOUNwKkAA(Pk^#YP&P$`eKIaI`(uE6F9gQ({A_^k zw)syUn%Ozte##UuKWYIgCev}@;E;^U(H)$*h0PxZb-%4pi4JqZ{?Hg=9|I4pY`k;1 z%lFB6jn-XdnVmZ9`KYP4beSI9+uV;LoNps~2sx{g;{!#f3q;jMLmn&_x?FjzC(Tb^ zLuK6vm_BN)gjd12ilvhsKgvI?V0BJB7;*5cISSM)S7Rss?WlGwJ?})MGS)=mG;yQq zct>+W(jz*!lO{su{M)5X?bmBJEzU{rHgwh7amyIaJ&_fSdR&sW#bdW#hfI8gs|rnLU3k_$|_@ zkD}ev`cGsC+-tUj-UJiTc102QKm`d>i%mggsT!VoFKZv~ZpA973;t#@OBY|SF6Odq zJxZ2B42bn!> zt>g$OF`|%VLlBGzn_aD(1TAE6isXeXQf$VsG>&XMRyWfMTHqE6_E&xu{hHNC!Ze!- z?xo^hgS$iEKgnB5qh_Jw#MK~9y$8py!`Y+&h&6rj@jtzLCRxGj2IBehSeV&QJ3_Ed}Nf>MKS5qS(2%V5j)od z4#AkfgC;J9v;N#4Xsp7$_MA~@s+-1gt2%;e1%QQo)a^CsA2!??W^7gnfEjh%veh>o zsZ(b*JE7yi>u5OqF9xvpB)CLog4vUw;nPrubAL*nsSQ0R;N$A zbW9x}`g9P|VDdKd(maGzgyk^M;|M2yw6mVDytSV8%e5zNT1buTzcRs=whgAnZet0( zKeMVBNK&lnnp}X56&d{JVEG!L&6Os!;*eJH3gGbDU%60LrJ*oSs_;cJg+ znhbRJA%y+}C2F(G&ZsScByAGnF61hVu>4ymS#FHua}10>fRQ!3R3=eEFMwPXqFy~C z65V`6Cku)boX4iPLLg2G3e!J)L1LDlNa9rX`{J_6qf8T7!w7j)W-LK>iOyZz*+SFv z@=h8?SIQ`{95OIo;6)0CAgepQy~E`(_tymDf9J^MxwyE@fe7Bza{x{!=97t-6 z{o86IVAnV+0+1r!!WH>dQf~oam2da)ITdzj zRu-O$W&CF7dpE3HRl?w&#LgKQJk=1XXZwki<@sLxkO{kpTsHY1=&I=(7v<(BIzrnB zCNGYDTUduHgN>3Zalf956^nWlIWy9g&|ss3MU~;0#GVM0IZn)u1589TXZwC);W8wW zWwj#6jh2xY29E_6eeeq=&s8M|F6)=`nUsPQpW(f7&GpC>_PpEDCQVZVX3Sqm*<0jL zv!v^k>~IlM_D4_7n;g_d4dFpZcHL!8Ny5Aj@zY#6B|)r?Qa_6em$;cyoo`H98?+Qr zz9RB}U#}D(RRI)9o{~cNJ#m%q0gCr&jfkEc_l4H*s>GO?;ZVw4QZc2?>MK1^{Os>C&Vpgv{Xa{ zBu5{(@{?*Hmi0tQo16=oTd^fM5r>N|^u&-{4FTFuV~?D!`f-}dv3J$%-iH3wP-jEs zzVae%ole=p!(#8lg3h}x@v2YS|JAa zRfZ1%uIF@hC`1xKrPS&D;Dr5M+#|_5t{nMs7T}Z@QSpu}Ylhin1?UtEA zL~}2{!L*|oJB$Dzh1DB#MEj!SCxyB%@B3wSCX~rKAxi?4f8eczmcy_-sJ%PELv$Sm zPT{YRB!62)>o@1|vln)g>q4$y8f*LcWV=ayzYzLo-%wVDmEIj759o$LY-Sn@7 z6uRD+VW{yki8Qn)bccUG_pECJa_Q`VRFYSbTJ9yyi3HY+m_F^8%)sz3Hn-V4|n=6SQzi z^S$^;&HL;{N+hk|4Y&QJHk6CBrk6?=cbTf? zMg_=;OM#e34O>LYwSYo-n?Nv1+w>f*f`NoDs&Yviuf>_pfloFIXo-p`_cM8tl{xxFWy4hE20vWxcvw(k z$@?P-L1BK^lf3@=bNT|wkKYoO2`We}6^Nix4qGB9udTus^e#iy_^24tx#diF%28vr z9|&D~U- zC}|a$MRm$ODgee6EWx+gFX;0U`0+5y^DuB8qB2V*bRd{>qP(ff;7BL^_*JxB?|$T~ z{#y&tn$X?Y)K2d*a1=%}ESVr3N1(b7FDuVG=N}+$-HtjQ20raCyOeXhJ;U^eUSzvw zv-?PdFW~Q~B^fjc=-$`wNa%J3M6OGpFGQ%G@*f}NrW+gp%j=@291;uwzkh$| z{~5!bfcuwS+4H!39|dOjGc$>5Dq%|Tew@izRX3%ZqMZ~i6G*J$cg%KuV@QEsZaYEC z>*mLUNlq%pJzCU6D<5KJmoK0>B|@j-%zAb&W7Di_v2-dbe{qlVkIxf(ZJzv3(JFtU z3~ldhu*)MTspdQCLXJpYUu3$^dk`8vOE=+ljw(IIY_hUSK7xgNaRFn7TvQCZ13M+; zvk2%(%9L%=ro{^+vSeX}emcT}TX-R!q9rG2cFnx`e}8DW9xIn))Mfwiw&3%9K5eTU z@L*Yd^zPI?Z-9Dk-lv#Bf3=^4zpYu(Z)lrGoGhEVJs0`CD-_NT?#*G9Ol+PgS#Z0D z-^}vaycc_ZM&J`1Eo4$oz#JEOQN-5;n7Sa02qTg(<#S5prwqMaQ}$hJTCOr)_ zBnzHujF`23sisn-0d=n5gkJ|8{o78y+W3WVowqq{kmQ~+yHqGqAEfrCrWamwhj!Nb zsb4@-uZQ1qB81bnbec7}=D`twzdMynO5S-S4E<$wUqq&&45p0eaYjia&C09;R3%2$ zr6Dw`Q$swgSu^FAb2M0C>qn1r+Oe90&h!J!$27d`zA4NoM`>(oQPe!*o8RhDMgh07 z`Y-=F@gB`2)W)o;o%$T{`tv{3zI-%* zS!Revm(DPW|2}$WpviTd&-YQ~{!86Z%NaArA@r(jf9kH4b%hEOHw~|Ep~-srtx0M7 zL?S1FEEgu&4!OwI{D#pQHq%}e(@`dK-*B^w+rC;V77zU65qU8R+q$f}IdP3vL-#e= z?Cx-UF{4t;F`KlqB2N!MTE$?2un!>}A42(36s*RB818K^A||>8?z6@v{w$GK$jNw= zUwaglOAg*+5^s_Jt3;`}-cZ=kqDrz8!}HTTHJzFoifr9kec2c@TCD~Ds_<$=WtAd8 zH9)`uxLjVrUl8;@A$1p@A3x6>d<3P>niF{m*w4^~Bb=Vukl{9=89Z(zRNOWdB$4T< zn$D;B8=Y#8=fl+t*R;!@k@H)3ntn8S9)g(zGW7QI*VA@*WUy`rR=tqv_=oY(=n#Ar z@M4;;6sN@?UkrbGzs4&m2P>cYx{e0zUe~Qfo$o5f746sQAcyk1fXAp;ga@l zuY|xR@pqMny)7nvIj2vc(Q)1C%$g8fjk1+#hv(=NvXFR)s=$>F#-e&G20Eb3l6aQH zw1*`98*?fVkkqe>1>q=*ztklEC6B5U4*7e)rtc(Hk2H4Z8m8AMN~kI{2dW;IJjl zbs9dJkkf|8BL{hpC#3zl&LC(}(24*TP(@ZNq_jmhh>G(e@_l{^nqEeAK+2ax*BDGy z8a-C$5*(>{gwx~8=*4_YW(EH>+Mcun*+2UKF4Qakfgj(@Tv~3$3)+1yKBi>$gs=!a z6@+tpm`0#moLKo;Ww4lv3x0wJLXlpI-&(|P+qaY587ee})exH!acxAlsv|BRtD$?R zU!n8QT0@g1qs-cBIl|*?V=Ba+fZFh(z+s1f$8{Ja9uha>YiOVigRr6)diDIdlXz0f z?$YyKXv}>js}9h(e5x)jutNV@i~Ve64qY=8d5T5Gra{Lq5S-eF)K`ZJn)0d`&Og_b zhZ%4>%JGX2^Zp=kvaxPpQ=9Q~FSLYfhZF1sad5F-7TzSW(Q${W z-g<7_;C9Sokl<%JYg0?hmQTS94(0jBnx-eW(J!b@P-kxViHH+5Es@|yC!=`FmED{q z+^h2tKD)phLHf)Ff1?e`j|7mi`n_O!kLQv0lfmA@`oEj=%WFx7*0bJz2o)LJHL?Ii zH~GUDk*B^rPSIhGgesF%dGiHWuG&BxM1uJ$n@X+sjVPBe+~@XmqVE<0)#S0Q4q81y56K<&VBy@lZn3zy> zQOk)pqofHjxupo-N;TYQG|s8U?gDi+2;09(X1=QnkLj3?E**CgT>Kd#t)JO^pcH&O zn|Bk9>BVe<_S)*Lq=1Wp7$$}PNIr^-I18FQVdTQU@k#4b2a6=RnP0O$jE6ui7ohXy z*ZUyASu<|FOG&M`53j_nuB5f;nwquEwU7EHu_PgxM^k$F%^JMrpWo0f-25MZ#a;gu zKQFbETr43t9FOMNKSI+JcUX?NAOOfSbY>az_F>UK?EniQZHGmw zW;jy!9W}{v+NclJ(4?7Ihlj(B_RT&Sh3;R1gvm3Lw+lpEu)mx2WeUy@-(`>8m1fc` z>twOx)xEKFZ`V9`8zbQmvALh^3(Nh@;g&hbclO|6Bvudr8-#u2u5li6r2nenqUS0R z5LDySVbtJ&YBQ^tw4t_7X*~MfnnQZ)sU(Ul4Pi`aEI9c#_@VA`W15YoSU+#9+2$2p zc{wlpAM{o|6{p_iCkK>eqdF)y%mjQ;&D>b$9%&)O7?G}MnQn|v9xE9+5V1&TWKKg3 zH)Vr5SqMXk@l%g{b}f!AWq2tgW60Bu^s?xslWd)zW-;Bzn=JW$KOPsAX-SgcoZOg} znphs}XEjMG!4;y_Bg}%>r)22xwqSZJdMICaMw^j>dXdQAzk)Ek09QE`T*mLiobUI@ zlJrwW+TTx%;c^XoKk2aB3m%I!usUodg+MJjpMGZ@)M{(eGzeqW!e69(`;_EvFtdg( zh=ZgcOdY=6O&|A_MoWYETO0VpI#M}a!11etATFw4(!7`G6mDd!oWm0Ra9e`|?F@{U z!AkqCjlJ*r$_A+}oTsk>BbKHVIufQSN_u;SiRBNp%TD@AN7P(j8~jH=!0_e=dAP@2 z@8%*P+zC<53HnDrgwwJyBKsJVq_FQJwD=edX`ezxA)*95wNA8>2qaRMOlh8|+~kN+ zwBSrMRRS1ty%zdFaB^R(F;rszLqn^;_zmdRn{@m$SicL7$)=f6gp)lOzG}d$eqmE8P zYyDD9hXZCbvUfQVUvnb3`AkI-yWQ5!ErmnZPUA*`Z)xHijKJ>}y`kLddp6z6_jKL} z3EBMP@D_|Xt^CNEFsekI0iFa|Infih-Q=?g=Pf(FvnxM_#S_g$!)pCsw9~2d<8M%* zI=3f(DsJ8Tr)(BPVWcp-DmrQ)8=k$VlI>2C;>2XwRp%0K60)PE`BWlJz^%n@Z3vK6 zc+q*hnXlRd)$e5nG|)c8r5as|;L3o38QC97tzUI2^0aWp26G#jPl+NPtU4LCK_W+T zZ)f~L0CKxobU~7+lSAM$ROD+Ontj47%I>hsV6H!Mmj1v|i7pvg5sAWgJ`@1{Y#DCa z2gy;(zEFUrjex$7Qv$N=mT^Pi_O=d;V^a|CC84q-Gm_%L2H$7Oj)PzhjFQ=8Z-a>O z?pFpIMRCocs2UIBFqsl1Oye7 ze^`)D8qtxE-`v&x47#XRznx^m{m?w zS0E8g`DP9KPx+ShFQO2GZ?Q0=S^d?u(w`Nl`cnk_2#fPc1T!O?=+uKMh0~^jW-#+2|IA})-nIybtcDc8>9E2$DhXLf6#Rs+coQ}r7r+W= z$pZ`lrHzPJgbx>DP;Pd?e1@JBja5Xu7?5I>LT#XBw#ZwjDuf^Ij`%Yp!0SS=xd;#Y zRq@3V1!w6TDL}MPmYlW&q+EX2#b%FJ_)lZ*@%w)ch-UbY1fD(L;fxQq<~AgdBh3AR z3N6V$`Jtq(cXA@Mpv+8w5i3b$6{v2C%$d4n5ZVWgLk>8j%0l1eB~xdfPO?b&uw?Bgo;-$B@{>CV%)F4wtnP?Hi?cb=4^Xv`ZNdTA7WdoX zkL*S0dIsVg$zJ!ysmEijIoC_*C(C8=UBYDsIid2V0OqHV?Z3l)UsdOl%tlk*3=Lb0 zxgqp=r6k4FF^R#F^(4VJGS$(QawzZ^OmMhCRaXO|NIF02r*)$n9Yl2$o=Tv2Z80g% z>-MxU7TG1Qixq;gn+36y86^GCv>z+3BT1KSflKi9M{~B;`l`K0i^%xV^}$=rf1Z*c zqBZj@5EEzDO#sXkMG&WAx~q)ANg8WS13873fvyD5&s})g2~g0kvH7+xYW=BI{P%k{ zd_tJr-5(=fWf(iVm8bLbE*HbGN1mm#G-j`t)0cOh9A_4qH*6Ib?>;d;#U|Vmm#nvT z#eK_o4wW8>h?xI|(`hI0PKtIU5K-RVZgDn_fq5vNl;my3=c$d#ui){BBZ_fIQYpN1WMrXCHKV%)B$;o?i~fX5lqH%Qsvk z0K_nW<1p#iHAaAF+9r%BRSoCKLb@a416 zwfn*Vt@3H|NicZI6wKPtSE`bE6x^GT2$#zW2@SI-*Pw-iuXksQH`}KzFy#rvnzkF= z)@xpRtgr9z)*+k+AXc`!)g0d)ob42>yPt-9Wt?Bep7I8CYP4eZ7;6_4QT+IbaXhbPaBfUR8oCCg?fvWlo`NfM*zHcK8q?h zgr*Yv0os)+Z5TZ$;@Un;?=0RfT{F&4@B}x8O%gI0VejXXl0L6_{7qeb+Ip$@Yb0ye zS|jzE|U%BU}c0!=@Q+|w(GI` z%g%u&!9oFm)#btt@9@gck&4vdbaWPl(%O`=Yr{|MZj(0&9EwgXOl)nLC1&q7g3(4v zm8cO%t$s;(tOs2*-9x8)LdIL()N~Kyn3_Qe<}LWH{LxJ|ayF~_c#!t5J3sYQ7Q!RQ zKSp?N-&NwPBiIM5-FiD(jqHg|znK4idih9o5aPzea&E=WRx}5kZ{&QrRDKq5MN*hr zolX=YW)*nRX>dK>jKVm2TGv{07i%NXYq|qX$bJb$!~;|)0t9 z-u!p{&i;f_$wdnQ1;_fvzFjG% z`)iB!cTw>{`^q8AxKj?KHSl$O zY&6Mc5-V8TIq-&>hRUGplMSEIVydX>HdHbd8=x--v`J8;7t;3POQyESzkMJWytmao z!lP*vl`@Y=O9*>m$A(C8mp$ye{Cn1k&eP)cx<`N96hg`q{5ap})%kV7bH&~!Ah)T9 z-6ywoq4e@r_qR3)0=TrH^?KtZt)kngx;Xj$Z@3@o6S?@D5`co27PH-ef<@eq65qB7Dk%i5orRw79v>?*3P&)%DKABy{5);e4?fP{F+_Y5d)z{j4u_LUL` zXTJ1C>sGBR-cN83KvKDlrE+nf4jI4JRr;r2m@&z`LU&v=E5zn8z) zsO~*XNx3Me;q)8u)2O?muL>Fe8V^U6rS1z+AHC}>{#79MtyCUpDYu{)fq@K8Nh#14 zISbhn_q>cfzBp}mm4nUCxJ{pZ3+H^kLMbi|0RoJ=>PJGt(Ut~}a6XwG!JHbW>ZUi} z@t3}{anr+`-GL9VJWZ_{UwBCy&1q;!*{N%2StbpqKbJ8A>We%COT)Uew9m*{WoMKSr>h#YU@y-vaP%SijNa*Z5UE`J zvptwGriv<{;MGw+VS;PK`?EU=_~3GUHgGisU{v3d0y35ZWlHz!1*B@lp7MAWOQPB2!qtS@>{y=i5+10&;nzc9w)Gv{#Vzu;V=po9 zv$SmC)o)+Jt=%N}DlI`v}rwWwRs+6n65e z4Wvv{tp9W_o^d|pAYVG~wCLnScZUz> z=tv#MVz;k6wf(LyK!5bEyO?n*cPy!!yOUSMM~MP@Q+)7HQ{pU zwqO0{z4RUb+-r{_-nyy9bm z%_fKFlgXB{7O%G7+_1|>A0$1Dy;pDh9763cB6+;b4V1a!& zhH1QaobMi~x?;cDR_?TplFtNOVoH@yA2OdG+#H{}pj+;s5d8vjEe;C@*s3b{rZ_r% zOW@}ZeWTj^PVtF6E|nPe2_wCyoW7@}xu=z|g}W830C010bF;sH&(6iI{hmkoy^t`E z04o=lFc+6pf!oagY2f5yX=m;C|2NU_tZ4^p>}n5v9@!xqW1K2wW79j z^|Sy0d~bHC;l0CCNyUf9qn3s64qC#$Ytp73Uz!{F kIq{SHC(<}Xu~3-V!Z diff --git a/docs/static/favicons/apple-touch-icon.png b/docs/static/favicons/apple-touch-icon.png deleted file mode 100644 index 2357f01bc1a7445b72a28114ee44acc9be89fab7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6326 zcmchbWmHsO)c-GyfP^5T3^jCvATZPrLwAQ#QX)Mx2!e=o4Ba3pN_UrZOM?h1Gc-ek z^nZT;ch8IG?X&JWcdvWy*=wJ(?z-Q-KRZ%eQ|TemGa>*09;ztAbTPL6zd?wNxz|=9 zY%vDcURFaE0BYiiZ>;bz&y3c}x*7oB&jtX&9|7PBGZnlA0A2zBuww}T;^_eJ#4Wp3 zM-uY_|FxPD47mUA%I_#m!psnOs%R(>Y!G8Vq+~r|u3*M2Hd2Af>iaJ5=WP4XF8GD7 zT})s<>QhaidV&CRMp88_C)%16wEsNvel(~toisD6GE)+nByub2uc_6=${sM2U9S?5 zSQ1Njpu(fn#=4ZMpu^JOmc-$XRpT0((c%gXUWQP~VB-Y0Oe#gDOI-Zz8I$nYxWkVm z#Qo53iCV>yMS+pz#{bu&b{WaYtMd7q&9igS@l8IxL!$BmW#X0<32ot_q3MQbVb2Kg z%wj{**&db_GkAGQSWEGAy=v#+u73U}m((#{^T7kR%MmGh1;9wwm)M>3GpF>sbhZf@ zk-EdGPP_TGCvE6KQ=bP}^V0(@^2@YqGn4Iu!7ZkuYA;;2R)}u-`^xC?er z`LJ}p+}}?$Wx$@B_ks_yTHQ{V7neh$3WxSHOCXRh9k1?6pySExO;)3uRv+1z!Mz0$5^F3KUT&CBgS`yH2KdZ`{tnl zx?7cqLbd-?&(+raz7ldJxMFzD$3_4M;E(^RJ`k$D)%oI;5`ZKsyh7$5zL0*GJRptF zYx2gPX==d#;!mryvEJbY%zvIn`fJB!;O!{`qleE7H5{o8stzL93!o6$!&TtDAm5NO zH(()PWfs0-g5G2GZKQTRX0sZ_UpW@5+?In>9Ym@pS#;!goD}Z;vCy0Sy3L9j`g7Vn zAGCP?sC@e)IOn&js@IV05{VgPFg(6{PmG+umwL{0Z>2|MpWr?5^U+~?aKVuJ`BYWC zPf%Z#nJJupMaTknyViI$_L$Dil+4r zoeA(^lFKN%J8D}#=yZG~mZ3D`H;M(Qe`t)Ix(7Qgk-gBoR;-iZG9`@~KSEf0Zc`Pl zv(f_iAq`HWh0WeVUtbVoKBeDw;n?ol53PrCkEb(E*&};qZhcbK z!8bRQ(r}XAg}pagy44K?P113(bdtUb0h{bgq4ydmdU*Tr9yrqN((3wq^+Da#td4_y zMpQC0y~x@<8BrY0VzX<+$3&|S(ueFG$2^B}+*zm*M4-9KC!v3Zp2q!r3_ku>Kma}|LL&R`Uu7X)zg}n zwxtD_RgAqb&9Ci zr zgHis;CPJ>v4g%j2&&v7?i@}`!`~wdTKz`_J%)l^P%BRc5S#L}~$4>wNmg2D)oH6Xa zfx$wvRpwLd;eMDuS;Rrsa@VGdxBL+V03rnF5xVZY{svhso*nJggCug{dgg5u*v<(N zp_OR?JW=1d5}CuTw811cMt@XrYr&!o-^_j%U)jALQ!f>uvN4?FlZNvMTz%nCIjQzud!$r4LLoWTs~H zST3%e30ozVb~zdC49|aPb2))CP{076)zu?zk>rt#Kk{|YjcX)kQldwW5W9v>w-~r= z%PBcao38v5U`!l!o}YisGthgIu41Id1V%QC8t>{#hz!$ZrO)61qnB?ZgH@0xSzdh9 z53R*M`ah-u%aJ-pXzQR-U5Y)Tk&f5{QHRk+RI5`T`9wd-^H5{W@c6K&$qm)ew45l* zXDxSVynPj2YdYk|5`1XH)Qc^0{1`f@Q9MgGcmABNpAn-?#3RFnGY2 zDf>VHX4jd9A$12S90ay6ijG09KTt-dttoNl8^>6@-`Y9y^KcVkcm8z~{jYWE*~7k& zbE%O&_q0YfA6>t%Qh|~0k8^)>FC**Bmp)&Jv3EF?SN^!~Xlmdm&pZ}WjaXl7p`%Nw zm8sRGenCNzCth`XWWNNxO=Mk1H&$>+q`u&m8Qwlw>U)Kiw56VSeKlv_S+eJ!3X$s5 zAT%rXAh<*VTbUJwjNcX;9|T;yKEW>!#!Gz6L=Zn_6Mv%^aKV0aOirFQEAI2tFqsMf z28h~24bnm7UWMyrRbS(S5OYndjgQR=9iO~8)DjkYdoh+$NV;(Fn)V{NYYF{*b3fUH zbXxan8~V2D{Q`|Nks?}C^cS1bpR6%G(o^9tWcS5ar)w(vfrgwU$x%O$0Mv)DK**ac zwiI!;*K}3y=No_k4|s9m#M3wu&{B@A>XX z?qqc=7!}C0Um*w?`n-o-`Y5d)zSfoB^$wQ-zi4Eaw%RRmGn~ z6xVT}tFK^n6;M~SYv5%#f%6zi|20K{o4{a9Pv98b4zlb@BP(scc-aT zNNsc%NYA#r46dP*D`&-9&z_p3({0L!JG>=y(mc}!HM<13C;1$de1sp`z@TTTsiPje z`R&FbprPQb;fxbSjo7QjwAs=wt`}aZUwlM;jMS!)j0_N>o>A|~h*|&YO{T^pS+}{g z*T)_DaGEEYwn4X(SL_F}B4)ppjJ%ODv*P|mrCnfWUrQ890ZSr>@W+r&&;}4ZKIt!b zC@Q}hcpRveW;s(Vd%s0@i=++9s(DRLcYP;^{$*YI6bP>LZ$t-i=Gy#82-I?5)Md++ zI#Rq6y*G^-PutT?O*;^zHZ0+^hm92piiI6*$zjWEqw6+~?d{l3YwGNL4R;n_MJbNF z_p$d1k@J>s|^NY#sAOhmk}=eJiP7@HDyn`y-RD@XeBH)vglGv}{4^z!DC?Es9u zY3Iu(A#OPlla$*4c>^1=q9y#)v$4rR!E7OQaV2XGJ|x8oW-q+@2e}>bQ)M4Xgfb(m zX=m%RLVThz{)wp~m2q~1lgGwGonnz~60Oi{ZB)72*sphF=J(x_LF1tV;^^oE@pyotxmj81gWC@O30Bb9mkML{@#V zeabII?>U6U#09P2)}#_LAI6@qY6N1`t(b8#i;kor4j&as)1P=LXK+SR*1G#tA%#ht zc%{C~&L&1udS<(K3O5?Bhpt-4CYl{QFf58%1abesf=32bZR=h^EuwhS-1=!*3o$iw z*C7_nUQJ9S`t(81%*T#^r7JVzbuKT*BKz1791MMkD#f*7|4M9a9~|d=AzGqPMeRw_ zEuL3$*FRVpTcv1zNefQckv32b|Weq|znzdQ_ z97mdl?!T<@VG!}A^63%8C>rSv_wi+nFJ9l;EZijTH5_g=lcQ^3TzO2~YHT#H$|H~PH?M)(bGAHX zB9W@W&hVOs(kY^eY?|4-e|7&+iLd_;mz14)@bGa-VBC|=vXzTusSs&V4vWluMDezc z_=U8FMXkgotfJpbqS_TY-?ZW3w()0s@3V!&!QDr#9E$?^;)KwKGqK$3Ps{gs8;-SJ z5kUSUxUvI=2|NkGGio_m2Z6?XARk}>+U-x@E1ri!7gJs-9lUY9?WPg;ugdBU8+CsP zftP}|oOKzlxd-(uQs&R8Th{&bC>{(@ksY55Zcm3N#H=)g8uwmYf=3i1>F}yvGn!(s8dwWH#PKFe+KW!y#AJBFHPq zeX*yLcl+tM+k@COjEg%3tQ7a^uNX7{uu~!qAbo?HZFalmbK0Gmq@P{A`V4^>Tkdjy zfh1sw3XKJK@PCbehrnCj+dZ4UHqe11^UF?7q4#p6DWKw5-F90Va`o@&StI7OLZ$q& z-cR|}j7?;tl(o7@v^rx@QLLshaA;sB4aYZ{NNo{?JAd?9NI~jp93OecpQnMR<$_X* zB3i#!TNxU1Yw-+Cv?Jlj?($D{Gc?h<`G~d^RvIx}AU9&iNbxk1NtJysV9qZ^y9EO+ zH>$ihMjg_S%le$6z0vR6%r4FY&K|lgu4>kKE>g%UoJA75b z;ED`ysE{5_JP%7A@_G);c&NXU-ugw0>g*h-_rz=)r#WR8vi!8?(;7G#2&T__P zt4#m6NL9ofBLRTDDf(!zuE@dKe(aDW{T$VTjm5?F#e4kXXYWn|4&cHPM-ZDqZ0CfZ zu;kgzC})DmT%Mb#_%|y@RaY-c*(8ekEOOomAUMkb}eimn{uvEW%tQ8mPO#2$;6gGemLKH%X(dRdqTs0i)ykLQ_3Pfwuz0)AZ(1K)q zr0?8guw2{?BQ|V!;ftqB%F8yLieXVv0I~DRu7Yx1hdOi6vo)&HCk%A)F*q0^1rLaC ztt89IMIk2z6j%LcwqO4cQ0#5aW*;93POw&q(5^saEpVFsWU-hTIkw-j`~B-(Lcg_l z)jTJNDI5>Rr~=P7*k0xmHSI6rbNn9n?5zWHe0}&QF+sL3Mim79IMN6UQ|}pkFKb`YQlyIyniTt zC6(<47(Uf2O0=qVGKO2WM(x0w>58oaUpyB3#u{`}LV&w{LbmCBWZu6sz9&>tp zbDxHI zjp6PVK@U7^`(a>#4&jd3NoH2EA=T_6U|$Pq8Gd<^gie-sA9~49aXAJZ&br{&F{c73 z6zngjte!ywSTBljOBwhLP(ZbQ%ATo6N`e&hwm07ex5KF_OvZsbK7B zW94ZpZVk7^7~r|U^XI&Rg1iFH^#z5*1tH=>qTB)k;sOG)dG1sH!@=3r#=*|-e|He* z9d^Pv82RfPd+J*GFuK8A?HrtJ89n{nY#ANgJgou1H+%h - - - - - #da532c - - - diff --git a/docs/static/favicons/favicon-16x16.png b/docs/static/favicons/favicon-16x16.png deleted file mode 100644 index 282e7715adedaea78337224ed63e24a6a41dd9e5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1327 zcmZ{kdrVVT9LEpP3Rs@0$U_{>2n1aUrN9V;C6=|eh0>N%kT>)J1uRf#3lzsJ3=ts+ z!azVCMiW`4+my-NAd8dXvJK6Q$Uw9C#}=I+l+l()dv9-dE*QO%q0 zHv?dWA^i&gv@`&g3jv7l2f$Y~G?|(JfFoJR6SLq3WJHBd^lW$K*foi0mjtWl*wqY! z(AnM~wAV0^VU;1(1sSfi*Rt$Q*;qXn?UpVxfZc9~!q@jI-)STNElhpHu6^Iuk%tUb zAfr|I>S(p4M`94U8U%aHd4Vv+tGkYkHK(o1-!=$c-UQVQe5_{JR0%L{EIVv#%f=do z5N4H6Fy|eFDWDJJvG$ihXWW0d5*evNKlu>uflJoXh^3>nckZ<30MOU$~W>3Zg=M3+Ce46Hs$!E=Sxia zfws0BO9u`KYiMcDw6*78I*z#@$gW`^gU2oB^0(@atYwi6X{62a$gNWX1eT(&1eOw?_-o9%8dk2cF4lPnuUOc*k?-R0@tYMJqp&VQTC{@SvrAZMW_qZAjX?AZ8r zEI+QQ_;sZxfWU609y-kLktj5h}`IOGR|&KhnC z%lSFU(VZM(3jQn@YCO11hS+aTR#;fL&)bKrkoI+b++8dIjv4-0aXKO^q{t^dM%(3b zp{4i)d#tbKSZ(LoaE_iOxP9x3lhe&-dO{EQjYrW`s(>ZlK>VFW`4hAMmEzpr+@A06 zdoS(UwQ}X^ih^IO(nwV`1M^XL=Q~a>{4U!x4~*oyJwhK7C!SU$0klw~~5!8mq z=4bu`3k9jX#Qwe%IAC_SVg>BI)oLFcqejiB=}z0RTcYtW1quqA7}(RTsey&}no! zH8PS)ql+V>;v?hYqhjBo(c)<|#%R@pe+kMf<)y{C{}<4ve^Nrhv3hZ)MkJ}FsH!WA zOO-_wjZRfWDOG7?0MrfrX>@a`$_NYlSulH#`%DC!f=S^r5;>Th+_TS#OkwJyzn41& re<`NSEb!fR_TmR|Um;2fz^m05ib= z*yaJC%_(T`v{N{gLS6S#fc)pX`shZsf>GvB{o(;YVSX9}m4#U1hZ1%_;sym*~pM9ucZk zw3cVL3{ryYbxF6ko97wpTWZ8c8WrPdw_JjZ4Smje*mvO;#4x8QYI<=~=ALoaw`~b_ z=7kHw_)l*`POh}SGK~!E2(0aEjFtxmyrQH}e+72In+R(&Owcxijm|M;*6CpkUXE|v zx6FvPPvuzcbrZ4^`)6I9B-7u}kUB0*!ZQ$~InHAj?O#K?|6XmEIYTy!9WG#w%#Ns2 zH1)JSzilo_-e!DbM~oUz-nSxPY}Cz$iy*p)L?7uC?-<`)Gwl)i+u(0K7S1b>P7t(a z{mkGBns%2zgPeCAJ>tEEpdUBW*$cr2k@@YVYa{FybT?;#=;QB*8XfK;GkGrRIQuj9 zc9b+iOHtVGdY?5o$`G46dX8Gl6+sA7Uw;2Z_@>^eU)NXq&FSF3!ZFsi=}+XJH(FKr zyNE#%M5M%U?B5s-a6*f-XK8K@R>@I^Tdb^-r}=d$UVneya{#%8%qL*46~y2X*G#GR z?iAX0ONNE^#Q2njM0`e2?r7hUo*g{ohJy0b?;R~#e40nnFHTH%OFfM;)pA_?J8F>b zUK?Ilt>)#NxGl8VcUUQ%G+8VA1Nd6uc+uSG!{Z4*B|wG3$L-hk zRb9!FTdKjO*3TudIbq?GtiJRC&3c<^R@Ql$T~CB&s}s*;xj*0kZY}D(HBR`zf2a%F zq_fFqr$de|>U7CJp1Xa9`c^9=_8f|PAf#-YI`}M8eY?#(zYRyOlt1=DX{Id~e_FD4 zv%D>i;pKY&g7uh_CNTxg`TVWC@vA4o@Vl;;-eMW%Yk~9`%h5yPA$!{&*TZljBb%bme)bd?~!H$z<)~8$exa3eW}b z1D@|kaImbRro4J%oa58)Zm0*f#FrcNAobNVWkdja$=IdZRKDZHr26xK-OF(Lvt979 z%lQ#ILow&C*D#{?!r4@UjZ5srV1cVtbAUmCR#&z9c$ppBA`VuzxbQ`nnA@@?pt~WF zpyXX6HTu5cfU(h>q<+ZcGRhqzG)pjmJt`xcX2?-oyx@VWOjBFh^PKl><7JBkY!rd z+IzIGk_m3Ay>`!fD8Gieg>Tm`mFjG}QaJ{hiy$X11|ylY8;a>38cuHsZOiQ4fzAk& zY!aP$QJP=>ZzKpEW`uR}p&|BfWkorSxo^&s_x{2&rd7Cq;5%xnRW&a{Ejrzw65v+6X(b-MeRQAYS zQ)kWd{zFOqsUeaPZI$Xmyxac$z|bIveLup*jPIDw4(qs4K~2%5rR%HUER{ zNPsv?!{*nPhwq0U^6lgj;zCY&Q)}{7W1U)rV7&Q*tUn$j-{@0!i1!A%L?Y?I8f2%d zAf&x%YIBI!#djNiUsG}KqvazbUO29kq$C(TOWwU0ob+|-=6LV0|Fh2SJNGm3G>$0d z7Ian|?ZPOMAm({`hQ);|01yd8BHqjlPat}m zncJG#*qU3L5D2yefxT@VJ$&5FAMK_T|z;j^Q|QF$W{r;W+Mb zPB<=-!wmxIAAXXqKDhjBmfM8VuL_y}6gg}5Wgh@!41QZB}hyet_0;1r6 z)~R+_R+lWRVyktjb?MT!)L|i7Cxpz?8}7TO-@fmi=gZBT$e?|TpYw)$?*8|_`|PvN zJ|~8eU?dv7dmFS}jb|=3jDCh;bnP1W+}1FT^Icn7`F(ye|1q{RZW!gdrFfHA6rxi|2^RHX5)$y zqDg>j3fK(311f+YXt#okqV=uMZ-bVrmT=Ah?*bF35dI48f-k@d(7`DSo41AET<|fl zfab1S`R-p}0XWCmzpkILcT>3D1bz>ygttZ*{-T2k-UbsBN1bj_@F%M|e1?J}zzsCt zNUeu!{sKm=?iqfk)vB&K3|^^P!T0S@&Y|yP@VE6iXG*cUetCt;-s4uHLtbqAtA!7F)s(G1 zHEfOb8S*atCHXV9Rr4`L>ilUXDs5erTJVs^*2C#XqTB5cd)4$EK84M-kN)EP83*=r z5FO4MSEMeQQ>w;psaCf?!n2U)0hS*0s@c0uHF|@khOGQ)d|~^><)3G%bBUkL;HG)T zYDJG$6N}ZAi_2B!{VtWW*HjZW`&8O0rHFCb?z(Lh{vzkrJCvHd#b*tdqGYWhk&fn+v6(`?FpGA*(`#fsE z%8HX`k1zTr(sPK#KXtkFRgZbDE6+6k!z&4l1g`@x2$!!o=gZK~73lEH^2faWTTUpt z68`%^u~Uy|{K;=E#)dC>Qpx+x#}sDLH|9K?`>37QUIX99Kp6<-6ps!jecl2Sau0gW z)^@{>nj`-f@G@`#+pl8qR|A%ssx@Pr>#M;Z!Ocly3&ZB=$m|R@f^R@5Z|D0+dVEg5 zWuU$0?QD_X24sVefX~6-V!jgCaTNQ9^4H_&H2rpi?)u)Fp7*s~_NaFjaz6q-;N%y*%ID_o^|)lr#mil; z4b1tK*}Kh*ZGWz=!`kWHgo|fZ4H>x1I;m|Q+Q#57dU(NG#D`;DW|gXuo2!X^(c`1m zFXuO=8h^JXbAD0ka&@h?>zTIXw_V_TJe0S79|wQYp#&V0oP;=`dMqqc>Dyh3e7#nV zD890Yyf&S@cIfJ0u2O`&`tVO_2>$3``<3X?mK>lT`T8uKuj4C{D`wTqYjv*jf5l() zaOUeXc6rs9{91W!qw^O(x-rj(JxikHx3>Ri`-irEpJMH_#H+c6Ya7k}vKEkBYc%6B zCEK5i2&YhvypPCVu4^&ASoK-#RWg?JxN@`&&i^*nUemYx?6sK0IYAD&$e=qbRr}g` zdQAR(e+K^cdV-iRP|CM!*Kg8ZH#Ngwyo|s2W z*6}$~tLL4`1uoT{Je_r%@IMTy#9y2c|A}UQ^25r@W>;q!{l5;IlSG>jHoo9Vg!6FL zy{DY(iAenQ_oBzb10MI#4OOqfxiv@!?*gek1@(x<->#pemz6NC9;P38=Bc>y2Y7S? z+ri0r>M5Fcpq2dVpXiX2|E$?g+X(p`!D{d&uyu%wzrX$|*Pj4)k{|v`-_zMrLu`la z#o!o-lm|xhqwnb#u}+HY1UU2r`#{kd@)sR^;C(QSdj<1#cuSppG&l-mJrRv#xZf)1 z^EBuePn{b1%|Keq@x?FqTkLg5il0&HZ#rL3U39?Pm;Ck#WINXr;dnZ~lYIDZU|i^W z!uebyeZC~Sy5qp*)(4K5ADj!K@VEUB8x&r3d-+L;dot$E#Mc)PM>Fp8oygegyCM?5 zSl>59{SzJRc}M0SnTJFV>=_pFo8-XLn7?jYYu6AzV{Z6G>z~43=6$I}NFFdMOr9>h zWj!(L0aJ}yAFO@Kb8gW3mD7)I4%83ht-JYiDu14yvBMR71~PXA))USeVx#g;TV<&( zGs`UD6|WBZd9owaiq8x`n~9pv)`Yu%brU#Uutar{T8aG$@9)d}w)!cL?3MQgPXWn8V(B6KpRa?F z>z+1~jG?ES!kxOGUH_B3BKS-q@OQ!aMKFl@vstV;Hs0UiBRSYu@K+En2a|mA-(V3r z*1696!n+m71n&XK^OS?X56+USWP#Ss{tfjRe&>Ou;8UQV#Yx^5fQw_vf%gSqE%U_* z&7b`6YdEjN-gV9uV##&>4%bV;{opVdko$-y?iwQe6FBqH>zv7@`}!|252USBR}WsL z;#&JSdDQ1KyqkhL@9}q@UeWBT`R?ZR;h^l7VJK^ZwL#IEDXtW0OOi{Hr9GN>G*Q}o zq)02m8lbg|gnVOzVYrN>6Gn<*@E>mtc|ZRdd9<(yVmkP3q5v^dLEm)FR(s(G&*_x0p8onrOJ@6HD|fZ#q& ztUi%`PhF-G82{bb+uhxJf#>AeHg8aM{WnO z0(>ER%bZEG&lwnhcJa+`4Z z+XD7jCv9PGV2ye-R7SM-$lmUh&-2&z*ZSM7)o|O1>AosJ+38k&CQXQk@u(te~=WXKFfBo~G#7Xz~_RE@E zuVqBOhUSs|pj%y+S6+lIWbd=a^9U!On#^<5%_5!&CAUgIMt2~4o<&+-gjV(?UxeQv z&SVTg{xhIb^KiwsO)oc*9D9bKdrCVT7=Wm8Ko#7676CSo0wd$y;ZvYc4? zU1)7j^be6QJS0#32bhCDe;sL$RN{%$uw{>qeVbQ?-(^L7e%hYhva1sD-`_gpheN)$ zMH%DeyA~mPoMHo!E564$!{8np_k4+SFEMkXA$Qnw*5Nqi%Utpuu*b9PQOcRZteqak z7*My=Jp1($_lzlja5(wIuigvXcCE`1H^gUc`8J0DkDl{f;o`>`%a`~iab)6V_S_!u zD)#hMyA1!CGLZ*r$k!aYXv*OU1166=7t^hfSt@j2P+lk|cugrM|EI!LC<@lg~Ym;MXzzZ`|EhB66RW^OE6e8n#d4TNAcc%U)3_vd4qhft(u% zD__P^pG97&`)_7_)J6Ncv&DBig0+kpS$9Po%lu|iJAa3|?F_jGa?b^e!7=V{IrR^j z|1GZh7|a35T0Zw{38sSgK&-PS$GK(|IA8bIE%)q(e{SzF*LB(%KSJ^^c~)-l3bE!^ z&aDnamp^d7Y7kB5QhwhHdQkJP5r4IOyY2RA6XuG&&hhJzul=R8*P`;`EPP7OOQ9U( zJMlfK%T3(udv@*~Z%qFc4#=*R|3r{)#|;@%H?6KzoR_Il`*WNjjeGug$`@PM@&o>f z?X~`Pt7A$h&h~Mxo3pE4*`sjQZ|f{y&PS#fcrE7p&^_WB9E6kbkXYV#3HeVQ^ABx* zVm;@ym6}@c)A>eumXp}yy)36palR*F4#rxkUh>J6w@LgDcRo?#$D#us+s@cq#-pHy zEu8azz?To%{vF4Evv!#`q3fIYa=3bwJ;vbi{j{B~`^bOtk^faF2ZwyAKfMQr#UVO) z39@BRbqSDqlGMQT*beSFu6fdIVx*pyy-{*1JGYX3cYprtfBs)0SN^7@WAMgj_y)O` zfF0mlAT_#ipv#a#Ms1WNt3}*C%^bQX>wdFVu1mjBzynn0>6Q8nl*t~VIp!Y!S-<(5 z-6!6goQIo?S~Xtq9z$P>(3VPFT}?jZ-(Qm0C3`g2++HPXW!bNjag$!7GaopUl0i%| z+5fi3L)HUsS#y@7gXFU6!f6#sVi$ca{thi$_s@UM*OoDx&A67oWzuVP4?4d_JQ+be z8%jR(N~h@+ZqZHdkusz5KiD@1ziu~tW7Vsi;SQe1l-NXWwTr(c3g!Vu+7GQ+=WLb! z0^tHDnXBI--zvAy5pHw)*X3*M{nXi-pKj?l71-xpQ)X0X`$}$9E`M9lDXZMRcR)ti z{={u7lej%~hiw~~Go=3!xVSFJED!F5UcJDx+qm^)uqw9lu(u64e}Rs>5^Fgl=w@&D zP5LKeiy72LKjGQf9Pl46_k07}kEZ@H99zDMZ9URYY>cgs;Y)MYKVt^}Zj5J=4tKk* zBmN`LjF?e@ZOSuu`uA?7cEK~Y#%{D`X}jo_zGIN@6(8xdw2HGv{&`65b544(x^Y#N z65rxFAJ^6LAAEA;gw2*masxU4IG#L|HJAwpQ|xQ|k6%CGSj&IOYdbJD3eZt`sB5AZ)y%pVX(*z&)Uc)-7>#;bqwLZq7g5#{69)3V6RXjIZvc do=a5Ic=K+3krIkL?_kZCPdRzbi diff --git a/docs/static/favicons/mstile-150x150.png b/docs/static/favicons/mstile-150x150.png deleted file mode 100644 index 4711804e678602768e782a7edb9b7f06f7ba75e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7393 zcmdscWmFVz{O{6Tf`lxgq;#{egh+#cl%&AAbnZ%b35${f5|R>90zW_{7nWRUq$L-T zu0^{4>pkbbxG(P8`{K@=IdkTj=b8DO`OZ9_n77aMHL1uSkOKe!Ds3$_BLD!8`rk%M zjB5#~yC}u&2%S{)Q~-dwWQuDD5$>21s%4}H00i*@01#38hQ{Yq4^Bi$ia{&OXQQB%MCINGM zi!SNb{(&2Z2i{&D$Xd%Uy{SA_b`aKA87x`c%=idegF-n7bC+j7YgTDQu4LQC?98_t z5v;5Kevk zMv zdjYd~qCE~_A6qN@gT&FGxwM9R?R64;XLwAJRw^=6G`$5Bvw zM8C66ExmCqXCZ(UYlumF5=2Y`rd9wfu+i?4H-U}RaI=} zMK6nV;+sDYgqpSu^ctC0qy>zxFO&EIv{ zRy8mmhWKY4?*q${jvixd6mG4qu4xQ}#x7GVj9`u|3c_pBRgs|%w?-xTX008~ktdyo&{DRGcz6v# z4Z-x3GqXCT4Aclc?HIT6WK}-tqgSL-c@n~-9eKL=WMjABl}#snoPP`fXM!JlUtM=N z-)biEgIr6#DlX$)lVKlYTq~{6Syd2B@cwbjylDN*@(cFe{u(@`XZPG$H`+HV2{r3E zL4XjdhP+c{_OQ>>f_qO(Fy)cuXDu^5!qs(ZxR1dX^ALE*9@cWh@)-+Afl7fQ`36f`f#(_p_946`io|YkXyLiEj z>50M9x$-xbZz3tkrxdH283URA@Ny~}c!jIi)4z(npLy$w5p`Xp@EM7=IixrP69?oC`Io-=9o8d zSx!@nMsp?!MAq3qRdrSCG!!i&Wo75iW#;nSL4oBDC!dW#b9f}VQHQ1YZ8W1)M;1x9zyF3~3BA?;SH+WbAm-Rac3MaS3dA+FYu550>p<+@3F zNze&n(VFm~6AI5*1HI?+*FvSC26V~PJSruq6geEoYWvKtKJqtdX+6~*0ZvT$!^5xW zVl#EsZb%+0oq>L0=;9uEIQ(dP4BF0DG2vx?5TSF!<$3VT&vxWIMTZUAR50$HY^?O1 zMpR!=ti+W8sM~)m)p=MvJIlZHQ+;8aT;&IaCYRumcpn80-RI-$iSs|sYrI6UCdpAy zvii4m_S8R|Dd0|-!BEMpYy*Sdwc^o0_(2*(K?1-&V{7N!*BhwPS;uX<%@IS__4X~u z(~U;QM#$hGSmJ&jh|sY4)=zh#&g62p)x6$*^QHCnK&corNWb|`79th$RfHg#Gr@Rp z+WazAFk#&@#2LWKN{~Ee-%zUE32%>kR8@6gw+8bXn^t#RbkwbGryP{Q?5iav&N@fh zVF+bP-Tn%xlP4Xi=f%WIOQqQ~yghCiB>OnG_h%W8^l4jlba`F2T>vdBdzVfN@2F7` zCS@Crmas4y6y*)=|L}%QFg5LTL~2JmkJsiqzMD%gu>oxYV`?C;(HR3!B7C1V7T4v6 z^=qD$cfQ&WCqp%i*A!=KOu2h#OTTC7KKpQK$B%EFM?9_jU18%)zCVHQoIy>85 zi5L{>l3E(4(w z?q6nFrP!rbiD>*zCoMzA5SwPKs=pIeK!@Ht zHJrbT$7Z~6_u(G&fqx3x9bhRpLB_=f)$c6cL?;T$hDy!Xhd8W_ng-D19zuiplTOW` za5}7Rf~YL=(}wvN@{%jy+OM!QV`qOzbg?Yr0OW^&DC>G5^>R4lpLEud5(hm1CvN7U zhB39CyGyi=^Jc-Xmx84&m0t*hPUUuC=dS+C$KHUoy$eQ+C1Mm%i_eH5A25CYHRes; zTB`B{Y=X(H!^MXkN27&0{%6XBo_I_6Qjrt0$Vm4qSAo)`G&J+Z&*u(Ho#K=Vq+wnO zH&(a{#;=E7osL?neVtpmK_#MM=W2d6d?7s#2l)pNMQ5`HUAZY19({;{;+eFxcHlSR z1#pa=bX9lI15L#?F#Pz+K22P)h|Ci`gWSE5!;ugHKbcl<=$fyu(40O=jIM$wc zT0I&l09by#VH0gz(s-Kj4@XQ?u?}(Y>uStQYDl)tAWO3OlmduaZh>ZZcRMF6NNM!J z-VgtlqcZA1j83IdtpzPf{aCZyaI#*2&o%tiyh-APFo?GR&1%dboTO4U8U!ZrXkkQ}R$BaH> zvWh?DdwTbY5Zv)&ZS2;D*yV7KBkj6qek&s|1H(_CjJGcp-!XrRk`gVycJ`C1p!eq$0*gIQB_@73o`;dZkB3HP;6 zFUtD5i%;tXUX`)3g;VsuFOkJRk(YBLJ#PV@Y4G?-FMPQ~yYqt_A%%Q1?&loucQdX@ zI3=%2g>+y*-*ZOnn!6aX?4q-`;J3|EzU>NDBvm-2s(g8L&KsqOno;di?aUZ;yG2t@ z?&MgQX#~B75Sm4YlQ%Is7hH{3J^)hpU?4o8K2-YwEcnfhPvEkNp6H(~+hMEbA8Xtv z$!GX(ZgTkz{CFK*7f*pl&rf5Qep=F$FFgj0-~a{J@pODVAQ8#GyI0lTyMj@Izgiak z980~#o)ubFTnjRq_;7l!)Z-$7=o*}SlZFC|<#cDv6*2ej2LCi)Sz6%xu4ui}IoZ&n zJW)AKc}tJA&6*h+iD55NZ%&f-m)Tng4zDKhmfxyIs$LPTkwNMs^4EyC8h~!mG0bbMkFROMjC-EIX&Ib^Wa1Gn!e~t zsZDlyeV2o@^yIzX_=({oTkM4%ZtgZG=TU61&s~$~M3FJG@$BzD^7N>+aQhL^_Qd5* zM2F-ILto4XUJ;J@W?@WDU6J=&su3VN<^izuJ8!KH4_2T%Di8ys9 zdwQbAIL{cS<9yh+k$6o*o(E%}eOOO274c`LIy0U-V_4%FU71KDPY8St(_$0n|3In_L~Bw6ILX4@ z1m7H;XMLnINH7cvlPx9a=tEQVa}yHvwef>WM1CDJ_#M-}V1Bs2-IZhMc;+D6tviuU zqpaz2=UMa#+D|mYz-;;nlXAJ1l$d-^Uk@Yf(UXUhe7~|(fQI|ZPya~^G!3Q_YLXYF zx^2sVe17L#JE=S_njP|SQd~iVGRWnwrqE}TCB*eZEQ3a$SrL0V)}`J za3$$ku}~Ov)|oHVxpQqvXWp~+VL9}9TPTJPXUTH|SrM~Ci-|%uK@*yO^ahD;jrMep zL5v^489MYLwz@$#f?B?{v7FpVkJ?J5T}l+%Ihod40L*G@e+uxShPa;Zz~)b8n;(%S z#ES&N{m+b9A16~nRHJ8o77NOF@p0&4!7ARYTi-=>Z)ht_LCm6ARNV8dFHLw(7MFI>nKfv)ckw1gBb;FVv%_E-0p#!tIe ztfrJpVi|yvrWH_}jOcf7N`+nYnhPk7E~Z+Db?1D1`{2QUPb2L%31upHca2!dle{Y? z#iOQ0#&<;cke5b6GhYKD4~NR^mhvlk{&9Rha@H8kzub;AT z*t~>oNgdYTH}@8lQHapfn|7<}UH%IqABwUg3lVjj`{DZd-x$xMovW_>sU5A-<(Odq zmv8zWw|&AJt)@N<(1i!z7G_?!oc1lfwM^bIiOIKq{%$a*kNV(JZQ`9?RkVrBBbUOL zUhhzM#QT*ePqtq!YKAMuec#A=)kJ*GjCCyR8-6n4Jhp-t4MzEuHZCwuy%s|UeH7Mf zX1%__$P#YFX3E-rl_Qm;(l=EK)(8wrKjdnCJ9spw6&I<9LSoMRPwyK{zD5cYktbQi z`Tc2KxbQ~oM3@;BbbJ*5k7kw~?LKO=99+T}&Ai>e2CN~v17Qz3y;WKw*G+TCCa8{V z!ioes;gsjC%=@wD+57EAdfk2%&tXF!TL$g;K<(!=nw zpZfUg8;!$nC}C6qzDK|Ch(%Pl;GN?yaW9*H61!@Q#?TgR^2Cd#x?q8g2<&5cgYEb;)tqi~_k_HRT+j0T|#U-gz zezo_qZ|U*X;JowG;y`G9@z^VXq^CA^5veCr3-4^kIv8@;FH`+k}Dn(^pQ7Z9y& z8R|wsgP3>csm~b5!om!fOSYLQ%BMM|DKy|#N*;ZKl%9S8%3}lZGOarrbPXJs1?fYq z8I8tj3{6^s|F3ICtCI(a=-!{l{7}t;FewrJkjFAtD*m>0HJ|4TYPa`PgO}oOPrc>l zuNGdyjgK_j0y7fRnW9;g$r9l*2lwvUy-hn8`_`2&D^0*M6y2B-OD`IU=0 ztyq(tlutv?&o{#59HMVjS5`L2l7T(EJAX*VDMd5+6+009cUPkfXV+~L#eXfguO(lp zfZn^jl(XJ}TRZ#sxeI4mjB5q*yOb!8hcl|QLVeBN#Gu1X+S1Tgt(MlR#G;{usAB!uKZG)h zG50m%T3ZhL)?=>lb5@M$)2S78ae||#Se5&m22t$4j`=;>Tx+n`_q;r~_BY=tW>zh| zXOe6);oer(f*;VUMQd`t6NvmYmXd7@o6OM}?u;cCI7#+vLwbju&`8E{6|K-S6a?pP zZicl%A#5Eneqy)8$j;Dmq>o{adv zL_oHY*57Y6QBN`;Wu^D=03yG%DVzxo0bKJ)epg@D!uefuqdR^|di?}GgcnW;?YiiP zR_t!48vu)rbI^}zfch;R6-f_GOEBg}lmiY?kzpZUBK&;<7v`udzVc|AL2(GvPnA?I ze%Q2uOv8OC^_oGV{Gbf}ocz8d_C9j1I)6Wep+MpJ^+b9>_{V^v*MUESC@C9UZL+j* zVMN{6Y`p}8jAHNP7~Z>^tA|NDf?}Lojq_vAg7yYq$33W&Of`>6>c~q%i*t8 zs$Ffaxt=OW3Vy-cV0B}kT*@+FAD1bNwz9M-5AQK`yLn{#8I#fsM{sIwGqct_vi|3I z^&6}tC>#S+C*pTF=vOPlCm#|ahKx#+elF<$(=4^^3~tW(wsvhk7dW?@Yk9|vn!Hz9 zgR|l5I2lB+HI_PRzeU ze&VmH4`KUw6=oum8B-TC`{5`(y!vz8_fp4Q8LDE;x)pD$?>g*2sOQmN?)I{r62A?J z3oUT6EFqDy702a4rY~XJm*Zy})w3NW*KbNLycHh+I%FFTKio+-Y)y?4>v;o_ENjFuKd+84M1X~oItqfPat7d^ zIz|`0@Rvw@CW9jGbD?6e zS|h;lU{2)8_gnQK8#w|ZO}4{P>xraaV6iI@R=_ScQ>MTemH?oU3vo$bfbLvrI4X32 z=kjeix=A90QJ23IwtzrlKg@0d--X`kD$>`;x%od_NR|?QI_xH7e}(-3CXT0bYje$N+9AI!S>v1}NTXta@4kSV&Hi#$nwE&OnrR@9d0 zR6N0-V9%-f>{%2SaTryY7v=Kiz94&bA1EIDmukI@>YN`-f(o(v$rKq*wC}SK?)d$b zv9j5wRGtCoJEDB`S>wND<JcX+NRh6JeNe|L{w$X zJMFk&*fFeI!bk*@Yu>>1V^9X^-H`O4_)h08Z|WubaZJA?F`6G6iVlu13G*{t?uqi+ z&-}h2o4Wf7$zcxeMz9gI$2NpkqG~b>s*((G!b3N1`R|p62okS><~;-NjEQHU3k)ZX z2yCU|IkwOJT-?Rc7E2j43ju8XEA#$PvRL;zFL2Q5ymtoaD=Z9Y;_%y3^x_OIk5R@_RdwYJq?%OR{@t)C3Iy7LV6)i zj=Bd&-`I185~EfXXu~gQG7YE|JN!a6$M)Cm#2UM+D25M4t(torKys4#&pfuKeDT!P z{y$knp5JjaKk9s_&gzfG{h;IVQ8)Lohxj=z>!08hRbKrD= z`9N|19CKIyQj)@7@beEC3=V0p8372mC`6zX^j!41X$%DPoN5tL-Sz}r4JMp}vy(r1 khI)WKB!dxJg*qaDZ6-ROzfYw<;BEoXR@Yand1@E+KeSKnm;e9( diff --git a/docs/static/favicons/site.webmanifest b/docs/static/favicons/site.webmanifest deleted file mode 100644 index 34c88fed7..000000000 --- a/docs/static/favicons/site.webmanifest +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "Kairos.io", - "short_name": "Kairos.io", - "icons": [ - { - "src": "/favicon/android-chrome-192x192.png", - "sizes": "192x192", - "type": "image/png" - } - ], - "theme_color": "#ffffff", - "background_color": "#ffffff", - "display": "standalone" -} diff --git a/docs/static/images/kairos-over-wireguard.svg b/docs/static/images/kairos-over-wireguard.svg deleted file mode 100644 index 6b6c8bea4..000000000 --- a/docs/static/images/kairos-over-wireguard.svg +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - https://my.domain.here-> 192.168.6.1ssh reverse tunnelingwireguard: 192.168.6.2Other network (NAT)DigitalOcean VM (4$/month)Wireguard peer: 192.168.6.1PublicIP: 1.2.3.4RasberryPirunning kairosHome network (NAT) \ No newline at end of file diff --git a/docs/static/images/schema-validation-preview.gif b/docs/static/images/schema-validation-preview.gif deleted file mode 100644 index b7f20edeb543ca0cfa76f15c6fc49241342e5292..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 44517 zcmeFYXHZjL^fsF0B%wo4fhb8#sG*0BG!sHsG=c&GY7`47B`N}fC5_Zrd)BkIlZ&Ip zo|E1XGw?FV5F`T0D=F>RsiBC|Kw{L8I8|jFR#{sg<>-V)Zbu*W->s#K(=$=gHBd7$ zQZq7BGv1?aXrgXtx+NL}Qw<|iyb2C)XpA>9*D^HHCKzg)?a?y);anp z^=N92ue}}hh|js)^7BWXE?l}26nQJ?ZBB5YfAG`Pko!p?&vV1_D{j6Uy;;!8IC_u~ z7|i0nV@I4}KQCsNeq=ZOh`E~>dnGoOnHZaz75C_A!qI?)=qtDH-M-z_cJE&DgZ$n{ zEhCSgKYM!b$y4r!XLoNsPl|ZnJCIgemo~KUqPjTq(22~9w=XX-UcM}SO$~eKn-)lafAi3wWgvAK&n!1S$!>gJ z+;rz@Q}f4`rS(sLg{}ONFZ0V^mX_PTw6^{J-L|mQwz$;(rM-P=nZK~gUtIpWu>5s# zwPSgub8)%*X?AySU-#en?$!04g*8E0Uhmzc-id#`t4n?NQ~Tb&?Qd@A|Mzd;?Z<(k z9|PSHzfEp+}ky>wm$Y>&e+oC&-Kk;6LY^77ABioCdYsN&dHtb z?wu)nH#0RkGcz^&o;TatH#_xvj^8=g*)zAYEUc&#wsZ-n|IPoJoSzrY|NHmnYww@V zZ-2jTN%!Bcg1=vT|8@!%I(rso#urzYm-xL)UENDvg5{Rx()@{Lj~ot$~e=wT-ROf18_Y zYn$t$&5h0fkc8ZXQIvpp4Imc4aSBCo<0+b4GYCO zxj16|Lqo#PTnamL38cHF1o|Ib#r-d%>i?TVVQsFL4oQ zUIOLJO|M~zZ`bx~=4RBzAyi#co%3HbBqEK^4>adzHr>UN6Lc-!zihdW_sI5ZDR}ky zF~R>+s!QSPFV8G4{~l;5eAAvrWJ#i3KV*H)biS{B?9+$0oo^_auFqYIvcKhc7oPwA zsVJu>U9~O&y{|a8w=keH``G8=8T;$B(NE9!mAw025&8G`_s=Eohd$B)Da`&-?vJ|o z?K;O>OY=vYIKOT@-(Obn^K**vg~8U|b5-r>dzEC+<%N@-SsppZZ608%+TS^BkC|D! zV$)rEd1`Re!z?tjD0qG8S~O-x&~Q`dL>uaF`LC&gGV;-osFwDbzK4!lv>}`BjvsGD z`_5Hp_|bk3*N|nykC#WybyQ_)-dNce@$>1_Q2WQM64Bsv;oU3h67k9Oxq9Ohu0YFW zg&XEa-V9o=PmtR!Ml0{9?w{hRrdus?HSdQRFSgIu6jcgg>ts*T^+8mQcuU)%-4}dEUguU_&3`fH=KX&J1o1Bk+2RWHCT`F z3Qz7|XWw@*R7emcCqX15+a7A!#GRBqAnqb2#S0=Iw563FKCSUbS}XH5Tk-tp%^jZz zfbHai!n^u1={(tocCR@Q+n=Qx-@~SY3fs}(l#T;8)Poa}-)d)A-p$rLqxsWiohtHA zupN3{pPlFMso>zAySu7J)uC2cb|0UXSC^_gn3H4aMu%!zH(d4EH6`){xq7wK<+Mx*#}+qLtT>Dy+J zbqA~=<78Qzv5CuK;L1qgj?GxB=J0<@g3bfa`Xe*%SXPbH-QldiG(O0I^cCJsSxox% z)OIZQ_tS*2>Csq;l}Pfe?GU|F!%Dw=^r`ujzzW!@+<93}(wRN_)^_q`=J7@;Df|_i zHF0RW<@$gf2sap6ej!~UTa}-6Y6@txH0|fx)5&+lA}nWiRLvm3$zK^dYtQGBZ?j-18laYn0I7>l$U+d`Y@0bEx+1hz#%YCRUe934q%-6Ls6@$A8Si&l z(4he<0mjx6n{cdc={R{$XZp5o`MZ&o_Z0LH46SV$3E^EYpcopUT27!V zw$>?6#*1D3FGEaLjimU57i;a#W9&>ix#JjuCkZmUqk{lRa%go>cVSx8Slx|ds-Rtc z3^|K1BHT;d9&679c}$a01_XxncN##XvgC0{JH3DO*ZaqYGo$bf@l#-ua%3WNO@$0f z=82mJ@Svx28{qDiFq0Go2_x|)%Yv3KpL~k7(~dToK8>O~Ow>u@;^Va8gxD+G7l6IR za}<>C4mQOBJYG|s%4;cEkpO#6d@TsFiy;>^!BSAs^O1$~^y1u;AXAzSn5fTTS&Dfo1j1dI-|9m$X*t5K!bi~=ne7&5ZgaH#cou|21GjDri| zXwNL(oezvY$sa8FUyF_|z1}Y+5w#vPXOMhUj$E)>y5xMR0}S#5PYRrZdJR9dMR;c5 zuBd?WBAdQ<8`jWv`d{0}@3q~3k8UuXsgSc#|8iO(c8A3m48J1xPK30A(@s0k8G(~z zU&~k27h4C*4vnTumjMIng0mcX}ziJ6y-vcM)uHLcNHD!l0o zSp3wyOh7zK)Y2GVXhG*OFTGmWUX|w)sI!#+8}1a&5*MWqk5j85>-++PRkNA&{IXbZ9{PMyulp#MqX^Z6&?7} zLI?YcQw7pUrVf2{*Ce+m0Wg#_+A%B#PfBlFcr6(f_5z6FJN#F45%O`o7UUv?o25+N zQEN>{_cv?ZZE~Ca+U1WP;=f_44xSbpBfZB|4=u)~> zPzm7$Jw;BR^pQJ~UrZfcKD({duG=42%Oc4vP{8;}mOQWmhbwiFvXiI&yMns&2{W_z=IfAi;t@p*a1gI1Wl68 zh;x@dC`p|6Z8GOSWnHENDBpfDLv0?M)>MbtU*WiC`bb!m?hjD(Fd)+Lz8Am0nd#>J zkRa&37_+@H=ajb>ocK ze?FE-^^y{DBkYC`)R@Cal&d@c?p)}1dRnRL0s z=Nwc#!4_ER0+oCY1~#Y2uK&8gBgn!c#MeR6BSbNHO3*fuJ!y?DiAIPo^E4q4r6MZm zy4RixJ8L1IvGcOr!xe^#cf#w)ZC?P1@_InaLUx4!)#QRp5M~QPe^-$CIz^gySxkW= z?l%s`^GqQsvV+5LFad5`D0d72I?JahmdMCKp=$IHSG-jg0z~0!lzGR|7%FQt070`v z8o^oxCl|WlgJ;~QIC9+x$Z-VrL#;_*shqc!mAT~n{bz#NDRA11`glCdB& z4pf&4>MfP|siHZLa5CotnS5Cc585+)Y+)s^kw|yn@mP-nxrU9@xUv^a5C=o_=QK}A zqytkN2@D5t;X_Lv!B;qPZU~S$4M?I$tCc*#Pyw-Yc*`ST3atMi-5`?&ZV3Z!0TMI3 zQ`Xcg^(L~)2#{+rt)?bf|7dFLFOC-%qEavWg)D7MaQ`EZ9_34z5dbYJaC6xBfQr-% z-#(89)df8m5xMV);s6>Pz<=0R>N3=de{2{l>%|4exEH+m&nsH$ScR8yyFEfrqK zX<8zpFUhJEzMcokPIBbp{Bb{PWg$2@n&Hs}ig|N^>>O1kM-^L(l*^dk)#pu|6LFIE z0!klnF^PTXzlYrI{us7wf1_~9=!lVA^N7q5h|)s3>@WgbU1Wq&LCAQ#VQxENXs7l! zL(cv$YV4lIn4Rn+fyS88Zl#f35=>pspNtn1uc$t`k707dScFBm+!{r0k}9`~P56Dt zVvH&~&XXOZ%2^*ZRD6MXftULM%2Pa**DsW5{hdR_;};MXx63b2@(qPl{b8c)H0Ax2 z0xC^@_aAk+)>A6&VzNI8_~w&uge2K9BIZ!t{V$f7-=v&L>U;Og>bpl|C+2sXx4aT^ zWT!@Q4|&M`;L_G8`pZ;q$$ji-&HHJpT()-Vuux_W8*?*3b_^l+ha!s+)fx{9t>>uR zwvk;s?;F;i&$ZJr2)l?*mq$tV=L+X#f7jg4sJUMadwYpt>i63h{o=#E986P9j?BJO zxEvhLZntpwgHwfOaRvrcU>dM)ZJ?T6t!>p#-ldl*Tiw7or~H1r*62WvoO>1qomI35 zDvjP&;B6d3Ni0%ser3IC_@qd7^|#(%ddcs6Idt=VN>1_zT*=dlvef>v|JKXWmCIjP zmA~{Ye;r+(^{hO*qCB_1{N28Ct&6A#or+h9@1G{3K15g4lFFOERus*bRacai`W8Q# zDyz9rS(JgP>o3z@t#}b#`ChrI!>X#wx2ik3O7N_z?{itca&bX_<*-P(dc>-F%(r?x zx_aVS_3w)6>Hg~3^=hH=$3Ipd7koc1MSom*_HnJ^V+~Hp@$*LziU+agLH&5}YdrB( zo@6CYdVlrzipqZFnr+rKJN#-Cuhk$^Yj#!EpssE6-fl5OdvUVAy1}<{&aiqUqjGF2 zcZhR2=t6ZG4*g@js_5$*-TT#xhP5d<`Ph!a@n>>}RI5|6>rAXMUo7hmW!6Q%;JW%L zkA1D&y}f#jq&&&v%3N#kPHph1Y*>)e2`qo>l8c$USC5jaKiiT&&)}kzmH*(Z-ruV~ zsB8LqIaV?`f9hfV)Kl(yI(MF!-B!?e8fALuZ*29$Ce6wmlyY&0=#W0rPM?WtqHkb; z7ubb#kF>9iX$*n^xh(!#%SBZE0#B_Ns3)OJzjzoiP^K-Ho90G!Pq*fK{%!t5Zl1fB zp8+-v-)IcQH=lLE1f6e2FBvWHWdE~nL|tobPHp{E*?P%$n|P;$S2{F}Xjb+#DdwrV zj)xJX_I*XiNyv*+kYw>5j?4ZLS$n(VjePjU*Vu*a*5+1Lj_I&ugjBh!4&;SRNSNUY zR_Y8-th_^GPT0_$o3qYq_aR6>Ehi>^a+GS4xW?BIe$@cZN^V+9-vUX~3eCnEEH6Hl z_3M<(qshH#HGZ5Tag(oMT#=QNX_O`uvv)TXZQl)nNYsnChWM&ePlmJXnV+}mVJ~lp zr4fyOVv`0BSwjlM61ZYq?mP47FSU;_t*CGAHs21RyFOiAph(ZTOY!+qPq?acLTS<>td1a?E0nJ8jB7xgydVVf z5mLzH?%z~3eSKv%{9?h>x*uxjX0(oMrFOm^?*ilZm zZHFZGu~aTrwvh<+A@&KVePu#vVS1u~bM8)F#tE*N;xRQ`_x?OCX&xc9mnU|OAdNjQ zeX~kV^{GTGO@>dAN=naY7?wIs6I{sD9g?Le9^Z5aNA&-Vd?3ZF(O{>k;(QL2 zG7OENK(z_d9u(L zAP@n&J`4nM;AtX+IPRa=3`oi!0b}ySZnlZp^M0oC03Sl%84lc@C+3fkWgy0%>r0%z zA@2K6;tW>Y_r_1=EKtRjV-kUD9C#f~ESUiJ83ytRlq@3rG(mC?P0U#+?Zt4dRJE=R-s2aC5A70~PQ`h?~*jZZt_(niwTrr;h*)-~ivSV#kQ^ z0J;f<0`ukkxKjiTNQW-uijiniVW2H{z#w+eTO6nd4I0IVc~He~@*(HbWx{CC+nfP= zti&=!$_p!IMU{*&h30W!XSs3{pc;bhAGH&IU}&cYd*B$ESod zGgRp#AgPNtCsu^8d;ZQ|l_cmeiGG)Mf5P~r#ZaG?H@M=10HSMJgru)=}e9+u%E zV2y~?>w}Z_LW$9O@zXR|2oc^$gojaKYecaK3NeTP8|1?%^qI3v864i zZzmL8AFY?aD6hX?$H&X^UT>V5{6pH9a9(Tstj`nsL$wckx5GT28aaLlTC2I${>GrL z_@T@85b5p<{a(vx&#^(9_1 z&E#)&eRhG%r(?UXy}N$V{A@es_>cH&+Q-f!cWXt9PC9&5Y4CpJ*zdkss_3AIvwgN# z9t||`70gf9@2To&`}F+kaQBt-%(D_|M}3R;Kl$?p`GB|%Iiyu0b0Yaw~054Z?`ZM$jw6i=Z9jcu;dF%f$F!S!<#SMy0nB-kx}ssB;UGzKL5vE zbW-4{9hkkXaI5o7sIyKh2`oN;k#1 z?YFrkb{~yHU2jH?hw@ETy^SgLAMpL29?hgY--(&tha;(zywX`|^?RCTk^a>T&+J?Ei6&NEn3#zPhD# z*2V7b^`)}91hd9SF=OMb3CPoH_4!Om&GMbF1pUUhuv zX*c1P0U2>iy)Uk6K;lxPm%heft`FWmx0{yB5)-9GHP$pH95M-y+8wzRD3%s21Z$$N zOPAH%wpu7vTW8%^-6W-BHY zH|`uTI@Tbj)j_=$Z1M+#x?MU4IyPJLP0=Q}crFXJW%H_2K}7dgF-&CZ)t=|C3wyGJZEr40-*-?zlz4GD9Qc+QjNE>!;@T5)fbLV)j&s zLMajc+c{j09cPpD=5Bav=JnkExS54Rcf;rhMZ2R2&NcxhavG)w(ZZA_>sGj}XwTi4 zzmbo7sNU5P*KY?NjP9}ib8&BEt^rD?AJ9!%lKp0=6}ziH&b*r>7mf6n^LGXaVXyZ_ zv;vslJSW*_uRmOX+L<;I;0^&3rb3*2tV`>6+s5V(JA*laQ=mA|xBT*rDpkx_y4XVV zy*ta_RCiBvovc4Kt3{6aI|}LIHr?R6QIWS@hAB;dM&_%3eFg5$r+ob@zEUZ8MUFn} z``osbdx7)(_-Xu$ds$V9R~1^s zZ|kZYcs;|yFi(#;b-D0pjrUcR7vg`Pj=Og;88*GnC{!H{`YsV2;2vAt^~xY7aQ_)g zkEj1SGN$NRj#tV({&Vb75#7vJbMocC_9^elMvE!XwRgmrnzw|B)yG7;cu5k&C6V z)vZrTdI@URXIQ&Y84}hBf1MOhqOH>z;%)(u9qNu36OL4#QY;ZP7{p#oI%Dm>t4Gji z-gxoOl}dlivY=@%{1PX|`m~BtZ?jf46t6GJxV7$T_)_QRy|_t+jGG^G+as>My4j&y zx?Z)Pk2hXQX|O(P^tiY6oa5!E-zv|VmGpiIjlG=u)B2ocPj6d9?3_qJdC z&puu0V*Jq$Js2VER@@@#=RW6=&nF()zf`vgboJ`%c-$EF+O#V0Ktx|>8vII@gUxyO z$9-LI9Is>_sXFgj()aCs_?2|vdb~qxt@5OW$%C6O?*?Hfpc?~X60UwD)!YCB(9G(+7^~Y~{ff7WW!8TlKRf#K4sp*DAaDGwFXVSb_1Y8l>a$J+ z^w0ak@}>&eaFeuW?bNuHv?f%zRi@4F!v`z6n)D-z+smdx-w2LQjp|OIs^5&lf*>`RcZZ{K{NI(JGPk--349 z%Hw?>e>&G+A9*COZgwFhxY^rnJu&cObgW12wtpm&M=vqBTzYe@pkA_e`MkZ+#j1xAH5ly0@@W!hK?J?MB?q zjh}l(=R`ij`R1a#3Dp5k)_$$=0f1{3+l&HbCPGj58$owKiQ*s3_-r>0)QNAj4+JF+ zLxs^$VGh)(l@S^F&XG`Fr_EjCs(q#G?e-)dKy*XY=EO+NkW(q?$95t@*J=xN{u+Hx ztkm}_|BwjXahl}9g*rsqY1P-ZDA$R)fkJ1Fol7atshQ|DQMQjrS|Xe478?C2gc?(M z(ycth+ycYYTAc@W;`K%bmr49}GErcyWxvNjwayj6n^%QtE$~hNdyNrHT?+6+>VuO2 zYPtg4-ABq{kzFa4`k7GDOY%M%$+DGaMgjI&5dYq9h|K(We4*h)m({uzG`((b_+Rmq z6!{=pX=y){xB^lYKv8^%q_rK1U*Oiv(wzYBjd#?WU~RX8lAL37vmjnXsPkCmjt!o3 z09y|P#We#s0^lZY9cER3mThC4WNA-iI}0jNevl}({i|nCEo(zv9)m1oZQl;P$+p+? zU^`-COzB`<4n%Y$07{~;HCMps04N0nMdFz%6hIxp-u3}{gKcv{)ybU*wH(-bCvrQX z#4M&Gr`CfAb>LdNa%#24n*LLbF~fpQX)c09D22jyM(!a=6)q)Kx&^Sm6hbN9Q1?gx ziC~jcp!Oh`7oVYrg(-2EDM`@B1J(z))=fFqI6vs0^%ld00$UJ6(}JC)TccB0=|*9@ zVqs3$7!?Yn>4J&-aLZmI2s6x3Dc$EPu%^VvpebPY(tWx~E=7l-P8_I%1x(mq`7yU@ zUwn%0y5nESMPd*Q|g3h%vOz%rT0;Ov+7SAccSk70J@% zF)-e2U3{BU76e5Ht2VQSlfb(oA@)2liUMd6S$cdhn){i|W7zV*jR>X+k0~Ds*(GS3 z-8i7T0@mU)=G0njC%}5~ZPFK5=n1ech3Uuxqw!4B2?m(}*~&?D{DHj|?Dcv^bvo9s}uY zXAX+d5c8^n<3Lx=$kxk!ithp97-lpug2EJ;5kW8xL@5$t3}VP(`SRRObGi-AyF)FQ zAx{B!U1KVMpxc@u>hWN!BcO@}_g&tAp#oEz0#*oMkptL@L^_7T+H)Nu=mbO485leS z$MsMWvd}3WNG!~S02qEdXn|&!(-`vcU^j2D!U_lp0&h9q@CZnc2UfCRJHG;(@;Wq~ zk5KKqPao+Hy-e!gUs~&{lIiPcYhSDIilM@1CJcaa0<8iU5d&?!!6>fWniUN^rmcjF;i7SLVI-DE!0ivp2@ zLDdBi95zPyk|82bfTI=xcICdU!k#<*s~du;CpcuD0%33M=2o_HfvF{cqIr8)^0Pb- zf*=4!SGIn&%Qh0Q++@g8J2eBqN)rtE2{4Qf-eRcjzxU-huw4aAb)x%+?@$DuDJOp9 zuRYz2*Y6F2LU~Ly!B@=zB)93K4g#YI?Q#=fls6bh@r6c0U_8cNELiW`5enYdeFYQ- z1KW%5M@#?;kp~eJ0M-maG((WIE$((J@*Q$Swp_X&rI~ra0lH-!ncgX?J_1sueLtZ9 zfqMJyB7#stQ*f7AZ`k0knB(+JH&>cx?dN*~ejl_)$=SicE&&^vPEjH-HF*#V^&zBy zt%-*~VeTM32oifj@k&o~5Y%f0P>%OTQe#w0+r1IYy?lw7P1f%&$gcQ@Dp;o33Y5Lc zAcOW_HHnRAWvUQjV_z{mCK!KPffMR`@7#tVuf*JAJCOxjsA+?CGqgeMq|H|8LH1z~ zR0SKO$?H`?QeJB(LnSqqfi(D*oEA}7v0gD!9;^G6uL+ylG!x_2){sJG#G9LgwE>6U}2RFecE*Q zZNq*zXP8-4yMjeOIqS0>g}q7xZ1XH{oo^6TPFpyt-FkvCxCu^q=$K{WmD~z3&tjs9UNP#U zW`eC5gsqy@t2qG%EqS?NV51K9RtwLF?bF{x75>Kz(iAqbOn$` zXpG*(L1R2$CB6sU3<>EvfZ;QvUxTk#8+bMMncQwtZ-AIuoKYvZBh$~galy$bM#m9` zde(Q_bg(v@5gg1wBN@YL9tzD&&6G}E9#b(TMxhzvl_F3NW=YtvIph>zV)+=S@JX(5(^1A0z=RPtbQ`Qh29@;KyVa>N`PM@ zjIDqljt>t~g@M^&;C;fWgGomgkFY$_p-zc|-QhDmwx*5?WhYVAr;m%p`kN@H?pg1E zitL+N4ivzY$acV=-wT@kwZL+vG+TQEdjtB+n*p~JhtWMVr(xN?N!C%R zY{MNScdo1Kv`tFA!%bAxd|jpAU296S-9CchSf!w3V9_CE!J)pPL@maZenl9)&-&VD zk0zs>Ry&uQpWPza!6FwqKel-Q%N((M(O}se1eMc_kUO}1F(M*NX*u#H>+gZ(Nb`s* z5zEmf5z!}rYd0gVg)U!KT8{b{A!i<8nnh$8EJu|r-}H*S-o7k~4`qdw^qEdXFyfZY z5O&>a<9C)d1@@?|!uzJP=xTcXGhG8d1A);0T+Z?Uc#4J^K$ zn&AO^eRfxrZ8zT0YVrC)otb?}&i(3#ZyIz=>a|)!!hpTnhd&-lUF*&)%@keL^$3qo zylOce^;2~(CEcdafA1fw=s%e@O;&b$?bl1*EM(b77sRcXmqdRsUw7WUUQrTV^((qa z6kWX>%{v%f=)^BEk1qYy039eJMQk)Z{^BHc^;D~Af8$KxrV%FB(lFEFt+CaPR^9`T zkMU9;45hz3*uMGN+9YQ+4c-=d|STO_v@OdOS#Z&!{yA!+C!)7 zhMa01J4NL=MZGn*H?#jY-V>!2b|cU0hQ;CVq4zg_-~FuflKeE{#zbgc$BBO~6XDZM z#G%JG%KY{kuy}XRiI(On&MI7u?D(i|^lYH5Vqt1yZ2R@Z%!U>q-UTAIO&8DaP<6Vm zyEABloAD}MR(;!t4-+FV_C0{wM%m{NH{FFhU>;03|EPO_(DJb>T4Z9yM0k z|I9Mt^?t#j*L%&Scb}#l84pU;$S+3#|l z_f>0aI@}sbE8YBtUD3J)E!q{jg-v@RpR5e37W<+Ns#kwk7<}AZTYq0YQM+$87xG7fP%EKRNvM;t z+91@+@3S^+aNgz@(y;SFrC}2~dc&|8$GVnZ^Dg0<(I>5(N~6yry$b8OhSjml?`Wp| z*NoftOjR1U+pKRG^GQ-RCSM(tubXtZ=v0|>x>;?8wX0Qh2Dey8MF0Ipxlm==b1Zt( zRB$rU#;o_ucDGrZl~Rv`?}|LpsjU*#_Yb@?%ejAHAoN+4`Ox+CP4i)flx^u(Ncm{n ze+4`I4_MBob_QF|3AP`!TuAYk-Lpi$e?h+?_o?ij%dE?{&DXk4N!rX_EBs<+vi+as zbpB^stC=FX8}DqF>kHeB-Ez2>H)6qE5-f1?nPOFm#d zON}9J1=%BoX_3WY`w(6rSD!I$>137pLtLg}B^GWRsUMw0mTr$1v$14|S@Ut*oscmK z!jmzFbPgUqgrfXCsJ(kE7G@y$fI&1r`CnxL`9D63xGDF4=h zq!b_iHn~XY4CTCSIOTZxyE^f9vMba&WIX+@O(ku&3jt=kG#_I)ohkJSYqzI^5NiWv zDx9V^$Y*onuScrhI=G>V1+%iBIiY*n#a--QKwZTqMYrNDh*|A zOlo9e101$XUt}FrJBc}qtV8~egxP-J({r6v8MCW=8okM^kEd6+}AfG!w-vO&CYk zh@2Mjw{2Gn3ZD-~$CZtc?)cRJ|Me!OzZ=ijiJS&d#ZmhvudUv>-kOOH91v5Or4oZP zUS-H$$J?uFf4$}Y;e}!q-qASw>+MS)GBKNYXR>z3z2pxsHEgt8J+j>+a{*Mq>d}^Q zN4ZN>f7B=s?$$(-ecB!NMlkXAK;gW{r2!M=F~aXdp{|F6kf(MmVYA&Lw7xQVaBC3(w>ec zr5O&iT;TY)L+Fi%`AI9Eyy`x$5aP+YBl7o4>qQ#2h6x_IBMOD34f;0>ll*l@cXpOG znpGR#xtu$S{#)8axFcliw7x6yTTv=M(irtU@$`4c%$Ck!!h=lRpIRPepFFFL9u?;P z)Vo~v`S?GhlsesU!~12eXKjt2c1r&Yprw2{f8+9l@B7bS8uhwk?BuXmOTGgQC)a`GtecvNlj8m$*d&WYe-?)-@}pP3*y zmA4=UxeuNVoIV)c_qB~ul(kQ0>S*HH*Paclh8HWRKE z`hx#KAHnDJ#Rre|g6+5a^l;BE8X4sUpD~W^eOxRm&d!$!Jg>9sTQ9u88b6?KJGuIi z@%p_<3MzE%>=WMKa6?ErE|i3B{+jzQucv+MoEL4SuBN;E)3NId z=T0bByP0j|fBqhUFsPns4Or?{ww>MrQ9C8na$IyzdSLH~V^!}R*P|8>T*-fLIjeG` zvdiJ)@?P@slJbzEZ&R3whwIkU`{g!X+U%Zu(EjhcjHcBOn=iB{f2(K9V+?{aaY;Te zebF|b|Jt@iOMMV~w$?`1>$I7RIPGmauXAqC(#1UKPpZ#%Dlb(Cc0a!~*p}MT!hwfs zwfBF6Kd%l5{kZsR{oDv^aM4t$;$OpH?^R54jpqg$5TXY~MJK*ZdK#lx!aa5Id5S+< zb}v`?BtK7!eR85$9auY~e4*%ZYo6l7^C6SSlqd(5adUU?+rFHd??zPYT+2Y5DPU|; zCv^h}Bv7Bl_zLv^(L;Guy9=0}bT(n|0o#$k8V z(V-|emVxk*(Ms+75mwagk9oXP)2v?=OtHMh%cV^*vb` zt9C||L4y_M+x(*@SipQ1w--Y;O?nY^Yt;6|qoAEH34E2%i>i7zMnHFliFeXS*~2C# zy+nwrdF_zN@F^qdj954F`CBTM)h`LsID?Jay$4k7{`-zssWa*tF`8C0=t9~5v$Oo8 zB9eG!lAscLg=&@HZ*pUdps4QfhuJ!*Vjy0pw_#^QBkOpwOiTbf*-?a9-Ez$-rzI8L z-=~b5Q3m#P2i{RF##If35ZQ#$bgh@*PXmSlKhuAx2wsitJ3uxSo+#dgEZ3;$Y6IpV zFT2?%t${4aJ0BPeo&9Yoc0-ORCQ9#~sd-cgLPNe_h?KR>j_D)TGtlJgg2_ zW17XkHn3N9Xxn}8zwetQ!O=BEpQRsoSC&D&?^%sHqU;z}Whm#nU)z-z3t%5pK2s?Y zth`|tbe@F|W1{(XL7rnK-T>|aGsqKg@FWL?v+%i^+H6hz9d=QkTbdX1olacmgV^srh~ z@=x}svoYZ;{THeRT#{D-gXEcYJ-kjb9MFDoj_JyZO0dI4l5hcJ)0BzW>a%f6zaswp zN;prBaIL%XiKGd_v%^Jz!wpS?{aGd}S!Pf`Dx6IFb}Hx+3!PMp1qDRhc=P87U{wD) z1UGJo4>V7Lpqqc02uO_AqbBK0C26LfG~{ZdM!2jeP8=}kVnm**#nz9nADxKP4vbS_ zaqid!DF&ENXQUUaSe8%R*rsSIAen6YO>H=PF`UJ5XFY`D>32>B*#-vvR^dpJai_CP z)dI{Z07ht+6^=J%XPXVwpuX8<1;&*l* zFXFOHV-364ugV@h1NQ zOlkwn2f!LU(WDM#s>D6ebO%Hk58wtErb5ji%L_)JfZ$p@U6MrzsKbhr<3432j_>|M zN+BRv)~M!+56?nmbhX(G5}b_N0Wpt<=*NTg(jn#N$(mhwocER`X84F5o(ndQWNOCe zHZ?Ld_iHz|fsxY?jU)(W2SnpAS*w+y8D?ifsl)E5)vvTO71T!c)#%WG?OkB=9ZWQh ztd(R(h=*u6YnhG(;Unz`d?uQ0XPyqx=mKj8)b*(Zp(~l1sU+M3J5y*KN>Hme!00+< zhf`w_BJI!tn%cArTLoF=X^0-5iRafgT>|vF7=wc~Sd%>Tj#-p6Q;EhxLPdbSAGozU zufMNGf29_^qgH1nk6=;P`|?Ly`E`2_^BtL5eY#;$(qiy`ZR{7-^|c~b6No`R6rhHV238u=%+H!J0J!q zmPtGbNvr!)ZD+Ehc1E@qFAi)&=_`k^l){+EYq}?|=bAw41bww{2lLQ~+U!PB=Zo2n z^L6@y8vg4X!UKq&bS;)v*I|jrjm`X!B#(N&*F(%;EA6(p3Jr^QXiA+98`9jt9KJ-} zP_#370GSN0!(!k4nPsj#vJ=GWVW6xP#aSJ0uxUL*Pg-=5P#=naz*J&GH0%S>LhiKw zMKrYTTOaATW1Y@0L|=SvVz%~2fzXg|hupzb2&?OV7fcXBa66ciC*CO<=$-7XdDO?$ ztb`z2YxLupW^6mOGiy{4^0ff0y(46F%}`Ig4pnJqJOO^*SBrYUH0`R1?1Lz=YhQL7 zO7%mOA23bgA^qoBNI~t+`nTPR5N+K#{oz{m%lcbMfkSNd_zU%+0D5npW=cS_r=eam z3Ev7OLmY-UtU}Z83Ol|JCS=vy^g!m7_@(7xcBgO*;E(@M&QERPY{Sh8_Xj z%?#4wku;|m*Zx4v>uZ`j>@@4as8$jITDLQvX<-b;Q6bxe1buG?p2INp25YmK15B`% zN1gsK1DVcrxeT^h*XwrI4jzGcTxRG670%q4UhN1bsFBT`4Ycllz{ivD^);{vG7@Eo z5(mg7a%dh{Ba(p!fz6ZbEJpL+{x_!(MkcUp^a&)b2ZqW4`deI|xkonMOG|Ha8rx`+@EZ;ecy*izr$Pp zGbM_dAAEBKPbebT&+4PXLZ;>y+srgxE~!V8P*h0MtR0S)Yo`0>+~vAs#urcSHp??& zAmhQN*rHEMVOj%METjM>UdfOw^J^ms1qF`J8DqM@Ha#TdhhS7XWT$kkUOYoH$;h;_ z&?1_&V|N&V!cyW~)S{C@3TsfgrliDTrPMH|3ua1KhAu@<<3Z>-<9y2=u+IGgy|Igg zO3`wnRV2PO618>og)b}cLISx8X&!cpn!l8L1|*>9isvQq6S5z z8?n$Vp@j}XP&$Sldgx6tR4D>MREnSh1nI>}H7CF4l>c+axX&HujB)RqyxA`@%U*Nr zwbq=U@72>Mj7Py85?|EVM0Cgy^+;F~HS=iEr5^iQp{xf-->M7-)gHxd+!|{~Pf_)U zK(kj>hHCEW_;)t%t{XqmXRS)*{!xc3XjTPeRyLrt9v{$xs1@u-l^zM;ZF3dw*^O=}U*}p9VDJbK1q< zg2WuyPN&o#)ny~U(Lc>F5GREAo~-agdDC-tonkxnHg?tBnZ>?_J^?=}6lfv0&*^Av z`wdqH83u#jv{d>U`X6;}m*@>q@Cs9*g|N=0MHH%zDrg7?fE$=z!g4i2_Y9{m7~UfZlx=6$iZ4I+JiEj_ZpvtG3xE zGHi_)QuL#|>u#-zgJD@}bC~5rG&VpzKwo1V)NnOq?jtj#sDEJWi^qcD)@`oB`{A-$G*#?L7M`HDxKIy2c?}=x%A3yAydB~vn zkU?;n&zVbxKhX5zt42)+nOF$fj4Abtx*xy!#MLsuz$$XPU{$tIb>p6a(hL+e$oU2++6R@{PN%QsQ{?%<<7Hy-8M^7YK4q9pu zuX0oVWjF5mPEm-wzsE+?v3FnLO4&GV za&c8d0iaJbB#bnc=ZM*4Uf7DiuiufBmi9y)VGSQB1Tm4dD`bSGW-J6lB3 zZ`uMUXvc8(2 zMISzL9-DhJ_2dF|PUO_;IHZaq{#I4dC7@ltKpSfwliJWh&FA-Bpw4&rX1J>rXok8# zo@vj*4rd&_rYK)>G~)YP3SE}3P;S>Cn(eF%S38f4Z-HT+j1#dgPwd*`J3S(9X?IN0 z-NTpMB=8?gOy8e3SE-i_V3ZuqeiXAcFL+bgu1M7B)NB#|Io1IUs97q+!G*gtKjQ*jw(E+Y2NTqgY(rVnn8bRP7hSTwT3txOVr9M- zxfgNCr}<*r1m{2bHLHCb*#+l`UACEXWWt4CnvdZI{H?&N$aA{2ixE8?4rfLe40nke@v)mm}Gel9Yehb6LSi;e1nji(4M7dLHR@ zkC-HOKA_tO=kIvzu(S6!sL55|sfr`5?&$Ozc!QzN9Z$YU z?7F9|9=Tu3H=*Da8*qY>;2t0dOZBL$?e-}2ANeC-g+Fnuyw}AhheWFLbX6ad_E6La zMlG(7zKmzc2tVjW%Cwx2Q^n<(Oi?(zJq0`Ju@TOiWhQmwf{~hprANn_*=80tV=|U3 zZ&c8{47MYj?%23$!B1gt59pP$wSY#SEd8j+>%?ZiFZnwxp zu?=_A6W>GQtl&2^3P$43UP$t=N@kkyH){5HiXRpoLf=p;PcuF3loaDRW2`=JP0NhqIMO^GulVjlMsc~x$rjD{BcktETb-|h+4@%28Rvc8^yZ)RQ#A2auMu5* ze?zk3#M^UMt7C*$h)%@5UisO(aWT4KoLY)K&KxP}2lsAx<}Jn?%dqiq zs*bfTm(`$AW3f)(NuDJ{4Gt}n_^!K|m-v$<(r0>1ZEoaF;uIAJG!c=$-$~h4v2u=m z=Zs2iGgqx#C9h1ziv|5mDD%!UU{1N4dr+)1CVPdp5PaXWwU)-x~Xxfa`tL(i)45oPM6J6a-k zbArQPIaJW?h+EQZRo-?)^@UxBCdoOWRG5iIcoO``5krx;+pgDU`p0e=Jg@Dic{%@y zDY=0@966*}R`@goKH+!D-g9lqvATbv*=DXf!%yY%qdlhvzXu{(;U9`p&QIN>{?m=R za{b1!%QWrZ%~n$>_f5qR4m);@cddKW#9^OC*WZb>k%-9P?MowSTfzgBuUF^$ojwYV zRrTUcPA1%e_vP4M%+x-n_HmCn@yji>%Iiaga;>K2HH9#-p0KjxIeVX4G9Fx9uU{{E zu7tShKKgp{!}=YGfsgYoLS;qrRY;9i8(%ZSTRRcdqA8u>!T`VMx7d#tjLIBsB36B5#u{7fD!`I>^#Wq-t&f*P+7!mvfbL1-VTcofhM<6iS4&8E(cXVZ>ez# z+^)Ug^<%iZ$JT3b0zRMD^1ddgf!rSNiG30vI(hH9-;Mhv&9liK+pa{d<8k)Gr_^Gf zKUR0G5A?LX{Hf<7vBK-};PQ`_j0Q)e&7cqV@r2`o)Nd+(>|@I>>HXo!tgElJf207v z-gS7$Px6D#XBOJxOUH2Sy3F0fFAW2a=TCu(1FC{G@e&vIS{`ex`QPoG*3)U3vTEZG zu=bvt4{eFb@9{<5*WBE3v&i*wn|;frGU}?JC3;4=vZ*@fe#PzVmB%L9K~RW#biy1?_U2lvfZs7*V9ipda3?Rlt6ir{;VTXh}!G0 z?yX{H?CM=gAEmy$QJl2q)@v`EPhVf3=>McQI!hG1p1$=D`<#qF?;VNZ0WG+uO|V#}E2Z?>X=XSe34xjs34f6Wa3^`8T$dsaxM&zS5V4 zQr^H;DW1-28H>=g{)%i1H1R=^DD#weY>h{BIc2IGb+mu*UENYnrDE@7iJ0$Hw2 z6JN-{@P9WGZn>F`ci|x^W|j8}EmB^#gy?jc`JlN>d^@t9TH3o>P@o~vnpF0Ic#n>f z1Z;_zr$HR4eX4x1Yu^_!K}Ae&B4*IYc|3B7id@^AO-s0V3Mu}627G|uQb*e_Ll1Mb z!(QTUtulBrSzn2vVvlbk$+OZ;(nnO^Ag_1AM6KYL30v37C2`5Ad?%v?J8UpLUWj~I9 zE#u6Fa6$$lum?R4m>mrGBE*G3z+~kihX_JS*L6yBWv`k<=qfaD(}9r_5+CkJOjJuu zZ%NFYket6Gxl}FLcN`3g6C;>O>~5kY&e<~bCHg)8uuc?8Wz_A%1if$Yrcae zPNi^M;tmlUA{;i+KTIQw;tiE%ama@uf_1EIZ3veIQOyDb&y&$pi!tsZ2sVL`yT$S_ z8Dvn0$5e+$C`q)5AnFZ5YUiHrBJibS;^hPdBtL!E_NJ_5*Ut_;YYW#Wm_m?FM2UB;tvftOI(~+X@Yg&M}&8*9x^zC5rNMW zZJN_g;X6Fe8o>tXXAlrwa*FrlL?|GxcTwgZD03Et<*x1(WddGEVBN<@|7#b5U$TVn2 zuHtxdw%`T&GK&muo9ha*1pRSZmSoBiVaTC0GCyB;*dhVOtuM^sBKD?$ADXYZoo_b7 zfNSfUFwHeqq~Y-EN{5fC)`9jO*qmvuvwZc_vYp>5Q02Inj5tGIZ_jVb2!ftadR-uc zuXEuKXcsb>x#kTDrq3R6&qaIB+@!P%73Dt3(}xFy$Xm=D5ml~!5F+}p(Pn;@gDM->jO9W zTj8*}kW<4Q%nm0=~&*Ux3N-gJvPfD7_w1g9vTo5x;v zM5%lz=Fu(@nSzS!KYNy=fJ^Ux7cdxKemsr@O1I&P_il%=VmLIA^_^1c8AlCp(^+>< zo9yeMQlBlGLq0`EFGNR!KVB`-xM}y`rsH#`cxzOrj0FAtoZ!ag&U)tN#=`pBy4Bcf z!MD3j%T(P(VO_;ov#Ou8W3QjbRz0@vaz1A7=F{3^CBm&@uVmb?dQ4;pE?Zfck~7@9 zdGWcVqCpz5_dWk@EeG zs4w|Ks1Vs+!#^2ZH$wca8R~hiFz@O+*uZ_UtJkr+a%}n3T@C-cT4Nhm?T>f5+}?S< zwJ+!_zajqRPV?6lyyMv9@`-7hVApyzbG7<`&z}c({%OMU4{Y>*2{~fWmEsH zmmlt}SRpU_2Rb(1&HfXZdpbzx&)sO5d$;U|u=A3uGj1yW8%9?5(^CUy8W7XNAjNY* zBc~sI@PEXM28T5SeVGY*!)*ullYX)Gc>ZTKmrI{O$5X?fmbs` zARt7fDTLKN%~`(j*i2Hp^7!0IJx5~|@v*$mC1Gc_!}K-7jRXD}I$&CFho9GsunUNA zY>K$D9pR!$yct0BXd?P-6Yp$?Ne#uQCmjD)E4j`4^}URkTZC)eLR6Aw^s|8I^rqzQ5PGsosx@5GdWvY zFMb8S*lB)ou=4_>O@;)Kxmw7GT{5qBu0T+(2&-i6=E`X2$p_^r2IV$?NYmM|T@NIIyMo z(Qa{wHYFm6az7(KJ5Z7oB)KUgULzw>0}`*95zPoHt<5bR4H2&i5zpEcYnu`61&P*x zM0;JztAff|$AERBqU7DuPN(utP+65s<$O@(QcLC9ZsnGC74uh6)lN&*!EO~uhYAU% za5|uj^pZtuy=I_f9M@Hy2AAN5o0l812@SXQ8m{Ry-U)8>65}T5G(`kAMYlG^?=>aq zG(QV&PH%0_-fPa)X(xP(bxyb2^kuhYR@Y_c zt}73FZaMe(ICtL>@44;#=I+Bck6ykB$$As9|0YtmH+sJ}UbipyVP8^K-?Nu}x%+*E zQ+*}6{bdjPD|82{9}ZN#9H^ffXx<;_)_oi1%;xYAD57BfPOz5kvJHUIrC2b(L2Rg?^F)n zoj5b9@n}@*)##anQT;RTjUT-?fA!w_;QjeCAMCP64_#rSi%X3Q+u4eh4CHnqaBR{H zu+ht*KjL6X+R&j#!)yB>bqqc13Rso^mIT1M7&fK?0UUP)Z2xGoR^=f4A}V*H}28ktw)mY#P|fI*`zV-x->Rc2eLE(HUq#5S)kPe(7G;IlmIrP zfs}y>5!~=m>CeBi>Bh8a4H}5m;3@HUMn-Q|{_(70+pNmp*%Nwm8jt6++UCyuozvI* zYW(=CdE3|1kB7cFvvErgPH1)}X}_9rX2(d+NYU8MXlzF0X=@C-IEG!4F|9#hx1zDH zPqSH(Cn6trMd*FI)ix2Y2hj!S){mD<+LkN+E?4WV)IVNnZd-XJIeY#y$Tnww@YV6@ zhf?p)uo)59eRJ522q1NU9iQ|04Egh&zY9zaX-L;+HVkcI>+ga-83bHoo63fW6W9fF zMqWSO5_!EP!Q7HL`(6IYcX_?lH=jF}^gj;%ogewz87aAJ41fg5(}x}de0tw_$?M-< zuHDID4HbNr0oZvk>_`H;+w18s(`(#NITjn$fnC)PLN~-hxqyyk^PEk<BTjDKJWh}MJ+z#`?3BiAyKu>SgqQH!7QENzi{m% zxE#H{S7Dr06we^jZ_hq)dgv1=yt5xGB(xjXr(Xu|{Y3=T;YOol#jZ|iJ{aZXS0xOZ z>DRDxkCt5IOgPi}du^&U__*Rrb(gvBB%L0fkmXShgq*{8Oj3vw$0eU9uM^L*dZYGT zh*EK)fl=wjSsZ-oKM7E%hjb`AO5iIq1j-{ael0@qYLpA{^?|ALnKA)m25Y#93c!?s z^K39N;DjMULH&Fu9Mel+&6_*mLhgyK_ufd<``gPGxdGpiInQoS)<+`J6xeu<_M*FB zR=qSMPU}3L4iFE6!!`BlV=gzML7x3rY=u08>**<5o*NJab{RII<79#XQU*;9$&&Xb zT}^xZm&={}l1mYH{Z_}3PwTmkgJAT8M2w7nSU2#56@9gT9 zxqa&1EgyBozEV1cx{=^RWn$Td?%xBkBXRtx6$-}W5H5I{j!)})n-0aM0AcTtZg8Oe zsa9kr0OY(HQ=nXG6Nu{s4YkMOJGgK3cvVHJKC7zcyofo`E#y!Rf(T+)*C3y-Mm~qw zTjTUO4mp!R?Xdk;{DA7~)fX>2C%jx6;`Nn56};lK9|oMgK^Qh&<~}g&B>$5JJBhy& z`p)W2A4JGXSeoap{k3?F0JT_Cxxwv+2a^{DM2_jRiBi#QF!2DiA-6~x2@F$v%K&k@ z-^PJq74Npe&(8JJK%7U>C&J1cr4lq}#Xshsn@c#0Y}R|7Fs;Py^$GNu-DEAZbZE~z|alVKSfF<-_Td(@*>@lDUi7nrtI3C6mutw zz*!m4Yi8m1G^M;i{&;ksm8V~F)@s4gvt0ey7{8PP>!bQbB!43nqb4yn@MwCJ#l7RfaYcg-zv!Vh_wvP`#jf|((F5g- zncgDv7fjE^yi28zYW{Y$Fi(H>F0J2I?wHE3vAMQt{LsB(-TB_M3y0ouyD%*`$sg6IJDGdo*kPf@Kal-a%7kCN{llVVV1z z={a@bM`{0NjQ`88Dpi5Qv7djI`oCKBIWGTfuhR!{ta!4bs`a6W@Yd!~Sy$oe-8C4_ zy?v~u3PBAy3;FWy!UHO&FTR*h_X3meeLFvr|KaP{kKjuIomcqQ+|oVmzW4=no!Koh zR&@B(e?7HJxstTLx@-tO+0?DxMvC;fJS8v|@aB&QI?88YPTK!X*N1~@Vnm$l+I~tM zFS7bhGJO8A_QlsY+tQ?4;`7RXO#CmZT(GP1a#V?RJn4MqghQh4g1Z0R7GCj6tRlfp zW1#tXtXJ)ZVC=lsiet8SrQegS{jX#Us;4QW7SZ zDNFO;plnVf&(_s#_POR-Pl@#AsQ$8W8~AeMqWo}??|qAV!HXYlMnBY@GS5vfcX<9G zXiP4Idi`#X$5q(n!+M`TmPf{~#?f{&qmZ|1Uc5@`RJFZa%{y0}`^w8UwXc=noDlpP zv5t=op8BfV*gTBz^$FH$>vx1Ud_g?dyKLM3k$AcxR-()6)2H`smA{4mWUpS29h?BZ z)zefQ@UCCR%W6LQcPLtaH8d5SL7i^OUlYxXveW%Ey7oQti0Nkh53%Hg6}j zYR|r1Q0}+8x)O1*egYY}lZC3hlbPzYuu}9u`e?;nt>b~Y3Pb*Mf7Oj?r6xsF)7`>8 z=7`&AvDO9av6?E*w&l{3yHysKG(OZL=P!?bT*O`bSz`)Gyre>`H% z=d=#R#W}66>ec##C))Q-1ycM;bGj`Ht!d4zK93e9R)}LKk6oFt=dAadw z;|Xe?0lSu5d+wFc*44C+LXyADo*%HgCoY}(7ABG4sP`jm_2HsFW$-6y4^X)GNafa- zZA^5~nb}JIS`F8O)!VQ4zMz=f15f_`ta^R0k;vR%dBWV^c+F&@2p|Oln;8M(PT-7U z61X&kVNIB1=I-WfVdWn}y1|vzoa=6)B;| zdP7wfLytqlPAG((Gz-&U&0rcvSMlyJ-5aO=hJ zbI^$M3K190BJA8FE=5H+QX($*MqF8pxCSMxiHhxwieHRMOpC-fp%`S; z!Y$!xA=Ee?aaJEWPC!fmF%z57V|YZaa7-Z-@d-dM07S)NOcgY?T7el`YZhDY9@`id z+f0dV?Tvl482cI;*RBxPX%^S*9``0Hu8$HoFdoYQkPAZmD^z4yB&Q*j-Hw6yEFC#V zMa?oe7!1>=O{gsfvce@{jgYWJNm%JkSX)fkfF^D!B>pf<{NNIo5%tW}b%)0cc^Dfui{3ajtO&^*OB zF!{SQ>S#J?$`tjk7m4cQo})n@NOQ7)s4RNeEE-~mW?J9zSKucsZY4lLKM@&%+n$~(jueNqSI5C zNXWgLm)`ZpU!rkukU6iA*o|1kEh@wv!@;78UB+;JBEv8kPJ0q@3(Fr@l2O)|QL&U! z#g$pDm|1I{S?`hA7@gT%lG)mq`D!WiHCI-?9Ag0uS+_vZYJI4y8jskx{>KDN7|eD}6a@OF0`{FSZn4{4jrE zl#cq7jNz8Hf-7QbdAXByf!L@c+i4a*4Sj zrMY7Lxf08{QrvkmN_mGZ^5i}9j>hCEmgb#Nm%GxL^#FoemQGmOMDA=7F%;ChZRG5t zrv8ZLHdleZQh}jGfw5?Oh*fNnM@G>Yw&*LS_`YYce@t;;X>o9W@uTJ9C)|_} zB}$kDCBl;u8AFLKrNs7A;+H9j+$Bj$CE-Cu6KJG?OLQs)HHL-NpkXyYX>KVj*P^rr zU0OqewUA)1Sg^uUSQR(3tXiq8)}pN5v#c?uthuzTwZH7ua@lL{@^+>2PK)wx&+<1h z<$a~)1O4TLF=Z9p72Rm3^nT4<46>ezxMzmKDcEgNonHQb7A@F;8W8ffrYX&B2xO^{%FxW|Ia z`E0Ef4-1gXRHB+_{^PogV@ge7mQ4{}O_8xp(Pd4s15LzClwTQ9I40v$45Edh`HO_S z){0u8qJ9G5`D6rU(G;d5=Q7z;9NSV-)>1amQnAufrNf`7(<-owp!IhW8D-Cya8db&gy`pRC#rQuHP+fFxLkz+i zjl4(a3LwDZN$e|B_7FhY0oJ;<(zXG6y>;yMkJzS~PtBsG4+2_|U(m>5G{hIpUXF+C z;+dR51kOno5tD#&$8)YxAJJj$yvpr|tl9;<+a+~s+9oMO~->`>Ai0)U7@%bMMe!WaZRggI=q>(N7 zu3-Q{M8mon9Q8D0%69$P^3F?f-HzqmmmhZPI@i1jR>`qMv0d)Y6pQo~j!bbt?VBOTrbK%l-#QQSReed4LxZddU-dJ7!h-ws_ z(J9)3V*AvrHIO#c)EFGx$Dn3hTn{=S*Wc<*@@8JTq15Ol z9zD$+UGF{67&p*dKG6Dh;MMBDYxvtl3T&KAMCtU^A?pzVpAn(>5s``! zvB43EwGpXDMYr9fKkpx+YEyR@ntn|cQ@cdQE>WQZ^+&5kcd}xOCt+$*(=TmwPr5ck ztWxJu&*zAq6DpB$hu)gTzc;UVZ#nqhdhLCLLv(sj^Dh#DC!@3l$KJy2pnt}LAuGu4?z5(G}IxXodrk~Nsd>ZueS0usfeZ&a&3?B6$KL2y+#~D1V z6AxQsm}Xg`=4jlrn64cX(#ZL(v+_i;^~5uuiHI3Z45jq0JNF!Zj77H$LbJzVQ1w{& zXFNNB%B}FI;x-gj5Ad^~nHvnmG8T4M40;;F>56NvDkYASVAI8%V+2$)hPr{{9wTvG z17PDU&L{@*lEmJ91-3$i+A+9;c(TV^2D|a~L{M$2%I7)j&+`Wjsja9sl_AfuiN~%T zOBl`o8aE3gnn7i=C$ZbCvRh&}=BTV+NEepL<_f@9Solmjdpd?4jp5Y5LGy7C8#>eu z$L32#eWF7?;MuO=*|Ny&7F13^Iqm5Xi%qT=e7rjbWl4v)(vg?w zT&`pULwc-&%6^%~d7g#fMk7_k=j}+4Jv0l|JZ(lrUE_gT5I8=Qpk0??Bm#R8jUyTZ z*=D3)+~l+a5Z4K?O8_c>z-f<#HS#R3lBgJ)rDwiNsR_b=cItP@ux>KS`19voTt_*T zZH~Zx1qUsnvXzpV+zxG!T?YH55|k^Q>k<|=M@8C_ImgmC&f_}fs52pS=tVk54-aaN zG`oiRHjn4Dz;axnBMos-Ga9Ei4R#F!y@G|fE3nVuA<=D!d)SO|8qAxD+-5<52ioG^ zuCJ}HZ?yGhq{8Y6J=tE|UjfuE3yumPX7#vju#jd3`?e2s79DXAiLkRGS^dlL71U-%CVA53o?d_n!4IsSL< z+~4`zf4?REU8?%KGW>UKfIt9Cat+E3fY{mpm+;#Emk*SH8~_3Ev;N~Y z0D_7uV8QKS5P4yvH0(@A7_X!*zLy8!MDVNXx+>(cg~g!V?g5w(17iSg@zix3WN7h> z^CB4sfI2Kf74_8c(d}{uIp9ZmUz_iKvnB&UyZqYk7xk0P;xpg?p1+0baQOkw&s zkMSW<0Btr71tK9(z+Tqsjv-4N84lJ>GlozUYGls|QA~#r(aD__{2(RML=uaUhgN*jc8^Rin=o6#9;09@J#FYV&e3FVQ##{fDzIiN@-2Yz>SKIG6TD@_7c zM$QlrF(%tK%eWd46#@1TUL*Md(qYI@T1cKH55E|XJ9(QAfb;7oJbsAJmDLwLi38Bw z(jH-@`i*WHF?nEiZXD+jAx)q(62&8lg&$+l(Zm#Z2wAXXa95m!TP3^F%n=Sa9F1j= z-+{1UfbsOp)X;K?@1t>F#@}uz7B@C0~jhAl#dVi)FJ9 zM!actg6dN5>6ocNoxte&0Mfx67>As22aW`O;Wvi}C%`M=&gmq1f))@2!g3I> z)&?@cN&Eo=$yCn@c5Sv({KA0OC+Uy;n*5j#|65J?k0$-c2ePb{1_e+oTTNNErbz3P z^;x%2vQ|)>OeZ%IA{nsBsw<&j5nHl54<-!G5A&2}eS(C+B*fgI(IX-3P!zYCCNL`7 zLxdqFUutc-Vg3&S^dGJHzkl$L{ha@@{~XKya;!ds#sTLBQ|P#^NDlA`aeVG|{Vs@- ztvFV5tIH&W%{ib;nhta)!UUt7$@H&W&!x}V<7wo%P)^jTaXBt>I}yPFQHzpBOaCAE zA;bD}9)OP(ynjkCyR<%-02Pom67D6D*}3_p(KuiRXUHX>gds56!?{pmQkudF`ZFe| zQ@8*OklSGv!lmJ7Vm`tXZaPiFG&bPB4fHsYu1j621wE2ml?HQ#}G zU)sdc;$hkA^uErp z5l$_;7?{gDxH%qvuF=tSQ(l9pxRB=h++!i#y{Kd%!@FT=Ars%D_$}++M~`pW0ShJH zavuI(`j+L;G5$3fsn0J4OxP}c$%|99h?aYbdpaNd+b$-$@Oew*QsHylyQN}wU!~>3 zf&@>`5=u^~XK8umvS%5!jj7~SUN_|FRnhdh)T{F4`m$FQ?XQwIwF~K`T(ae&Ca&^j zoUKro%{+ZD`ckOYk<=85Lw*eD!OB`AgLv#ly`z22Y>Y-s*?P-r!^(Q=X3w#Wmp?yx zZM@oEDBEb;|Gl#D8sLPN#5}d>aIGCSSwTI^q&uiKjN8a=$zADi-|7~=Rlb!v9#fLn zBOR&yy;m;P`+J{4aryUt<;KHIP`Z48mLyWTI+Ohk}c{| zvbo;ZO=2+sEw?~rU&0*?p4c^uJYZ8BQ3)Z7f2uXiDpc{3z*9fWwV*fKYvJdb3W}> z<<3Hu|N721a^&OYA^gCB_w@GELGO{+FK^6ZqMT=EjwG6Oa50`*)%31muYt`K5cz!h zN)%yd?xShp;OH-WoKE!x&v0OKS@iYv`o-~ERR_PP{5KB%FrIj+4_obu&vz^v-|jn{ z>o8-Y=p~$Mp;*X%-AMYDmBDM?F#L5R!K_V?IJi2@DCIh*Nr16J5D=kk6>_NR2Kxb4 zM%NOH6{x`&N&h88UEo0-DkS0J6olINGB(2%ta4f8>MGOKw--6@e%z@!&-KwR&m)%+ ziqcF9Q!a1kblmLZWA5UKmcs2&wQaVeLsa(Dt7Lv>yuP3TZ`k=?XcmQ&_Vv@%=< zcFQ5+NDT>c7!$&)VeU7(%-%g+h6vsmzBUBVZZ%mf%KDwFA^JOAioFH5$wo%1rK zUCJtu8LqrK3O%tMCTND`HuBE%9I&Do2=&v6+COu7;@#t~Do`*6Wx9)Cr7E?RLp~d_u4n7JNcP8hW1?$zEuHH%=m|D>O(|jr0Iaryu^C7?7{pix7M&Yff z_AAzWiS!A1XJ~%6OviPFQs;TuCOH$~H4i5Oo2?K}z8@S|2U+?TgIgon@}*^W+5Y3Itvt;08zi&cT+Yxk=! zJH?1b%=R|>KO#}PwzP9k^2^LRe=Fsc+t)da*xP=CLDrBWt-i{N?ev=2s6E&78u@N! zE~ax*q)GWybgZ@34ZF%IImul0W0%sqNFAc@9n2QgGI!p(+x%Em7|Hpec8n>R_oyZa z*?4B6WM?Q~s5a-%%Uc80x6H$Wp69aX#V?Me4@a>Et%L>_O;X3*ZEieG{+=XA?rW~Q zbf~uI^x%?B@a`xXSx+&wUcQjI`<`M`UwUb9*}ir612v(({Fe2KlYhog%6NF@QR~ye z;xQJ5&(5FBwOqX}wm05ub1J>#t+Dx`etG@OH%A1kJ7PDiNj_yRBWFC zJ=fGBG_*<7*`MJ_Z0b5{vlVlBe^z+7sps_2RzmRpoHVMr_fUel<4Z${`^;leXE$>b zYq!Ic(MJdLuG{>`_%yYk-`qUpqxU0wZ~q&Hr3Dvh^D|HEVA1Sc%ez#ThNjNJ5;n2r zeZ>0nKxm=(7SDO_>vKFm#*3sbqxL>tQvF?%d9W%Q8Z|jK^t++;%u1VH>(n=!KP{gQ z)&quHKmQr})3$f8K|sA^aGu-l5c|6s^{4gYR?_c4%CzLMqh@0w!+cLJ|NWje{Bq&+ zlR)0Uzdy*RSBqzcciv_G{Yg3ZYWdRe?#I@@zo?0?R&Sl#oBZ_mchm5zb^qhCPhxww z92*C7gF0?kCH~#+J*T(m)A=x1ow+lVsQ1$+e}DNhb9a2W?RVqw!Fn)rkAZs4+)-`+&YIfNbw2vtZ6RcZ=V z9uHOB4n+%xscD9(JA|DI2-8dp({2jW9S_sn4#NnC8)$}~NYlP>-6QfTB7+)kH6CuW z9gY=_xB!hPriR;5xGtqd*f&L79FMrV9f1=jUe_eLIuP9gi0)}b&nBYxIMH{Th!>9Z zgAy+_g}2a-p`l&-S zu_-DoEjptqI%_;SXFHlK9FwORQ}B-m$F6@DtK^;FLB z>5;&v3h7Uknx869JXPI!iWW&$(@Iu%OgSdge=5+)kNPmbefagOmxqX9`MOuVEb$lmd}Ti|o|^yi+< z&%Gy}`|do)i=_H#rQUN)eGr%$ke(XUoceGg_3=(BAv#rJ2iS9fFtJDoFUr&d5ZeT# zHbGE2AjF7041oCQfa6j!f|`Wb1P(I*aS;%lk$l!XK3*Y13JsQ`g7|`sb+~buoPi`|uzO}iyucQRjZHyvF&tTfc_ne9!1QiEl6bH2wKb$On978q_%9_tbLY882L15m^ ztofbXMW$A^7#6&`15yG&lUzkPB^jk9g%bUFIHeLO8HC&{&eJX}a4IbdDy3wUmU2_N zL~_TBDI6l8vWXl4GUYH0SY1jZwPdVvf!sW^g`M(MVzL>I*@+psQaq()E#+?~%ZGN$ z>7o^7qLeCbfMbHh3uN%p0VE@;C+(TQ1mHT64o9cWF9V#|LVgi z`qf`0+FC9CsakTc`bT;Jmkdf51hCPcUGRwi00jZ)G}XNt^h%BBPUHqx%_(MZt!8Ge z_OVg`3wYVW`jZ5{yt5ETKLl-2Y@t51BY^noR8!m9uUA$=5=tNega{!7 zq(qtl={0~98wN!|MF>U39#opN6d(~50yad1P(?*|1XKisfRuCIFM`s=MsD{0 z&bVja^NnxZbH=@YCv#*bYh}#&&h`A>M~Q%LBBEb;-&3aEbGUkMSKYn9TlaQ+tn!^! zN>Me{uLE|=-g91g{b|5CfPN7gdC%u_rKv-XeCNIB_y@69A1rP_Z~t5sdi4yf6A1_) z!#@CTd5Ajyc9ATAn7$MK@Xq#wSFq@+6Y^F5x0G|PKKvIUr6xuepcX4}!PO+Fy$D<> z0_|y1J`rjuy#;ts0|Lx|d7< zJg9~UG)2R#3E*lGjD&`hxGBW)8RWq`W{wbvVTyIYWOvz9+GSr$1 z-Asi_ZLF3AP?i6PL;_I8^-UY=kqXk!tDpNRK;!-rQvPbt6xIH&t91jQKX27;CV-Y2 zu(fEI0r%Ya&&t`IO-A+2COyq&znWD-YSqzYnq1Hj08r?%m7;vhTu_lvx95q#|Ed6a zxZ3KofRxv=p}u8PPfOvUT0j8Un&hrNSQhpJ=;{9R{AmUMisFaLS7X&!N)7H|gHJ+R zL|)s$`Zk4-+Gk6Zn>6oj+g*KmH+u7}Yhb4!>RDStPkYj@_7%?{ZzHmNKA)El5uUhp z7L;HvA;h2|4Zu>X;}wJ#0-sz;cv5l;vP$g~E@0nH(2x(DjRl}$VXEd+sgUwn&oh|0 za0PJJB);~3MXmxR9{}>d=H|Xn$VZ@MECC>M98?kuxwv~T@?K2TzesR=+IF^88x8+; z23!r)Jv;;rLBoFF8h*;xA!xt`0q9G4xq%KFdDbzRCz5+4GCs^@I<~D6gW4j9HX4RU z!${NRZd}kF08FV+q6jwNAeH<=wCDhW4o#;w8j3-8KKM9+ui*6BQnKgumpTB)JB#9i zW_$=v(5a4sVohNcA0d`pu-^f?hYK3;bNAhZQF$9R=}V12T2sVPMMI=ODC#30w=?S;GQa(okg=zWr`27w8MA`d6}TSa zbGq2$YUX$3Mbs1iXncd|&+fB_UKBWW*IfHp=ycE42)*|^*K^}N7(-b8`=c`b*880u zwU2t9IlX3^*5D{v&Nq76_S{p!oZb6`fAx0H^G~08CC^{r=y!X5-~X|*x@$8;p^iWJ z`_ts7-al}k&Yx?TIK=z1r*H9hyM(Kur(g z?OT2%M&tX9dc>E0ij|Z4vG~SUpDHgpT$;buPrwhn*qwC-cds*we1dXL7jd3LTW9bcJXM4vomdCjG@oeYu3qj+T zlE!oL$8#IT^Loegm&bYdi6ZBTlAwvQq>1wUiJJ`*6}=O8mM8f5&y~)ftAjq@Px@Sw z|GBo|bA9jUhUL!!{A9E9WJ}OwThgR3f3l-t@>%cXi{(iXe(II;)SIBGcS%!S`BU8u zQ$4*?y~|T#{FedeFGE3JMv}ga<$sxI_%hl1<;(JyY5dn&=dbfYUl)_UF6Dpy(eU+G z@7Lw!uhKmO=rXOIQEea(Sh-HeADxEMr&Tc58&}RKyUeHt&!{EOU<+n6B-w~y6}o1% z(?>w!_@|K4AbFNpFl*E}YtlDsCYdE^&sj)Z<}8Eftdr+#3+C(_=N$XyoF#K)?RjD& zuv!D~C81XvY656k>jcn8GEdQ7*y^(27rd||c_E-+VOQfqVBbQJWPz%^80xYZ9=y0O zc`>43@nGZP;l9Nk6378j3H^}GqBMg|Rov9C7<%_xLf^L}$v3+864PZVC3q<d41pWCEt14KZ;y_lm!1MOa4(_@Z)CV zkBYt@cO*af+CM8@epUznyr2BDrr_u8z7n*n(mX}!(nz?XtKy z$IKPke5@W;Q9@U{*r?c5@VmS5caNm+cdz8PSbKTEWqBxgc_evxtYCSfae1ccD1E%)K6%#4qRs48 zMe^sN+tB}0NNKvKA!X&ZkxhR-&1t!^ZPX{prJD;pukBhm z`0n!RHPXj7-HwH-(#X1pOT!SwdcpliMO|d z+w=GQjQ;ZGKyv&3-HVRNdpU#|<{#TtjjsPPE%DITSfPIhWy<%;v!?^=j)|^&u`ln53B9l{rBG6?jM|LwPWZ0 z(tA60A8U=;5!NwoNc57CJ9c@`)26b#b}nP`>ZV9_ibP6L;*2xZb+B>&Wfl z`@4?akg@rA^zo;~-I32tFZV>g+^+s9>h)fmpxDvG2SG7kkJtpqO})Jwl<@8M`@W>_ zGO=R1tXd5<_4o5OwJ7ZG74+n@=5SA|HEp(G=WIr5!ZKapcHvoG1`or}Z}GI-d(nU2 z!@ZY-)9m(L*B*U<`RXYlYIyjI%3jQgiG^mXz&h)fDPu^{{5*U3-ur!ZM;`R0IUIdBT2^W(!^;EuGWuZyFflcSd__oqK^L%cfv=I;J{UbpO_r!>#Qd550cjMkrc>4I=N+398Y zX#VD-4CHLf)XdKPE8p8y;a5mj?$5Jof3sivuD{Krn7(9$b8J7`uwm(juS-w!q^BrzuVK`27!UvU)}R9T$@UuKZk6yJ2I(+)FQ)#D(|!8xz0HnK?$~?+nhv zfBGFc%r&npFwC^?5I#1!KtSr}4QS5gupI|5=5T8fLn(y{)3vAsk(RfI1N1>lhy9 zsH@h=@md$Ee9^mQO(7*N<6y7Nug>&TbfJb17NIsxNwY_Cu^T9`t;)pAZx=hy?q3(C z@wUd(`aXs>;}F==v8ebVqF6CV_}Es_HPxb~SuG%-RVRNM9+%*raCBX`?uv)rp4P9j z&ZMjwxa5DISMQDorV&81j`^mbt#M(M!DsPy8@670b>aHDy+*5cUOv-581FID?z{-e zh@m}>>jMxDC#y4P7u?tV=I+HDGz3Mt#y?X$S}aMQk8lXZd(xTT;g>!(f8=O0Mo zy{!9m?$?-fUhRYJw{5laiELwSba3xcX}8wQYl_&Ut|H9&(W!Li)155`xBBHyUfGhg z__{y>iy+I`Q~nfEo@9=Gi#B%&*lMKNE%LP1)98#KR~IRzMtEA)cAnetrWijQ;qCFm zH#^&}#L(lwhV8XmFa1qO`R+}bP3r-lKZTTwC(8&IFIaC4Q2ki`7~9-&!90Pnx-_&y zT||BByLtFbsprCths_6f+^gMwC;IKJV3|V!ts9KP$al1#LbqHV_+FM+dE@rcX2aZH zod)l;-uzuiNe{hncw)e~uxmi|I%#rsLcfxFb{Ouu*i#a4b^)b+9C~~dMz_a%p=nE7 ztGbb1j%`gCGp84M6G(*fKo;s0?>?@dXlkH+{X^{1ar}3Vr)BeuQ+xIx?h@+!zHuyG zMeTjf--VRd`o6q7@O)2~8{Tz>?d0Q|Aw0@LZXptK%SE!$g6U0P$=i(ELUb)fYx|32++X-9t`Rfk)@Zyo!!p^Cd8_de*?K4cCZu#l=5u@}XrcD0$0T=A zCR`|ekO!{FV89l;n(+rX;Od4MxWS|=&RXC`>gqpNPA*0omp@iegq*Xu(2tEzU^&$= z*r6p1xgc(9;*x|aqc-iT;!A*9-xQrO;)E&s^F3Au^3&|k7GpL6y7+nljItDVb>af; ze+wxyu&IUKH=@!;btb3VZJE<>y7^|!twDNQ$Qwec zM)sg=JikLZhjv!D`8&L-9Mq=K(rCycGISnr*SrC>Zp6u;cAMm zu&vZ?ttpOt-k(BB^^aZcWxTw`q0IHIzFHD~s&#{up5MwqEDBsTyJ=~}dN;LAIaUtk zf)x8-tCZr1P)lJ`hWj%HA~462Js8QlgKL&!h}#@H%!iF?pb81XwiQAFLPGlWSU$>| zc`LqO#v`_0&MyF}-iBcZQXxPTImWu86CPbXNEE{3gdWlc@-{JMnLl2+m<;=1_r_`g zSOi8GDYNBS2&t*2N$ZhFuFtk}o>ors;T8#XitsSW=MK($Pq#pe>!41PtXhvIZmUp}x5EN9ZZ!zJ!2??vmZ-ICL3^@W(* zgm$wEZtUiDVd}Zwc)L=U#0met9|2XUJxL3sBLN*-rTL;jHDYmc&7eQhZ`;1aRum)r zo7ohCZU>ts#V)#Q&2_c^n!;pS9H`v%n_HS#&bGK>9NpDeODfrD!JdCw;dfksND&|} z2>qo{aTU1OZE?DtxBVk{z3gbB-%59`F7LL-Eo>=QNeSR;1a7(0QApo!tpq6(k~ z@qvVhA+r++h~s3$N$#1r2*e3A;zTT)Mr9{VBf_%s@CxiGUd5|Q#3>#kF-KtB(rogk z*;KWn7Hz_jJujBKQ_M$DJIo3xno$JzctVyWhH}0}z+PJ~ZDTJC2S z;*H#G(=2&t6+*rR5l2M;d>8);gNNd}_?GsBH|;jJ&IyXK={ohXwERdRbd9uIk8PUy zhuhE-`q-76C=-_$tjcjddk2vv*)z@bCN`SiMI4`oxnj>B!>SyiY5^j|DGn@>do7vB zUUE}m1!Q>x&T~K6r(!R-2du$1H6Ar_XN(zGer)Hdb`{WPjgU>!;%-SKm#~ecp zbi6c7eMLms?XksGp1z;Rv(^n*6G;OP6Y9K?%w1CB=vb;I8s<-f0k&|-PEw5xiM@Rr z7|91svF9yvpn{z0oG^qwAdP%wI&*6eqgfBHcAgU3ujPTs1g6xE>=|K2BEkUaQHyuR zr*f9zG-Bb)-ChbisyhLV@q=kxGuy4aTd zdZITpl*V+r$+`<=1p!cxa&T3#3@#$Y0*f#s+>P^oaD)iG^lQ`B*7pf--w#<-Bu#Bd zCbvb1(>`}RJ}gK}CMaD#>7G8`B>YhF=mOgXMottWN=^Ylm@q9aqAQjJkE1o#g2VraHQy*Ipp-lYz%|;coQ>A zyFMrSSxFQRRR`^UkovJ^fxsK@zSGKkTE6>_g=~eb)(;mxHa>s8daNQ`qVKyYhL!4e$1`2GC?1ZHJIO{7@tE{h^X42@w5p-2<7i9pO`cmRq4WPk^@ z(@}~t_iJWgKqfjcRvbVn3`eSa0+Ya9STm|f0kaE(WU{wO1=nUXvLKsvT@xR^w~G!* zRZw@XuE?ZI=*mf5*b$Oag0MWmt79dM0G1$pauj*EF+S5)^)_SaU*G0r^iV z3+5(!t=zH z(PJdt(v9rRuovX6+0y4}M!;b}3bv*z;b0p?OLrr@WC5;Rg_zGH!x4~(nunAFsYr~f z8^DhW@w=?7hg+wy!!5N+Gx}g@2j9f4Y6S^LGcSVdzU-iF!sD6`rpeeilWIO^H^I?p ztR+n(7WCArBW-Ybq=C~Y?OizJGXUK2ME45NRY>N^8F_PSme34vEX`0A9(z}poJrKy z*p&E5BpmC6?wCf4+@Ek{u*C4;b7)z>Kp#jEM~a~;1~#@XwCA|~cTE@)dDvS){1{Mc zuZs@;u|nXAP$jC?N9K+HV1%r)#V!#{0Jz~$w6+`0)JDqmdW!()QxuXMC>u2#yj9V@ z#P7cVDO~heKbw$79N_3(iWoReEGr*KH@QDHaE8=I96W3JE@JSU?MV4xhU32gQU*7M zvNu_e51prYt{%SNx8uO@#ejV`hA#z1j}Kp_{sl-iq{#r)gq9cN*^}kS({aS1EvF@P zp`yb~_0EyM0I338TbNS%LO6DdBeZ|+g#++LR@ZrH>4c)K?oHN98_?XILI(bMlrQgbeybsmHzr{Y*yeC zW!;YK%cT2x>raSF*A2T?4?U zg41sATCM@d1vffQ$N=l5Q;E~hdhQGCLc;Yoz7l3t6#@{lrEe+OrZgI$Z;08u709zL zGDU9Jzy2WA!D7~7hFG!=)@N~Q<&{DGb(c-dx5jq$cecvvyShE)SB0jY*KO-{q`WHe zlKGt;{ru=2257n%nzW1IJ?S%+KyvE`4Bn+69|xD3KvEN zP=y)<{nXN)b_k#d(3oT+M`B^BsYrYc0(y9h6x@(7T~`AIJwmiLKoO8lVdn1GwMf+z zvf(ptv`p7tGFrhj1xZJN;{Z9`1*9`iy|V)8aIlmVE)SRxbO0r5Zm1mQD4CI{Yvrwd z42=`_k+CEZ8EWfBUzH2gzc0ItdSrP0@^M8@@3Ez+2Ys%A`}82El)G#Uldy+LJ^8D~ z4MlOsJW2aylu6S~CCUK_z%vDB0zY0Fw=_(j3{qJa;?P$f^3&(Gh zb1%sV(|OT4^&|70-iEj5Uv09Pntx64GF^D%=YM43ZNUE93-1DBrWW2){{@YEFtzxR z)^7T(=j4{>FJk+VdP=|?RuY~`PW}T5IK&)!@Xa;e3l#L$;gUY}0l%)X4Z$gfM?b)0 zQhIyM@3y$F1E!T?HUQ*`fh=TJnkj5bmut|%FIJ3Nc;hZoVbxiP#()|h#k;vUU5fQj zg@&5Jw~^Zl%XaSWSBiReN89KlTk)H3 zonjiz!4}7Hk(41GU-DR`0g#KH-qET0vVD}TB8?9;M3RB70W2VfFCvvQPvo%(dB#X< z1q+uEtMD_sRr9QQ-dyubgIcTZ%8bBfGnF)2PNO0NXW-eH%<7le^d$ry)9lIVa?|{( z>8)BpLAO8eB>3~Sfe|Xp37I0dAtA^WyBhtdoTrp7g+kFF_(+wuYIq4gTMq&HbzrjK zL%169IQiX5*riToq&@QZIenxXp(~b(meGYXI4puYGL4)^SbzFtrq7$6{U4KKLHZZL7j#(d)*uX(Jk1_Qp#0up*K zNI>o7GYu35VAD6yKoGS=6*iEL!MmvysCe5Vz%-{k5ZExrQc;){FeV9To33?Z)Y58u=D5C#fXoHO>GK}NJXEuSxxiIlMe zL~@@#G|Qh`^3r<~qQLoeF42GZ6Z#r4(=`3)$v*6WOV>kqcNnFpiZHG35nj$ea&{yr zMBm6lvF+!LI%4LzgqbjHTft2I@s=?Bur1r=olWLCP`cr>GVKGX8I2Naz2xtQLiS>2 zAN<<9BhhfhHB>6`U1IsM_@TK2$7X!%dOX)MGM)wxCA`W^T7qgfo?|<@l{>qFcW`+} zyzThifeHLAiH|!Ae!MGuFMIHW)@UqZ6t=%E&GKEM?2-!N-&1qm`X`_ZMTS`JH2RL30pV{e z;H-2N=KW(8RwWjS$-oK{nSvBIu~^boIPb6gjLI56E`j6*Qy`cF=+lMZ zpZS@x1cV}4575x?l#Z5e{uFGTtG`b-K`xS6a#}ed1eAs%tZa$UDeZSpDfX-e#GY)1 zD^NlJd9rkpE+NQj^}R`oye9lUz#HJ90L$jwYsL$uLe|FD1_eu31_-O= zq~7eBt~#;{$kYVHq-?YH7IiD{qhhiJ{e`@9SVjcxWNVStf!utlha9@(`tu?bf#qjV zy1fp3mB=6I(X?7+47aP}rZQ9!;eMBuDzF_UQq;AXR37U@{U6`-sx+WL3P zV86YjW!1|omqBjk>E9I}hOhI$(`EJ%NvwT#qvB#CXU$4E-y-ma^Prz^3}3ZJTwVY>EtU3lNvD`f@=C)hyfr4;aM08*j7X5AXajy=-K6+D3pE$?Mc@cmhf)+w{Wbx-eb2+; z!e$LytN$ zk|&pj>uOY%=s>q0bK2j%S}7MOGT!1iT+VvwcvY_S;4l=v`@zks7o}#-Wpnj<|MJ_E<8R zKaP+_UPhAuyx-w$QUD~8+00`#WVY!h`*5RpywEr$L*D+>a7fb(Bzr1yUo{zynO zlc5Fg21I*#|5cXQQr%&)%KCmZKw7JEtaD3pyemrtT7X7iIfiZ`+L53( zc}bXE{Gy70a+OxNzFEYWWicM#JQyEZJ=wiAHL$nALT zdLvW=O>-w;pZ4t5yBNjX-Cl?cg3RHYy6i4^u33LJCDWRzwK~@{Ld2XfBd!o I-(PU}Uop(vmH+?% diff --git a/docs/static/index/1.png b/docs/static/index/1.png deleted file mode 100644 index c31f71f2555175c1c487ed2f0da653a3615967c3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6011 zcmW+)c_7p8A9ro;nIlFUoBJqCmyu((93@2Vh>glESMFPzh*7dGw;Z`c8X*lU%8|30 z$emkKO!00pR@tEGY#dU9mt&$_;mZmJ>O2L|0odVty4|V*;{|ND{8N{!SJl(yUb@9gb z`{{@6rk}({#=Sc8ysmmagT&-o_yy=Fh-N-~ANm~v^}WpX9fdsZX2uHHc!#Pm*KFji!3}XmZ-YlFY=pCphW%y!F>I35>l;Vd z(Ng@-;=*2`i<}9(yZ(i-3%}&zVS?;*hb42Akfcurf5X4kp9i?oV+ne5^OGV-<3xy; zmgCBePY9Q7TkdOYXz>tJ+)NU@uKM}I zAiAj)1#Fag1j31W4b}#wWBa*4Uh-pJ10;`57MdJZfD|lNSlA@Q?e(6}o<CuJ1XKV%_660`I~Gvs@a z*Ww{wd>Kc=OmPkZL13tyAyz6LHfNy_^6UsC`R-R&Q8@`4=3G_~cf(mdw z7k8TA4V^Th|MI5y(I&bg@NAqH;#pAKDHa-89}kgK%4kk{;t$R4eu4eY1(FmMZp?hn zL8QR2*q)RFVBTaH*NXmkc910J&CwtRFqiL~B%N%p9(eGo?(tD~dBISpH!ykdAjH&K?<68Ii+6}AU(To&B^muu_yw3)*`eG`?Vyazx^wz zH)EKF*!Nvusc-pD^OPuQg0ldJay;vWySfR;3Z?7G8ZX3-y3(OhjY-vuB8H@NslVPB zkA(m7SYsYNJ7sy5g-tH!0Xc4vq{^71sRS$uFTkfID*_J=2dsn>aZh2-Fo%Zv5CkjC1nO>kf>1_L-{1^tG6K}=D|Zpax1ql>EN`ursyxQT?vT- zNtm_S%FeN$6t`7js0$$@>?B$5bD`VY!UX&4Y4Lx+2rcE)W?y1Q>!$U`p#?}zBmsg1 z86G5}O%1Im*iNdP@Bh)35|x@v%(E-Xao2TfVX&cN@Py@%amCl z#vX_l1TD)2+Q}q7b6a_3If&95Jhd=wbS&53sI*aeYSH0wvr$;Ij6k}9+|;;0IyRaD zDmimkG_$Gj9@M1;XQ$y3zu%>RNLlVdo-?E+LMD?I&e=6F5K+AK+&?*IS&Bg{$PZcb z^YLv@V&->r<3qSWlOP=%2F_R@NYN05N@-zNG#|uMwWpWa)g4>2nlc&;lnjM0@eY;t zJg7f-mY~U`W!{VTWkWP@s&Zp2syTf6_)9-a{NBFAQ-AbMN^13@3*dXa+yeOsieBnt z6%$T6w2h<}O`uI4hsu!HBDg{rr=`|rw_L{SFOuI|A zn|^uS(&)TdHS%o_Tp1s?gX+KBi(IbWK z=T!&aAT-0qh8dZ6o+Kdh|Mg3BR^+hrS{nA6uf;1p#UUQIt^9(D5gJ=E-l-W!^PFf3 zm8l43xT3iY4(lQh?J5qY!=zJ`C1f(pQaR(Y{`O7g8{IL@M`j{upJP{Hzb5y{agn_D zUTfE0<1sJw>b#dAHsmW~(Dv_^I(`0VUC@fwh<)xJH(RQN({rQ0V{JFy%W+~D>FG^V zdQN3ld#!PYp2F5f-tdM{N>WC-Mh)>=*x8vp_v}hq=8iz9LqAr9@6xTWGnN;Yf0paD zNYIUy--oR{yh=wPUTacuAME%=6?GYO+94{EFY?vT8FB0e2 ziq~*t{;m9IFa5YYV2bOx!6EPCAGY)usZ$Yh)hFS?485>m->t&^PXqC`P753^w`nKL zuBD~k#b&QjnnURy;IB5piibD;s=Hoz^V{*k;*?46NP@V@T!0H&M^}c}p%=VfY>e_z z_^cqhLa9b`9~N#Oz15o|8~m>6E(gB%GM?My3W+zXRgRQ)|6BY<@CU8jgV>wWBBnuz z?EDWyU`bf=3};G4A0nimTECVZ%DPs;6qbo#e5?(LR7{9m5Rku**OAw=(-xpReKTCw zDe3#M7wq=Ud36snh*r2%efsPAzn}rqP$8;L>37ZH>uyvxbG0mzd{F0*oSdai^02Yp zEeoZM6TuG$+YEctVe0gw{FK*0LpJH~sOX|`CRcfmX|DNw|6_B|U1c zJla&lvx^c|X3o`WyzvjX!@r_Prt?_05P{25y=M$Y2h69c__q!XxEp93*sb}Dt6fA2 zel$kD7p1bwZj`|l!DsM5{_V`|)74#7!0v1OR0{?DW7g+LMz+<-h#1LosemyVp?pS~ z(FGk1zUt=f3IRFHVdZ|)#+{1>;kwQ9l%q4X>|Y*g#pFL9f=s4kEOw$KF*hIV+t_6NL%Q?|3|**HxTH%^ zW9P;^Mfu*Q#K+WJ({POF$4&;F;s~s^h~`qWGu znM0Ajne;>*N19`30L^4zsk8rd>!+|S%AaTcs2L@T_2RgqZAUf3IyD_&Bz)fped8Q3 z>jvLs-06XZKWD7s^J$sdGhK5MdTS2*RfTfR1WWTuZ7#PV*M(#lSSo%uw^%d@e*PJI zS3p+r>%PM-s5=L#4f0YL*fT?o$6FT z7^Gb>JBi%Klz@TI#X1kzkX(T-+MW>wigJ*}HS-jZ1aWa7D2>7*xv zex1d+xbx-nlymD9oDslf-^skmg2B3m7WD1?28}5M3Q@FJ9eo?kFg}@vX}hr+JjzfA z4Ptoh{#A*`x(RG6fB0MJCVm%&&t=aPPfRa4EhvV_NFfT9ID z&kkp3`;Wv?K1tsFp+BPQQ5^GiiSN%fCU?mLXF%+v zJiAllWEWt)Z#AkOYTgRDG#M$a^ayaRkt5UO03u0bwWx|*{hjq+yegngs}jZJJnsS& z8G!b1mYyd#>?=b`K2ce;F6U2*MymiAaAsOAh)O2?N1T%d!wDH!f@ldD^C$0(SQZ^t zqL1cP0Z`LBa-C^%EOF#D;J342$%((dga+@~=Y}^ohFNDE7H?L)sB#w*DfjBvl?>b+Eac)Z;Lz-8%$r4)Nenv; z$Vrr_0BCELuFK5>J^*TpZ!x$4Ns~ApL-`=LzdntA4$Rsbusl+d3RuGaIbpvLG}1BA zDBv4!(v#EGKsyc4vdx*#fzyRl4Bx~nn%?`;YdhqtH}1WBC+UeHv~IaVnfq)O&+rbg zu#?UMO^!4vrhd&GvhzUkO$HYru8ba!{R{!%lWm}7wu2tt2kgkOIiZfi=*dSEpnxBX zoa(@NO9x+;{+f_udC2e0FM+iOE9Tic|1OL;ZnF0A*YJ;dq3!`|XIXDOr9p1lg}~{` zjNXg`lD(@dTbZ5fAvLPBGGXhw;hELew7UJYVCaLPGS~RJiKsl)p_z-1ssoi-3Xzhk zx%wQm{lLG)c45%l%?7+lUqaHanPxeDO`GH1z&aT5TQ_D3Iec1Wg^8a`&In7xyl z%fL&%dW1`gWBaB|>F4j)3zLQIC}$S}rN_NEBd(*Giz_r$9OgXBCpIW?sz$JVVGn7o zu_usqMv8+vGx@yNrM3+TX7X~K#PpAWIF3fUgo5NF9)Fjw?`VgWG+u6v#cU|h!%m7V zXpKBI({dI#CBb7`GfBo&ZJ^Q8;1nK|`nUKT=iI_}TE78wb+1WvB;W%sW=7;_%^dPX zQP_J1_t}F(k8jS&1p+T5JgvePOkZZipI3H%ce%D7yH}hqI2v%pojZNhZQzz~dG#@D z4x$LE#y|PyLjl!&Av@<@Ttn-hvAqbM^U5TThkaZVW42a!2a_Em?%uie7HbdnR}0@1 zv=b27e>p_f%Xv`rfjH2qF#Z>^OFcBAKhu%$*4VbUyN`178RMJ8A_|!RnuM)ozqB>C z^W?2ZjCNHWk9AeO`p2NB|FmgSsbrKM2S`F!nshF zCy}7{7se;7cbjG1YhLC_aGq76;(8Fl^y75@Ky=UX{sOcql$0J!5q$}5Eo7h)M-5Q(QMch|4kA*uEuc|djgeW=_ z?%7scuysjU>TI=gsPzr1IrD3lxx5u3bC<1hB($?c+?vQm| zhf0^$`$AGlzJ}IbIq!@@Qs1B>R`zsPck9PF^$-m=B{AW-sag#e*&wVp)myw1xVvZg zN%-5Ee+9zZ;9A?%VkdRmnt!SR4bWDU3H2WVYLJ-jOS$KPY8GwMv>(V{kxbeVk!R8b zSGRb{9M^v(I2ir3#SW)BQ12h-6v86zyau1u1F3jpZXBXXHqL6*$6W@>u;ewof;nxw zV5d9k@N+2g(*?>UI@}9@Ka2OC&%3Z^k!!(gv|L+(Ab3zw{@==`CRSbKB3~8rsZ7Ij zHHz^Q?%xwzqe_Igjie90GI$FS?FTqH_ z;uHGkHfq zVwyZhM*Y9U5b#$K%s7Z`_jzqff_doe-spiV>2|~Yqu+EhOghNX;pSau>RY?KY9+jC zS%Jh%|0ehq@^|<^pG3iiUpQH#+y(K&QR90L@_4AsRY)lPoSn?;)7&~mk$<*iJ8s(h z1fYRUnd$2?i{|e5ag%?E{9{J3IdbvL_vB?r%9XZX2Yz9X7vqF4*hRJhY({UPygKLn zeCbMrSW#@_c-lyfOX7C#Mw)_;Pk(~ZNUq?;2(DPZjGv2V!Cru@>pc6*^ujC6+ixBX zJ88%!@6ELvt3{nyC}*~*hJg#Aze+L zeq>UZ&7AM;|8YM#a%Gfie9evL_O_)Mcic`~?E!>XHV9d$N3ei@Ebui(Zi)W` DNfh=t diff --git a/docs/static/index/2.png b/docs/static/index/2.png deleted file mode 100644 index 8b424692b040935411e53d760ba631971306e0cc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5174 zcmXX~c{o&W8=jG!%vj1c_N~TJmdKEG>=80{6Ur`2_#yi`$P$H&i6W&L3|SgG*^(_X zNM)_DC1eea@96seIOkmFx!&_E_w&5>dtWEj+|1xCGaoYu1UhSEh_nQro50o2NDtfv zgxmLl2UCEdT@VO#UgGou8(E5}JX+ zRiUAKeS<#-f9{_VcQv$Eg2jaV8pZ|rAwhFV-%Lyv@^34-_vvp?XKScJ)X(<-GgpJq zxL5@8RUg;2kBf0`><~G~#=+i(XV{_u_V1$+;>bx?8Rn1{R9TR?yBu)+RVqfwb*Cd` zD=R#E8crX{p9_{)!ikVo=J9R5K9gov4DDPvEylp3b)Nl8!_nj znIHFA5GQr5wZCY@P;r#3;tb7ewYn$b8d|Ynr2#x%w>PHjY6Sxd)vzbryYRVL|f4PvcOm<>zh+TyT-X4&Y_ z?jW&mh&Kyo`B(YaLj4du!5xr6JN%x2M}`vcmwvOA-@$|hc!bBI#(n-{!xJKHFT9;HOy$I)SI z&C5MWI0W1;>bumyf<$dvdBbmU3q7CO2ML^dv$4#q48eeB{2(~DV!?$h6%A`g99FFs zn#ZYKAi5MU`}v4mAMlQn8Ef89bifE5O=rU~s5d19F}%Co28W32#tLBY-Uj!yB#MFvaCsxE75}og1|PZzwA7f;lydxU9^9{7*-azVjl55X z#ZeF01T=5HRxLIB+(k$zzTy0|9KSMu59rKRNd9Tmm%BBFLzHyUq^J=)sB>2i9;>9C zE{2dTLOJosZQz?2tWuMQc=a064vT7qJFqY-@;Hy9C93(O{c;n$GAo=<*@G+1hVodw z#aOuG3YRrXd|VY}FZ5u%Kpg*Uhn&p{mW#!&zJ1AH^#<_>d9KpBvWF-uIgcq_J>WPk57B?Xg- zd*QWqyXYW%zlL~sH#4$#wcX#8jmR+}eHjn8Dm#ibmb~IIC3imesHm3b0Sg|kNhHZ<0}A22z-0;j%<=EzELu+VL9c@%DSTpUSqQqjIm-m`unB0ERI-- zmyq&14TXwnn>wV?TgSu9USn!4m#3y4EVIT~v}l8pQz!bqXgWPtBUacl5j}?kBwAjq ztQt;mo+LpNycRzHS6D1FdpO^2&s24_qG{4%#B4g9k zRXejr4qY;z75)CI2B}XmYmq3|QMdJ9_-Oyzbr&lAXgI#Wk zJlN`REeW4i)zYcl&pn-XV-@+k{f+)?Rbzrr#bTL}0e1z>B^CfmD4Mx^e#wYsU3{GHeGWdvt7O&MyV~xMcN;YUr%zFpvvp>Sv zq@sH^vZUot%69mn3B|r|HO{jS@$TcIrwN-maX>TcU(tr%vbCqgY|V1emSs%ziOQLB zphWN+*O5{sNuB3;$2|x0+i}s^Cu}!3Xhi_BK}{yjG0RqjlY4jOQ6-~h713GeC7X$v z>|p~*XW#{4GuMKTeUHaJ-<-Cllq;d@*!_#|1kNhy{N$aCmp%xy&FwSZcyQGOhKTz(#&J%q&MS`| zhOJ>v+#|PdqmG7nJrWo%*)<_@yjZGZq0Jt_K~FU(p^a5^Hqepgt?9!5LX~{VB7iND zeYa-eM{w~dpNoi=kE8mt9Mw7sir>M;;HKsy)=ulJ|43tpw%wByR=VCBM6*Fz)~Xym z{r!5e?!BKQ6e>>0ywyl;85w!{PyX8!H|%Iu!Ia^`}lSL}8g|xwjTu>uG zpXx?jo4_%$1~8~Sp@XS{@_FeCI9KMeQAwKe5xZuKLW_by*)E8si#rebcEW2_Sjb*z zJB&m@6*fSK92wTD%kNIa)h6pJ1>5R?8e3060TPyiSAWbUtII*i`9C%zHmApofi_RD z)r@u5cBj}ZJoHG_3vhzri8{dW`;@R@-qr)-H*7VBOd*7KgUQ2RE`2&Eh4g!XZ*b&N z-jTBh3z%_?UIff>@}vevWfPzo90`%5%I(~_4{dJRt;rD<-%-`;|whZkmgh(jtZ6Bmob5650VQ`Yl-(@V@Wdw3gH)xpGVvkilp z|NW^M&*Yy1biC%!j|CNG5yvT4LD@P5cTzGHAF#e^NYSG+t82vel>weil{~&i9Qs3N z)MWv#+QQRF`3`en{(FN)ffIq8Fg6iZbKGph}8)_nUiqk@-v93`_4V z&_HDcjzeN><=y735NU@stss4`inj)4f@q1B%8t}|7Lzm70nHqG4FLKiqY^+^wHSvv z#Bm<``MqB8E_w)XopVk(#}aTPh6y>l_yulWe+oS!?bgTz+LGf0hU^cU75&?&Qu@zFA&QC_lj*1_MzF!I^52s|TJ(L2Z; zI4I1|*+L27$>|egEa0X9EeTE^^pcQj>e9FSIJ<&Uj&8;3>lKu2kJKLFCy+lb-W1q_; zhLe-F6dy0W7we=anS8`OA32tDlFSnBC|Y|Tv1#8)%*=BVuz#NbieP<&v{!9ZFrWkP zK_ydn7BM#bf7|&hay)1tm(@6?wbj@N>6;}RswQ*b%y>jxFV;qJVYT@{;LNfEoN8PCL?h8xVgGPYzT@wnsS)*?31+SO zZW5@hvsE!cx+G+jXSChGYbf9SD>{m z-F5ikE(LX!j&DWeB*ErN?~JCTF3tX)&4LzkzXRyT%Gq>gwqK(K9uF<#bUoFewvRF# zcJ;H^UB#wI+!Ny#g0!{?2(nEw)aNbY?d4^pkTgw2C(&Q!)^Qxvhx7PU>-iZv3ZPU^ zeW*+(0n(S|>&HXcR-d=fg=F<;!bb#jj{bv^-*_B!@_g`V2vVG>>7}Q&DbSPE_F0h& zMCsE>AjM@kxWAo1|LxC$F8q#nEGUhSqhh}o+nQ&ZUObth*Gp zx&-eLPqq`ZHA`v}p`rMHapNiprLZ;8mL;@M?@i@}0^*v;FZ+M>(R|9_r<;gaEJ)at zLCe-N$glSM^ErQOl>-wRLa+>qXWT-o=tPDzB>dZ3 zbWizJK0xRH_7b|1@bjiHG>ple`=l5mPh2eT_>a%LVCY>}AH$E?~-LSU0Y(o@Zd zG`Z-%x!+h`7*&h>ngV8mmK0mod-kPTODahB>k5!hC;bymDS@(NnZe^Up=wXjmehwj z>XP=IV(tOy{X-8YJ=<9~=@&Upcb*zav9SH|)-Wi#~>jbf56>Eb|yw z_jf{T*>x9_nQ7FY69dJX4l-<#M{+QZEL+ENl~697SSh7CQ$BuRVeV-{xQAyV>wcLbT4}q+qaOTpNMP;(;Om)nSZpuiq=D2cdq46 z{sE~ta1SW=-F6ST(D|})JK@K99RL_+gwWjKh7w8BVrR#v6vg~x{WKERjKS7XfE-CI z-JK!Dlneo=RV4FB2fqe4x|buWh(Tbv zz!0u(z}H#bbV4b;PP9id!&tGj{5JpisZl3ASM_W#o<@42hhFuu^UaJ?H@Z)m1E}v1 z4gIz|R^_@(dAV#r=L^?v8hpPcunv*HMMS;jj2e2m1(%6#yc`*AB);=SZ+2i{!!2GP zaWgK6PX%%1v2y{Qqn4_C&ETV<6b(*s`^!q3`==aZ9Ji~Zz{+SU$fM$cUYKl(F02P^ z=AYmF0cIKx2{>H6AZV-=R$1T#yG$K9wxbqrNECN{fJMbas=ty%-tj{ugj=kKwxHX$ z&P$5u&(`X9GFfrSw&KIFjrzok&}a!9T16TJ0&TUkwQ>Q0M5KU6 z8Y}@&L^cZE06$U}Y(2t2AVn?VBVy-*o)hxITx^J-C+}5%0|W6hgu?_7s4_!l!)FT! zq^xXbML2RHac+Q*x(-DSpUFk zLo(ch;*P6=KfMW0J0>OCex!DW7+^YMPf;|G-WS_bWH*YwqzqQ@wYiDu%*2HHJHD@P zJ9N=Ha^mj#zS&tx$j8qF^wg-?FfTuh4j=;EX&GXjub$oe3kB?S9)x+Of@DiL_Xhe9Q3&niT?I$@Ifv7_7)n$}!sljyO9# z745Qr{YP4}<{-2&G(KI5yVVbrsbi9<=XI~ZhD{3Fx zgJY@ zcN|m8<^R01Ib$SO_WgvBVPYXC+#NPIxPqeJvcK*>tv8EMZ0Gu}oY^=o#E{osr*xiM zrvwJiptvHlh>2=Z;#{y+cbzL`;q>QoqNQ5DE@=BY@2>+pKzMC^zI`vU+chy*gN~s< z^?TM;SMd1B*L~JU1&mj}02t>gDcr)ue#zaL{GXS@LxK-a99vg|?K>W#HXqqKWPZ^& zYFoxbB?)$Z>K{baO26IoatY^z#;$ZBlSnHZWo*FUnQD)E5SI=T0d%<5zE#Cmd6^Ulmvm z5>MGp~`=a=<2S*{;$`YAo z;=l-^)T5U}WyCX-xg{J~b?egLmUT%eBzG-w=j8_Gqf?ZGD>mBJr3S5&;o%{dSh8*Q zzBL*tU7k7*f2se)vA-Q$+b}jcc-R~N4R@!p$w3isn=0?l)D?|`dd@gUroXc|Jg9W) z!TVdMiif6Mt+HIjRh0h<(Y%&M(U; z>0L|_W7@$e%<80CRC-bLDRRuB1nKr|@xAQ(R8Cwr4Nld;o)+^3ojQ4cfcZD&)a!`* z(r*FeV5z7f1y-@BFZlM|s2Vl1wF8yconW?AY(R^FA;MWBXKN&@x#SE$ZG_0^3&#x= zSP#T}@eOKFpZQ{?btjtQF}HU*mZ`2I^f$QzkRt@+=i;R$Hcj6doKdJo>9mOF!A+l&CMr$mCy$P5UbJPpxJV zbzfLi>-hFuV0xLEZdQJ5<}bmn0Z&}O_B_rKRwIF)_2>Xv;%<7*n6rSf1?dbkzp+X; z+u_8hMD0$BRjz7Oi(1zAuhq<$Ko~L~LSUzpeAB3*{yrOsNk`@*whg?jUY; z_5R-(IIpe$LakC8I}=l&^Ta>!#{P15PbNKG;S zf`-OYUp?kbp4VlqXTaJLiC=U2!5(Qzj@bqPL>1$PcEOW$8|tu^4&IhCrS@M!z(*r2 z^<%F7!fgmdWuEV|FvBI*^l|^=6BLO=mx>3aBUz|0L^+3lMVw}`W4d($C6{zMP6R9^ zMe^4~+bOWX;De~5WqM+G%#=B<2|uSyqluIJn*CB70ENIw#A{32vHqw%4_=l4GcKf0 zBKQvuX7GIu(5%vjM2c-P5c&YH<3YQuClP2QI+Xn|FzT`K+KU$tIac_Rx&d6%;B&mj z7Nl`{Ux330hQzya#KxesDA8<5PV2|?Yvn=B51EGD&slX3yf##VuPzLWOE3LG+?0|A zQby`y`J1G}%?n$T^A9+X^_S!nkZ5af-^=!k!Cl>B7P+g7FYce6&>&ozL`+>{C>K7F zdRsKYlRP)EA{F=MBd$k0&q^`A8j4lSOI~*Y>ZD;G*$6ckKVqB;#(lkmx-uqUx9`g!8tP< z$xIFIP+6m^c!>td+)C-nU6b4nPKK2=v+O%6LtX2f9*rDlJ}MAe!EmPj{5gWqulRlV z=g~*^IFtd;`?)Xq1c|sJ(E9ANBjIJ=U-g5xUxKq9-{PqV?MT~wO{6{c8-McNn!~Q= z^z`94c(UzP?c2Ulb4sNLk{v8J;cq+hq^DxhJ<7#>pHrbeht zY@8J}xLEeR7swDroFG*T`^Wowgs7)4n4sUN>i>%}A%0z&m$y%LkQ73Y4&5jpiBHeo z@t+jiV9F4yHjAK$BjApSR>TGOdVQtADe^JqZ##j8=48PI3p6DoI}reTFM#>m1EBR4 zT%c+ZU>$G7Zl^KaYB@_aR0fnnz6^!EeC9U}Y-1Qj--ROa5yt@=Vw$Ar?MZ&)?xa+0 zPl^Np3ap1|lBSpWjXTIXZ8-phpn@aI1`IfmSDt1TbO5gQ`*`$}pmzV@6RaVGpC@g> zjU8`{cQ|t{FzI|k@;}1Q)3)gSL6wm|q!!QXN2X%1@zFS*&63W*4`|mCi^VvS~7EV+%Ry1zjkG*|4hgFU=D8Mq}_!NUf-U7Lz;Z3Q+Ic^ zh2*tgo2a8-QvcMJLA543)LaZq*B)IQ+g@Qw#FII!iL10pw2NJ**gv;ccMdz+nSe0>>SLX;6w{EuyuZ5K<_Xm(R1yOMfoA~q zURU6-!vxy8zv+tE_WNtk3^V;8u;>*C{1RY`XowxXZCY~Y ze*T4kpdCo#_VI#}M&Nx2V0y4Klq2po_a3lL$JZq)i;f|&=7~ZbroM{AQ#fzc{0L)g^*GWO5(fts(2>`pv5c{^vnq&65PVK8# zbRq-OLLv9{;Kt^ka#96zy1G48mjKYt40-{(lFAwNV?H{oQ2C1*i&K2c&d1x{DA31P z3D^Ro=jVb2jE;w>-3?(6*Mh7p^fR(G(e^HC@NJ?^VD}A>NC;G6WEd zIX!!JFh9&yhSF%LdvT~6fNL3&e@r8{?SxxW<&0@3h(;r}5unF`$xoqNRVN-tkWTW7 zo(1~?Q7d#;fg^#u7G!`mm_$`6n*pmqo^G`d{QKy|+WS@Vdg6gvMOLy?%+W#~Fl%CA zIpS&hUgh$I^CnW(RZlqZYN!eP=Dl+Vg?g00)4v9*1=4fz)poYWRQq|RguM7uD6dWYo`P?m*A1=YE?$M;X{8Q+q#@>Q zoys>KEob>9x#e9aDSFQ)zea)SDh1S%Ta_r+K~kNgrR@nvVDuH>|?Hfgy{FW}? z=z~+;C|&qKM9y0jP`(8f?``Dy1ZJN%E~ARojH=5d^|TMy8qMPbpYP+Vk|!@oRRQH# z5}z>Kc#E(ftZfPR!iIIYkWefiwN`aJx+TNO_kQZ?PBNT z*qY$U>-^yy`Sl~@$1eE$j}=)!U3;DJBlAy7(TZqr@COr%W=HuKt6F=%3gq&w0(xMO zN1sUkbtx7^mydq^oia|^C}XcdRlZsGWz!YO^|puY!Yj!-yi&{wa>OMN4=&=wus`MI zPo9@AJ~T0=9R%!ssR%7*)BbYkqElIk+23CJ!P3h!F{6?BwfzFrMVAH^IlK?!>m;q(w%KdaJl45&tc;v{~CM7 zTUNFe7>Ijvou-$#l{D$W0UaW$onI)uMQQx90z?{@UBr{R-74-5p zb-f0)d8|eRa0U`8T;cDalqwT6aF;MLW4*ye%+4Ixkgfd9y7=PNY;3`Q%uZbIL=<^0 zJAkHSe3~8F2yB`1!5NOu3-w6sm-Sd%Ix36$)dv+O{)Z^ayYA(oG~Vv|xmAc};_T6Q ztI0*V2g^opJWk!NAK0p=;ex?ddvx0sP}#4A4X49MOXjNbsJ_&aoFc^UO_?bouURz$ z57EX}D{mDNlO|yTn~Mkfvn)kP6B>TVYy4I=|Ja-2$*%GEu~HpbpfpJ0Is%Tjv~yAZ zGTp2%rQ(_Zq-Jwm$BSj{Ee?O|g-jHeP}RJzIw6*UOm^@k2t3Zo0M--eGo*?-&M`TI zAfquY>QObIvN%BrkOEi_<|}Bv%X7S^GUD;^UKG6G zeElj^+F37>Yv8+F;B_{V^0g`xFYs2jE`554Ul!f0C5<0vcmpj>81Fg~=(g$IKvlcg zqIs4XC#?kDcxsJ`j_F~U0?tkaSm(OwusLwpo)+cpIv$(`N$z@ytQ^4Y+HY5&WL;W5 zwr=HL$N+4^fX>!L7*kgbHi?jh-TQLpScV_BHRQP>t3C|YxzR>xikpcS>9n|IV0C8x z?C0&kwkW7r{#ixOhVG50I{q6z_4wwc^9qu4%(o<67B*l>MWbbOwkW`%Y-A z1qK|=faq;^ccIKqU%ySJgA3&~_xt8w@`AL@p=6=M;-F1)iYf2QRj7f0O7 zKu1LiB$=~Q=q|la7yIZ$DmZcvS z7A^OG16$jO>lj1C_A}B{#i|}>-o-rNIRW*7SXkda5Z^-av9Q=TwAFwnudxsF7d|i# zh7Cr{_#FskZ~aM=D|>rp7{@&rtH+|j%tVkivegYFuenX(#ihL?dsoNJtHs(jY)tZl z<9tgxeb|Au3D2Hg1uM7eHJP|1>goW&-o+5C4aUmDu$T^nVXuQ zPsh=ZADl6Vjy86hb`}xYy`R7D{wxydaA5J`W7N5mlN0jA-EWKAJ4Jfjofo44yDuU< zJ}rm3x@=mL0cu42Th{4`XX&gC9+nKG4Dzi!m**gb5$C+hC3L@E!3U;or_T`0&uY=a zm&t32nmNKdxA&jN!En9#DzB)b*{>pcwGZvY)I;YNMH6{ZW=yxCY%*j|j^yq}ngP=p zR*5;`m&>Or?WS^)Gl*w{S7BpR5a)mDx0jgD)+Gx@D6TiuO z!{(}SsFrl<5Z>sumx#lNF-uu{>v66smG#fF>BWGV&v0Zm9! zA~mB|U8gu(lz#a6{$sKe#@hte&c{2GmTlVej;S55|BT5!8Fb@zyjOD>F{r)uif8ou9`|*M?LVEtBV3_r^uc`uzo`fyXg4h~#Xm zccW&!r`nteI;S!HhF<;SbE^5zoPNN^fHAf$GDWA8$_d{&Qg)SsUiQT!+8n?2T3EQ~ zdnILh&!f+y|7ve0pFRGY7Nz?bwd7*4(dB~kD8RSQJ%n8oOXR8ZRE|v3+mzI~ zV*}*@^!(FfAl;|h7r2Q}?Zn>Pw+*f|GbzrtJ^%?gR-r?at(xPDjrSXx>opPdm$Yu9 z?Gxj+OIEmXVrOrmT`*`8+xLCXv)B7u4ya(?@T>7xOhX$VCwg1G-VOpd4V$oqBMNR0 zhZ!wm8$e^fC)R$QqUnJ}x`Tvr@#EPYS^flXoh#Xd^(>URDv8~j^hsgRqKKRD`WxLM z1FpaK{l+%Eo{Fy(n36XYZq#W@zmW*4uNasIoA;ceMR%s+Nv5ozkDB+3oknXv^s3XT zRc0Cf`8Vb*Dnvxu>hRWqYojQczxoT@@=8{3KC%ZXc<4BDoJ-aix+Q*tM5nR7enbfEq1ATA5qNbvEG8UB^x>MGZW{uD|?B*Qi={=0RGeU83bWidr^v{TBX_tJy5btQ4b_SD8 znP$KBoiGuN$=sN_qdPYyb3h5@wSu}OWBGo;DSCUC{iOH*K1aX_$P*)uzFPoW0&Yu9DfYgY*M=)L-v^@HU1 zxDdUDn5iW1bz8ae^{W=$XRqZdKZzwcyq3Nv9(sHfN&k`|r`;q300eBt=jQ!xn5 z&Wkx1{{yRW{8KG4X-TX=K~F(Tm;a*d_C)CXK>6=uu1O ze9D~1-;MhT{awYNM~A>o^h1vlmUT*c;?|6GCoYXcwDIYTHsd@b(*QLTUQ>UZ3*Lyc zUu%#K#X0F2&mg8(yXlNUT2l!}VUU4qsZC5!xP5fJ>xy%qg(=yBqr!vk%}vP$1?eOW z%>Vtu{wAu*|20+m%ui7P5iOAj@Z0B>W@h85q<9|PXN2nGm0%Y+yNdrbN^Q>kC?nfU zVDD~5+jsPHF9D)fW7TB2JoP>0)IsAuy56ga-HM!~wI2ozs-|L((G-hEz_nBCzw3pF z57v9`+*JsRRH;qB37DWm`-r{m>5fUY-L)~GCC-u^(EZ8hwE!1Vn36#d)Yh%=UN z_3Uq3@1L!fBNB~}YQGM_^$G!zu@^J#A?1N!wv5o_unQ zZ{V8Mg%g9qeOPE~Mstbl0@L|UEkAu6&Hk0}+%LGC7K|3S9OZDT@xPVM5V4>&WDm+$ zYb0ECg}ikJ{%OYE;vlBA6{aLfOylKM#Np3w)<0+-k3*gqZ3pkQZ5LRcDV>s|%6knH zp8wR#OE-*e<1WT&>tDWEXxXa2NuodI!YA{afTTog@jG)ccw}?f?eb-%=-AWb+)K}k zzDjDV!>I%BO*fAyKI@`3a{+S7a;`&9vQmh=O8`kvgPFwCy7T;@K+a-t3(x1)^XLpP z;|W(u`*Dt}^pM6UW3pI=ME2BnO*RqMzof<|qlasYk@_}a%+1O>UlIo|?R6dO19gL|v zJ(wzFU3_0{*X$&a7K)I?KbiK)5Pmw=(>IV(mqcG!&lX;IRvXI3A?=@0gO+>j{BAyw zi;%n6l75n&Qo_FW#p-q;Mjfu($QpuE4CIPS zKDn?#J5jt2|NLZ-n4tC8fT@uo-`%;?xC5momZqcith`Q)Yx!Mw#|N~E*mHT)te7(G zV=a(Re;c-)+*N60&$ukJ2-lurbE#3{Uj6}$lIuw)?#s*+B6z!4vGTH~jUf$5qEQpI z1YuzrHKzv^mgTxa>!9B~-3fvGoXh*5-1@eWTcN^_U)BGOsFob1UOzU^Yz4vMZxSA_ zj#LOtn*1(E?}A=Vvqex9r`5(~3AMerl}|C41>z)IS)^|K(dVb4o*d_Ns>@_y*+DGpA(pa7tUa@i=sUEmn+HRYp8wO7) zGnOtgRw>6smJ89)!^ih@ERrw539cLT%H&aRp&f{^aV#~t&)jotD3V>MA1@8i>h z>60lvu#<-dFAJo7DlK15#hy2t{4P@_6NRs@HKwfzrsgBwR6hB~z|CJR<6WTtPvPc9 zFE?6AS^6YJ4SZ?ND_bc6q0!v6bpe)b*;z(bP)@x>)>l`2!C^qa8aJ|6cp5Zr6rlO^Rj`8``F%^%(~5zKL+3T4U+`sT4t4 zq!2@+4u%W8kEJwHhqsp}z0noHm&AKW6;X*1svDW zgWMU#7+;27&XDdSvS1}&Cd(=C73X9-v9ILlB%4h(%<}3m;*;x`Z;hz4-_?$hr-p^@ z0Iqgwi6`AF0LQ{Iv=s@CTc%elXS=!8RniRpaADafwahaF;EgF+cK~1=QV}p_01M4;FPp~MTpzc*!lbb|8mrEZki-Yjhp|gjD);oAw1fo z&UiDW;pw4dLk5heBRC+cPSm8-nhM3n_j5{6m|%ukh&TnJDiE|Hew#n!M(Lt_rWVla zIq}TiwXlIb=K705!1K3dl1G9b;%(9O5MX|-Mm@S%lRcBH6ZclLv&(M?&dB&@>SX2t z$NyZ1*f`agvn|?WO7socMGHIOCy(q!9bmd>U%BkOU)UbS`I>=OS>e}GvuU0S9dmL% zV$RCwdTj+4?)>O{^aA26&cF@hKHGB)$efVs=YD$yiwWXY?Nfy&sV`jqBN@6Vz!`-&4e-1a3vPj zv2*{L_Kl#(Jl9FIPci0H*VW=DUnso;Y!wHuxqO#;cggq){|AS6qtajaG8GxZ?KKU} zIpdJMUI@a`s4&{3(3mdQ*x=25r5$zOv&C|XhiRv6Pn`A1+*{Wj&{U`Kqs$8(bhl_D zz(REq-(~9vA;rPV5@UgC=V?Xfq&2C0%G+psa-);0XUbjNB*q$bXk#zR&8rBD+WjiS zRno+Vg#1d<8hrtRkG{h+FA_gjJbcAeFPbTU*3)@;d$=KH6klBiMdds7MD$#h!&X+;d#8Pl<N$$aDfTyRh}F>-_!OxgKUQnHD0}R5BGveXgHqo9fa)u*y8lrVKlul`u%{g z->|(Urqexi?S2}FJHm1I+J2pt&9T?E7J0}o^tNaX>`lKKaz%2`aW^~k&4Y;e$uTXQ zi92$8=1x4v@R+9NARLYHT+ z0Im?wqJ1{$Q2wvNmx@gc(tpX9!b<_|Q?#;@J-S*BzH{kmN`qq`4Whqc_NEq$T#0Fw z>@mzcNNSp3Ae(-D+pN9ZDhv1bS31bBpC zDmu0%B#n5oz%9h^m*O>eXg<6&t9>d9e*{M~=y0rHz0r}Rvp3xCIc78X=pc|kJshZ` zd*#GP6EJWgZ-(hAs=~xsF@!a$i~I~iZlncX`WTaawd%JcFwd1c_v?uaxm(aec&SmgwGD4M<=voIJD8O3Vwa26)7gb&Wz z?lNwARiEk}NlV)u9hzbVwn~Z-I?DLZy+zG0;GgX3v*$KOy02i@wc;Xy@v=8YBRoiS z4oRY%5_T*L+ZS%Ls>4iLglcJtytPWu5tjSf!m+g1IMxm0RhYG2{aq(bdqX!Hof7Sc zXY;&D=>lxvrZZW2$SA*lr6-_b!Krt1j(SAQx{4uu<(O27^HA#SsItW7do^f`;mjuV zs3Xk#MKO}gcDm_~ltqnv@Qb-sPhQB2yw^$D*0?}VPm*MhCu~eKPP=6G3r3S>oQR<) z^^6~X0P;QukJixdQKJnT9){FG-&Z17i8{JLBn8sXt*bhY|KhC&GaaaP|4FxFy$dUM z6Lc^5b-m6K+1@_CNeBwFLz`XdBvQtQZ`Nk5v23P1|HD98O<1op5Jy@*Z{L8PNMV(C zse>w_FJ4V`Og_JSLfA3cd4@j$qO2MqVS4I*tpC{JKFhw})S+y>xC4|#5>ov`{^93? zET_H&Vd*RPCZ5?#oui4zL4nuL4_IyAupb0o(XTAUE*(AKtdGAsLCxaiobv_$Cegs} zy*#hE&?HKAQ`$N@3@5U-zDpMuDRQLqs(2uW%^kRjH${I)YD?T8IkySzmEY$|YJ5Ey zN67y*?C%%$zkxNC25e2M@ISw8k}Ky*|G~Sz zPxLHrsRApOBMZYd&<-8O|E`C>P-pI95@+QkC15HoTfPC!XQ6TfrwUw_d#_ z+xFYT^IqYNf>gaKZA6`_XM}Io!GSycXYyNkYFCv9;=Qq8{2RAoqU>rSIeLbxeYVy4 z;G@HTKLp*}Ha6q`8l4OeTMJbLO0}Ks$V7T3{X(XSPac{5{52yj?7v$l8q~RQ^D_Km zan|b0VsFrnImV;an(j#!g0_HuT&7tfjmMembXh662f0u+nB?QPD+J_J#YLvn&e@Tu z0Hr@HKan!>w+$Pb8oIO?L*U~j5lr+n)rM|z9$cK^+HtC*XAWb-{Pui&UBE!jW+j|9 zVL!d-LrGqEKwk$wt$dQb*^29;^ve9oYqEhT1(=5K=ea@c`NJYrH^3aulG+&L`a*6x zzYV5A^kjBGeSWo%T0$;RvR*G)`-~}W#eZV;HxtE2ctoZ-N^4{EcN0OV{RP*1T#u2H z>449=#zb>r5huGCZ5_$|q_?$9=|9by1Hh%8QdV}v0YNSRXSra{i$#qu627~n%Rb~6 zGEEKo9@$H@(km~)W~g@=$m;^(ul;1HJ^d{pG@0v%CqTgxr=4MR=jb`$r9X=AGFKg8F2<0J5R z9aRCnd9()|@&_?B()Zp}AHw?jGV9#WOd7|R!d>i_5ZlB=W<1WtC+|xRdjkF3{#pE)$|PbGE7Xf3&YjY=d&<)ia4*|n zIq8)MY4HVe4otJAvH={LI;UP}fc5ffQ__%61nh9u$mLwVrM$28( zF%b%IallKWH|KV*to6vZ&gF8}g)H&`IW@BGm-yT}OedzrM57cYz0wx}!u~Fs7}2E( zhoai0@1)& zt?xrrkxSUm^hhHKmhc)qQ((lAZUdCrNDb7xfJ)1lX#H-xia=eT(e%*|GDK8)w`xUcxR zrS1gLqQUoCy!sa@Yz*r(bsC?P0K8 z6l!34)GGq(Qfns~cs*0d@97dDm?$YsV?cG9pt-*86|6R(CBW?N)?mu^~d23@;25x`9XU zpE?M;7VAL$8qMarGfly>XVZxK3g2BHItg(M{0eI1a8*d%@9By~z$D^QO8L=>4DiyY zkkv4=*3@(6=UIg%e^jISbts%-4=*f|D&Sp3bd_smfkdep&A)0pz!;wXnm0_TD_*Xr zqc*A<#$zHJY@&X~v?M4AfOX+3w%7Zu5?*8R?$I~ZUY-$I&u|cF??+Txc^f)}Zg;@D z#iFSKD5<4B)=unF2C^($Z5pYAQ2a7rUKQIHKxoZAHIk?ilw)4$+&aBA1wt{%fJ@=l z^8(9qKgt~&A+EkIyw2EpWzSH;?hBqP+7<#9fpJ-nvR_#WUe{6iOM#|wqN9_#pCMY= za_U^dSS6bLU!cr`zRrJcbImKm2SUG>$&~1iSJEQ&8bPj|8J|kc4lr+#^(h=D3=Uuj z!U7Apvv{s#)hpE6X9vOnnE7M-6Ih$=I;R;+GYO|ess2ZOM->1xA+|m@QxsnWU|eXl zjj_S)T?mMBg^ql*=52T}?A*HC{12P2i?D?YG|jzBoU1VCsq;WSJYT_EepL(+MDd>an;Dmblkc(c`>^(NXY;G25juV&+Bf!fLFOMIu`WzkvI4 zMQQL3{SJJGg8+n@*-(Q}IL72Vc`C}4(k6z})=eofBpy^nZ@FrM9lH2;EUvRYWui1^AM&~6*R!n|K5%apfWv}_VKI6&? zMF;F>sU*vNzyQgOqiTRsQy|}2KPDzmkZCCUj}}kW5Vb`U$l5rb@FuaPo~piC2F#>m z3w3G^5Um+^ZKd|Xxb?eFp%gvffQ9FPH7tklMq|<8`?4ILiniPT*{2wGUUzb?G^VN? z3FWIMpBLYWk0A2lhh6cDIBGHZNK=FLY3<;>uX|#=&Y|9gNIWon4VxEP*SCd{in&oM%%X_F%doWteaV@%;XsI_)DlV|>y+$1q#HR%RI zop;5z@bnSNAI#FC^U5J zLO?2151F&`pWI-4%K$UMlJZ=RY;#D{H_gjH&(1%15U=4QpFL65>s#h~vhWj diff --git a/docs/static/index/5.png b/docs/static/index/5.png deleted file mode 100644 index 2c3aeecc4fbd360589878ce0ca3c9bb29be7cf5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4514 zcmds5`9GB37q>TALUv`DOvzsMC5bThY>{D1G))X6N%o~iL)i^clPxlckc@2%MRu}{ z#*!&Tc7w=gP`;0UDh9U&V4k@dbK)P$w#^XVmEz;Vae)|iFmT^i4U7bgpg zppK=fv2z&f+RV56S>sY%8w~Ps0mnkq!jf7$A@N)jHJj%nBRmPhZ^iOLxX$O@o07KA$Jsn{Cat;N6M${~vzL;p|02rMZ?|2viS4mAK~dfH5gnX!8ArJ>=%Otgme z#A+*JU2oxU{%-VT4Ic&lu`XX1R%mH;UQOqsB_W)n^2x~T z4t0LM=l6+|H5ZNc&x*7)Lag2+RwuL6u9-3?0@y?k0w3JTKWALLH{A2iO*0r zk7>Nbcz)oiWAmZw>)?>@pJmAPUr8x+sc_8>3(>eKs(rk*9p9rA7^A56Ae1M=Tq7Bo z{j`9g_iI}MlEE!^q-fyewrUJvyUJchIs}kJJwI%*4(lEvhxv|>+d+l|FNg*Q zC8-odQA)NimM~Xm|D?fhsB)-hz6kfPJ7UoSSjIq0N#$*p*S$~ z{dnm1d}(Q1!3PVMZU$VWsSNC+#(F9_!c{y-^QD9PQkQ5fmW7g}Km>fCphv*)nZKtR z$qNmwhqpi!#~0urah{8p8KWiD+61Pf0sd9!rm!;l0t4=)oQf)6LB=o?BX2bZrHFL{ z8bA7IXA?4b0gvKFZDY}ki*+?3v6?GI!UAn!7FD7VcaV*L`eY0_-u*5JjsAzwi=bS4 z7WDKanpDViEP|?UtLC4ik_lTDxsYW;e2*QG7*Ff}t+%DV2v3I8gsR-}e$=vQ%N*qx z@u+nX19s6WjPb^G1KGH@)j76f^x}E??4ps;z^()uLgc({hb)tKZOc>q9@WEbBz1Zn z@hySgX_FX1lVA@1?FlL^61ig_QRVSdA&*j_Cx11*qKa8Qkh4hiNWB)rIO&9KS$_FLvRH1Gp92qaX zQ-xiH%e~y{Jig2_ZFi*En1SXerU+^c%%XZMcZuyAD*dBjI_GoDP430@j-Q^pSk zIHlrDLFizC{Gd0#L{;;z;}q+H8&~Tm$TdUhDS%R<`>Wu@lFac4+Pz6~;JB8M?=t+O zU>2I8Zsh1L)eD>#D+k6K3{G7~%m0&APap@b6hM7(-r?8AsNDH_@dDK!Z~S6LX=ImE z4EUh>g`zRLJvxl=V+!FkC25Te9m%*j-U(Ug;<>^l?HjanJX zwvN1Yk8R)&Y(@0r<<3XFE)BgNcfC;oISe~U!v;xQjBO8@(C3zj5HkX-ar%v=lh2hk z3M_S0UIR~aPgX?k?5Sj zyFn_w6FN*D4IC%U)5ojo_X+Ybe|-+y)S{gm182R<_|IqeI}@&bM(mN!42c!4xEt3Q zy4Q&WP?WQUbmUT_trw;$fEOZq`d4%5kjhfor8LXRMCohp`-wOGS4)~1^ z=5#`1_qJ2==koZyzmQ?e*|D6{w6MBxl6;P0(P6yUQ080rhSZpizl;$a4`XDqfnMZC zs}lOOk~ukts~mi5|IU84xq?<)c`!2b5z&E3&<wf6@}VQxQ)T{ zB$p4mvVH&9Y2l*gG7@~}hHq`HkvRTv7QaYxuK9QGx~^bNVdzXG)O>#cOqG&R^p{IG zQB>V&s`9W=^LB7~sDfKl;l&!_d-}M|>!e`n_PyzPR{hX*DT(@qdj03$32uKp^x>pNmI5o_!9mFbqs?zF1Ag|GYB14 zirv#7rpK3i-gm9;4$#zy;xMbYY1#U1OQ#~00Utm$6jr+xwm#8IL8bcRV94fBt>U#H zC#~oK6k~gz5YfuDcTarK`!e?Ai)A=F#j}zJ3cD-d;?y%kmU;c^Rpc;@*S>i2*(7>+ z^>8WxgL9<#?ZeWvs}f`9bb@R{hiSEw%EWX+v}2u2?T{7ra$`o-TYG&83Xjo5(e*r& zeCUK`qeFeecpr&gVy?1tr72-@mQ$(v8j+Z`%#RmmVJkh~Sb>m1I%;}I#P$CR7_-BjPRrYFjd5x5AO`0@wy<>uZ@)7peu{JdNzah0Z z-G@!bltDE1QyQ!6#7|p6_1vk{VDl6dT8~m%@m@{cAE&NXpUzohRuPRxH&6z9fxBkn zR&?_OD3w|VRat1zAaw`|lf7knmuD?(H60pY<@BA84`z+H0*!Nl1m-PL%aTfcEQYFh zmISzfkZfdT!&YCw1<=O)HSgb3${GZ6Rn4KN;5G>}T|eqqeaPq3!{w~wE~);20JG(T z>tow}m9p$U^Y$nrmyhjDtx@yazt7hBxxdoh4f|(Khr}y=|5~Y|>lmjn_m;YusWlTY z)72jih}gOG@2S8VcN(qUiZ|Dw%OY|JQ*Wlj#;~3DYeP62q;DmO#(s1ui_}3nid;A= znDlzby-wAmal*LU%jR@L@LI$j}^5|hx z^fzK|7BBi`HwANaW}DiIohoPesbWUnfG%3j?PSV%T*7)=|F|l81)(8Vek8LR{4S%g zy4)8+OwT(GV3Is@bFf4a$x4zfN~IQ7lb}+@#td#Xt+7sXtO{)WZ@RAInAIrYcKn`^ zsus=^;RkHGMtQEP_Q`E6k}Z!@&zrrs_MjX47dF%$$!T5p)x~=AnA9gXrcps|kIr*V zaF@uKd@;?1q91Qm#w({l?fs4AdBD9;^k@`Er)V-#5VH2=XPu1kJ5I1)+=}(TMR4{-Zp*QB^iBxUF;9($I2-5UYY0h zx^dx?lz#XPII=J%$t9#e+nru&UMc1GPNeTPP;WYT83f)eh*jL5iKW=G&*%)3PqKh- zSv(+-+v-zM-=bc8h<>ihCLW*8`v2Ta<<6gZ%qx2-1AI5}Okd^Yte47>nzG;WWm$Yo zQ$4+{{{0L@v~FKAn;Spu{>oFF)?c7^F1EHn)@8mzW^DDPn$RaZ&{mfQ^?s3>k)bVS zL-Oz30Z;L*EyHX5Iy9kAY~ZB4WltgLaJgva56FW;FM0P((*PSgaKc=55P@aX3|CQe zg`RJ)RbrUOWwQzU%l}>;x=-eyBr3dyi}(I0jL6Yx1tav0B}pD|XC65vXb^jlcZ7v` z6rKeSjR|-9Tugu4%uaml*^XeEu2Z@7a9MFnx~z~mf@Gf?>~;<>Z8k(q8|JGA_kXR8 zJDn*LmHq)_BbMSgOUR=5A)Q)vjpIoowO5$C#gR%9`l+o%8530EGW>;5o^mgA+@Xgp z?1)6n+px$zvuxog}BlzUU`_{R3pFPx|=rm-}mrvJb{2DotFx zAQi4~{I_&9{U*J!8G!tWFa%E+YOT?HL4TcMrTyI1v8YK$;nC zF;glTGmBMn#xl`S9FTl z&Tio6PT+Pt8>H9rCu*&lqyn>xNnKAvq5^(Ca5#e~s0>-zbOHOA#we)>Odz+lJ;h3X zd<5^()U#OiYn_uc@y9khRW1I)(9ve6t2xw= zS`-q|UlNw&63v)Od-&jloEs0xJq?j{_e5r=QMI_ax;$W;pou7c?QCp6`1jqPW(;+9 za8g;l*@^-aB_n}UK$Zhe8YC&G7q7?^-#NjFh?%{<`a{1yFp~dqGwn-h!fm zynPPzf(tlYKoC{VL=0fKm(+SAohP3?o(_XQUMfO0*ZcSqRNv6GY84Q z^8_<7%0vKSMH%_erEFmyA9X^`^*}(qV=B7!3+m{(0S;F0P?jqHC@9=@0q-VUQjL@8 z{gefjXexBLWc%TyiD=vfC>jR5k@`hIZpj3xa^I6cJrTsK@0NTn$PXaD8%zx4WzH>ta^o_RkCV*&IxPZRXGB%Bg2wOSOg^v#EB-?tE>h|FF>DKe aWqPAUy6xp}L;}B5SS-!#Osh<8V*dvX(U+V6 diff --git a/docs/static/index/6.png b/docs/static/index/6.png deleted file mode 100644 index 954dc7afe3c65c0f06a33a77b68edeef5cb1e118..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8555 zcmXw9cOctewAU6Ddq<6`h}o)H6s2bEJ*zgYAc|TsYm3>~qjv34Gn7zb)u`GlqA{yh z?NQW=-+S+mB=>vIy`OWwXWelE*VTSOLCQ=@KtMpDrmCb*KyV9yZ(~S^@!vte?T_$3 z05?^$cLW5CTsNOvYWmYsUfx_Dk|_k$_>8>L8AtG;|F@dV@j2NicyDhjBCsg<*&tO?{>fZ zdY5-qcOrYibMDNPxbM4ry?tzT-Ol*FKiT^5?e(gpuM9w8&sXKh%5S$kNUqfr3fY9= zpM4Q~J^9a-sB?PrUQmBLP*_^R_rw7R`^h4rR+J0Z?Eh=t-3N`?#t??CM4<7V#fCH_ z3BVX{d^3VRjmexP^J_nuvOj)|$e=bg3`BD+AxK}{6RQ+kW?;Lz6j2!Hi*N*p+)OXD zz&CG+5ZyGN`6Ap{8E$voBkdhFz_~LE(oz=bKaf1zZ`W;!2LfJOGLe5ATc(@6~0*%TIc-53=DCC%Ji5tS_RLI{Uv_mMB%z$oSh` z`wBKZmf%O|*EayQRCoYf=#2og5kLT$o7xT5c2_K+`z!hzEa^RvJX1PRXm|MX(dRV3 zAT?ndlqz*G#cg5R4zeX8L3+=9ko*V68)XvRP>LdMILOlr(uOa9VN&vS$4t_p&^;mIr?hx3~C_vZCiaIAd_X7wdi!u`n&1(aR)zu6K}& z*BvkU*9IQ1(BYUK`_uB2Bm^>qE{f^Cxj3xEWr9D-ecGYt2&<+Fij;d~X`a=*dEG|N zqkLN4@u(eT@ ziJmRQDrLF-T=1%m{(ibRcYgkW^!}|Vx2AQH0o7393{!#VF>Xjv0pYt5KIMx;-g~u`p35GKh( z8%CWr{&wrW*@L&%euo|(M+?*Trg^?*_+kjq%5H?eltObrxf#AJbR?TQqZ`v&d*t#H zJ=p$;l3di<{g+2&&mKq1A%7aR>~z23YI-XGnm!u?M|1YE%6L-Zq!3OTg!D*Z4~ zm#Fb4LhZ0GWLe+EN5B3Z8aC*mO!$vl&$)PHK}Fzz5)|%XobV96rl(m^e>iy$!OQ=99~_AwuX<^&8OE|f;ZEc zSzV%t8;+6!q1u-Er<_7ET>6yryBel~v|9ge`S~moO*%Hf)UMQnuexh9ADvCk)sB;HbFOD8{)y3pDRu^R*8rfp7b}?-Vn+pk$TQi zNJu5nXu`uF35e+FOX`+F=;}GS@ir-9JXul-A#-9fM(@3)9FYCbP~G&q!ZltxoPK`D>sQr^OY8Cst14H1VOzn~zD4MvBYxq)4&w<2}N3 zg0x8m?D<5oHVtPLB;q9ptrT9z^vJ+3s3~Xrmr1@^{Ny@o zwi*v7NfDzKcm=pzc)%)wHRAc9faF<5D>NxW=y>2$?at6zSBvSmo{i$$@~*l<26aoY zW-2o)>5i_+A9r100i&qWnA(E>>;8Dg++cHOVyfEG&^0*b(E@@j(?9^-Clrr-b;Q`Mthcb31ca}gkfsEjV`rrFBlK=9(w^pg#IHTdhDeq zl@F5?%54gFGKr3jc_E1WNstJB!gtg*EmJ@nq@+k#o`B|eQdpI$F5eT$5==o&W_pPZ zt*^*dGT+MWaT;hnNWOd$ca*S5QQ|g$vJdsgHbUmE?WasJqOrHCo>3@qJfx+caCAgw ziU?WEcXw&i^yOu-DBT3BF|QgT-`EK(JVoinmkX8hu@#XqA~3TG{iyoNZYMt*X!%sp zD?tIZ<2BcE-cMZ4_$vDu zU`-EIGUiX#(AKXmQAGZyEDyQL8=oa>Vvp|*xXtbFKw6%w@y5jxB7^<~L;l{Ti(Wel z#+>CftFGVCgkE^qRHhG1^Q7(>(gxs;c3J;q)y<$4f|(yz2M5FMkysmo_p)3FozwF> z=Pdkxy#P-{a^ro4x=iFtwR7z30R0fTjPk0`PXLZ+-9UyaQka|8QCCIid59^mpPb=! z{OKZ*Q%1{=CA9TILf5C_!3b$kK9$}}q>7u<*2U0gl{abesg5d2p)4ueILqIH2|5HZ z35{^?vK{kw^s|dD!pZqu(No!1S4rjs8Wd zHXDl3InVWYLu-N*4PsRdg?TIA_hF^Be03Fkq$gh=j;BDrIJg}cW@TQmLuE7+n+wWc zfh(9(jO2qu$1`Bq9o_8jLn9%a^NNONU(iFZ=$}(*%ePZg%IDLyX!AcZ^{J065ot(qxM0iMZLI2<40t1@4 z=-K12#1K;7Jtt2-eZaVKVY$2+gHs(er|1-V#HG1CoA2?0lcCMHD}WwdSrI^54&oTX z>t)^lPZF#PN3Ll>ToTU%cl(3#mJ3zDh(7eLCO_h%UjV0a)7Ilz?jd;@f)z}pC;Pfu zmOAddydoi8;(H`o@+Xj~u@J{yPLI%a&@~Rxcfq8yo^LEpH>u_C+_SnC51Y*X6tApM zj2Ffw_5`qcu;RV3L^K)#4RiPMqa|eAos6}3_f@!B?7sWskZTITJZvU*z0 z&$4@JQpfhVAy7n1t60+~Onwz9Qkky@>E&`!KY*GvbIPF9vAG{QNZKvTya2A|qTMFk z47r)`~OKz-JC8rqNpf`R0dlp#(wL)g3 zw^~Ya19x!{HTmtAQXmO64du|R8^4JuLO^KQc63>WHN*tGB8D|1N=KtIp8n&VRbo?Y z&;{2B948ov@>Yfl=q1N=u-}9D_`Y;H$RPY_!$&@yuvG z>Zfs7J%5QUrUFxta(5;2-KGWw9hnbFZ_N~S*Sq1-OPyVVaOzH5LXP!JV5f~{AJQnH zxrnuFT+eC4vc+!oZ}Yv~H<9aGOly>HX6qn-X7WB#y=Iey-M*-rjxWB*yJjyKkT8dobBTO2V;eAM;q_=k0%pGUC=*H zV{a`rD?ejZX?DHMV3?KGI-fgnDzo~DN>AT1PMF|~Ab7RUVQoj<_cALG$1;-2-ePWO zV!__5$iT=jWW}u7TRKqlfs(fMl_Rx(YMts)L5HfFu9vXmvifqo>y&ZFa$B@ z+}X;AeMOo40K&fXvKCd-&67?UtgsFV4Ta6k`IxWV)uv@5^Vzd4+Hr`CI{@|aUk@ax z5>jX+fZfmCm$RqS*`;hTSWYs1qN$^23HW?GxinCLPc#E2znB#ma5KEG*;loH$9NxE z`%t@re}U@`Ltd=)r}Wx&__~;L)%RmE7dN>sh)P9d))d<4OaRs}j_W0A%@`#!F?0H= ze&DWYn-`g3DzIG6j($KTsjql47$?@fHziRmAZOT*bnuE0 z%Ehh@jIR!0pSqc5u@`tOC48@Q_FC&@(tG`dh97?-&dWeu)>2H>K{IzoLkfm0MVh_J zo&4uF70VtAa{rTI(u@gL#S8pM&V?0GDuonqZh@CqTh_YF||$WS$=}o%fMwf z=-_8m_fy%CS{r?`jTJ??Rd`8>NAn?kTfL>5y+U?NOj4{3Wd*2={A?rFVMktxaU3u; zp03qMt08HM65H`tctd49R;L%u#?tI+1FJhZfhpelVgqKG6OaHaH7M6X6`BXRdPL$Z z=z#6^Yn+CS&!S|*pHUSqVPk`Z>(L*&Ip@C38t<#V8$k3R0oXIDGIZXDG0G>xOeSfN za*)iE3!L3_%Ds}oSbDp=k-`WP@)TAzm+aAMex$(4pU?j5#pDzEz37~?zs-?3pG1Vr zEx~h71Q2eQUo)lEz;zA^%@woa_p#ZFC6f@#Wp~R+2%rBFWe_FQ(MO+Gun%ycLt9Ze z2bvWrFgJ2(gUr78W&BweJ)$#5=CeHeb{VsekR!px$HC##gc9&GnLAdK>RR0B=NTw5$*u z)avK%BWa~Hyj6I$H~NZ-s|M7ykb&ikn^X4Y>3F;iMO}VTUszqayLr13Hzg?@$VMtl zfV3BYWiffI4p9_`L{*@+8Gee=9D(hdo|QwfF+*hDezlb-RT0;hprzG|S)U}FZuY3a zyDTG}z2)q|t`;p#jeARcN6x&xH9mu6-_y5NX;bpRyvhyA1!4s(m_iX-OySCoq3oEF z=tB|4-F&JLE^eYnf`WY{uke{4KgQhH#k?oq2x93?n4E?mp$oUbSXss;C)jhE^%bY) zDlw*Tdk#fIW08tR*cIw00@;zJx{vsJYOuV5<4S95#A}X75<_m!6}rHyaU#bURcXUs zV?n)%dWdffy3slP#<-~-VSe}+B&BP0-_)I$MdHUq!1$bAibKc@qAl8tEkBimX_5FC z$rmb+&V4}T4hMfF(tuW`KBo#FJ;5vkGHSC--x<4B!(H)POqT8Rvf!)_ZBvyskhxXjPvzC- z0*?emvKcc$HqLnOOQvhB!*DQTcJG^&9itIX->a5r{mj-$#LXwE=Mu(lm#WZabyXE!U5Qf9(oGk6aYSe z+A#8wLVo-CBb+K^jVJ;6*8CJCzrNLRjQ3SKJYxZVT1-vDj>fw6f>aL0ScLQhbNWRe zTWRx;=2|@DkJw>ZwZ_^7)gpg|W=EH{gp7eKm>G{MSH0WaYi&uG=3NwOpV812Iy%0JpACnj!p+2Lo6P( z0i?J+V|$R5$Qvd`T0^Y@N^IuM0WdvHW4!X|;Wr*IDqO&l=k0~(gDKDogjbX}~>1T9IwmBT9d#|nUVA+`Lv-b=*D6rv=hq?0? z(AdS$M5*78+<)ys1pTuwd>JFzd&pEsQ|Vf_`jBIWkDO=3=QfRsCp-5HjpS}qm@qh~ z)T3&l!U^27&XTQ+@@}sQ!rt7;Xy3sN_T{a$ziZwuiob+6oC!!Z$b5bBzL)^#PGXH{ zrvUa6gv|lB*vgr=O_*>K}{a`6n?8 zU*f^kwjZpeRu4URE`U&-xET2@u@`(hSe7=9CqwcfQlhS; zU@zo4$5*(CW;@cf|4Y5Uy6K&rTqjgJfgUnd@w7QPs1f@1R8Nf#4pw8gdm;34!|vN- zxji-bOykfcAb{N^YW-d`sDXBHzS0Pv3>`;g&zQay7O(Y|#YI1ULERy4O||YGY{O<> zqy$a^6Ws~MinE8Mv)3ndpA~l6JaRaex&M0~kR1^XGZT-mQ1?}zo~~{|S?5Iwz}Hya zy)TI#aZb;{m%a`5@jEJfQT+rdO6=nCTO=GN55`g_qC{HLa+WCEKk&nB&(4JaMiS5Ke zSU>WOI=949U;;|Opk=DELNx`t#ZRU)5M$Atk77CmZZ`56*&2eA;s%qx_P>36!4z=7 zkjAd%3R>1{>mN)g1WL)aH!og8)?+IYU}p9&DP6I=$C6onHEfRs`{q-U|J|mj@?AQ{ z-<>zvm$@4ImHT7w41Pi->}b6-ouOwSR~M0{zH0Qa^PWX#Z7C`ii7Y!t6+hBdOa|;7JeUfufct-kZi3t;cKThnVjxei6NW;ga9wu z$@~hv(QRxRyF(pRcJXl~xU)=C?+q-tPx0%k0rbxSRWT8q!RP;@>XsRqOi-pRn*X zBJa1vq-m@Vxr?+ipXGIl<1dv*wnaC5`VIw0mL=A0aG$FWg=;T6OaC!f9&V+vyPJ#g zAmizp>gNk2-}Xtp>6h4$)ixZ;r{1ilEQux`k{Dj%?QZF*ka6f zM)^xqt1{EBD%l-!L8DQf5DTx`d=xLlV17+78i!H^Xa21;720dvc3=sJlk6MLU7hW# zdKt9Zi;P;?Nr>C&!iBlVqQda&D7<;?+5T2v4Imvj-Na`!&W}({Iub4v<@r%B7=6A2 zHT|@1*F)Dj_dDN7J-Hu=KbmBf+ns7X8tTeSU&=+bcnaCPw8e-zFKJ_Ey)1*e9}o#| zc>gGMtX2gBB11#0s3Gi}FdBg%6Cnb2gh|t7XOVOV$wb{wmA4}THuu!iei>>e{6}gi z5S|J|Ph`!91L5TQvc$Uk!h4d5iavV0gW3$Rx*dG+r(db?8!LsuouCG^f&AN&gK2@` z5S*J;*jjJK*V+MXL=8S@mX!;`U$JvC<%+2MDbW&gQ;_J`t4v|f==&JB0T<58_qV?l z`mk+hw2((CD_;#Mm%SaS`SbW9HGm{+E#ef&Je~#7tGAIQ1_#6~xmkY)j@7Q2iGu`P z%kkG5zO-s-6#ifTt&WgAgZbPE{<*6L4$YPea;T~dsoSboGm9LIHO%%{uLlylU#clk zlgg^dfD)T{C2Cd^6Mn=%SAHaeLOPVDArH(dK6M)`ZT}meZR$HvxM(a9Us6_}C6XnM ziL4KiG8#(H#RgQ_l1xcfgl|j3yhi!iE0M zQm?V_UEc&{5Hg;JTy~N29fPkA;g|E3-~=7=fo2;M7@R{tEVwvDZMCqw~_EFq%-jT)}N2*7M5q? z#rVRd5Y+a5y~7@fJ2X#ejE?NDqS)6riR zoNy!%5N3bQ6Szx`$x&km*?zsY#U7x` zAzDy0zJyCF+h=yky==D!z_wH-r?U^tdjFN}A8m|Vru!!3eRVbxTxn)0DP3Kzb;elM zo1b_qqkjWEvCV!!RS=qpK!N)(g&C1O-W%!n{TVxa52J+qFZGzJD9ke*4LyqJ%nP>C z1yi(YTzmhsGn!^|RoGUKNo(VNACf&P`+A>rScJ+h{+z)-)2*sKP?iy4vn>^aDTT!b zb@u1y^u>mU?fG+U*u@cV=3B>|GX*TLsgw#^b(!a1B?fneFVUF#Z&}&=Sk9x09~n3R z&6NZs-%EV^SG>6Grqr%1+b2|fZjcd(OSIWaK7H}kMzkZH=7OqiO}Sp{3wnEs{cJ)3 zU%eg^NeT=3k{E)eTsI(O>W!2|vfn8op&?ZjA->{11dhOL9N8_wFNbRqX#^-;ErTnr zgRfaGn6`gant%OkJy~+(poG~6Ey3@R5wM zm1XmCy4!b%p+b?R-?$GX-NQ=U;Y|j-qo&Iz1%28zw|!)#ir1;vX}77jf6NB|#Jul2 zCJ|5WFJPj~sv5=|pFQRUMbR#}R1MM*gW%p@#@oGig3gK^VJRSP@IP~OcJ~`=J_EM6 zQGQ7fl9*n$n%6m+zPTg%k=uE*W}-DW2|W~qY-t>yf95m8X!jfS@};(4rV@IZ|7ziF zWCh~GHD2wRKpR8yF&?!WL@@gADtoiC+CH9MJ!~%M`#%FL&z2k~$OXHl4fL z$Uo)FBA?+P_7roWs@GC80(ykq8Ni^dS?j+6Mp7_r=v^AB{EN3~PB@IHL?_6%Fle!5 z=lI5NHb(~9CqypXdaEqLMnX2ZL)}vlGbo5T?RJ{ELxblnI)KD$p`yK*7Mky3DNiUN zpfD5NRnw^MDfc6jV!d}fa1iw`xKLG(=7Iqyp!BJ{?kW>*s!eDZQ1mQ0SIAJNX1p=w zGMMC5@&Ho!p@LU#dBXqM3Q3v@RjInwe@>fTSIGyZI@Xw2k3afpebW{&1XH;Kz)bx` z4}`&pp!p{zr}(RZ+QV05*mo#%^J6N?omv7cA9lbFO*1z#zoMx=m}V5Hq43;>$%zNfE#Fd%nv7=HN>cNJUK`0s zMJfF9Eez9(tQPzX=YAoYjw<)Yl*wL05_kH1HL*5z$or;|ppp+u3IPl*sa_c*?M*id zG+p*Dn~)&e=Z14rUZkOfrynX!HmlOA=D(WU%o4IGme*1g4RdwaNuDg)16u(u>H?Fs zVS-EB7$foid))92HY6a%yY$NIIsX}zwt>y7VruE3iz5F43Cz*Wo0!SF(P~So53HGVY zL<+a|=e-KgTZXD2spR9A(P*|o+Nv0vgPJ2gAST8q$6N}7{;gnGC~g$i2ktz&BqqY-#2WMKQ1pE}-N9MmYBo~RM;hu{d= zhNM2)7^_+lda})ld|+uZIlsr0N5oJ6to?hN*tf^jJCgV{45^m$e(=oC z$C4Cf3(L2Gn)g1WgwU6#ymLDk^gOz5mpt2MU<)ed00S_<)kX%)f@}R=Wz4!tLAI#fRXEgd2Uh+le~?h61^qvWykcCDz7SP8M26>( z5U-vx8~5qk=7M8e3?|o?3D*q&qfvyylSOc|k38v3%G9JU zcBPSd=#Hapvd0^Dl;h%EqJHyIrow=WQt&2L^{y4+Sx_AbI2QBoUp*oG@N|+A^WQi+ z^G8wcg5{2`+y$~=`|$DA9F6B+tax<5{Z7Dx^j9`{oLh8q-7HV-`Gf z8Smxr(!DX+VBU)AdqNLwXREn$%A4d9a)A303ERz(LMZT)R^wC_hOH0s z5Q$3IQqHsO`lHk9h5}5^B2EIs5Pc2Pu|;NA%pS{Wc%Ez99H0XJ$Fc%rh^(8+fk1u>JK22G9~F zU<{!riB6%n&7l&U!=L;(e{(9Ihv0wSg<$j7-3rjJ3X>|R-I2Tv3T*0qbk`h|a`c@Q zK5{6X=a2<6ZMzhpT@pnZAfIl-pEfE82FkMS3u&NvDvANM%@zrX?YVn(WZ>*t_H?`7 z-xzR9Zn}SxCf0$hwX!VA_bgVtjg*}KTepVz$GzwNGwL@_#R70}z!ZEKv_gDZ%U>{r&C2!LBkhD-OkltB1Slpi(rrJQt|Z+lY0qWcv?h?0UxnGFLPQ%jLw7_wE#9ilbm0BLDZWmtVV#@E zb`COD;_$|pcIL?1(HaY($jXo9ZHyO-}T<_IHZNKG8ZmD8=PI z2~K;>khm#Cq3eI7^RSxM<`Fbln+2jL6yn@|X!p(YrAxb$2&Cn)3So(Obz%*%XY-53i2iJ+ zC{UBeN4(dsAiGIOGs@LS5FRmhPmguc%Rfs|ENOJmHlouf#ENOlR1ki{`?h&jHJwe? z>b_c!zElGaD1-9#|G?yO5M2j9bb5D+ZU!HJnR}@I4zn;~10)2;S6vEJ+k-??yOZAq zJxc@ImD(OJ@5|Ffi(<+ws(vH8qz$dLLh0f^r8ZxZeEkrrTmjmkd<{oa$8$ zp|Z?f_DjP!^c1ob5kHwO=_|A6YG2vB$!8_~s3>Y*fabDjovKcl396x~WX7<+!{7X- z&(vE^9<@8t=%Bd-6O`YqF0+Ufo8B~~iguAx#{2&eWQS^KDVZ^?E{V`Ols^#h>o%>H z<8M%0D~{|nlZ&lQ*@U2WC6-6>yO7BiFW>=g-RWMfg|JE^q0fKCmq$LUB7J1fdd|aI z99Y}mv~xWF>?pwvm=%UVg=!6TTT}x)b3ak$duruUuSkY_KQqgzmG0Pq+H1D^13kM zW_cf0Pnv`p9GvqvJjiptx02X9sb}0jG>Oyu&>aHdtBk1^X8d(y_gUX>nBAI3fIka= zai~+U=dJBnHO{p?h$Y8A&CpNRDw6=0{Op2_NNh^b-E1G)@GUm<*v1V954lKglG1pj zy~ZW5#9o{G)#xzT1!FP#rjg~nES59-SIOI_wN+e1)9@ReTNX1Q^&xy?!X2o)I%2|+ zyk?B7*+hW>Q2S*)^w;%kp@ZU|-RbcD)LF&}VnC&+bt#Kv@+Hw&FVSMnP2Z8SnV)Yp zSZ<=!xY4wv4*A!HsaR=d7zQl@Ww(CXFg|2Mc3WT^ldy9wn%!Yv2iD2& z&M$0Se@uT*$<6SIm`v8r)_JY4J%(`K&Wc)-Am7dt=UNqY@%wo z%(H8jT*04+i+Zrvi|p&mzVnYMtPmaGHcX|pYgVocHlLg+&f{N^86%Ax#|lNvu18im z-REc*T^-or2X;N+*ER~H^Y6Q4a5=VdXG2g^v)fPG%+6Y+n7gy>TPup1l_SvJJ3*H1 zmUEARdUknDY+j(mfIPCPwBX<0vm#Z;3BXP0DT5x@F9;cIZ@b{&9OMGRb z6kVdLcuR+kizodJ32KM+D3m4y}t*A#y3#L z-5^69I0gszx(MZ9w}B$WXqx*^Y@X$*Qt#m|wHtz(g-rKp&DTSM3>q6%9(2faQXLl` zL=Lhdkwxd}k(>mm#N0Hcz513ChaqeTvKpiaMfA);dG^q`Ij`AmTh$C zUHtCUWY2Vgtg&3Y(4*&iTpF8UEsiCHnNjkkW>W z_mpS;euvXj4SS1|8$I0%S(mSgayzAx#!0JH-)?ShNB?=MRkS}z7C71WFeCSaXa=m* z^~8uevbvo@#|5xO+MvAonkrR3LM(i~@w#r3<4`LuvjYp^?<7S<0l_Lb!u1a?{H(O0 z5z;o&Bhgr-YEU*MV}fExWE9LpiH>lNT9nvWwaKK{d@^W+BE{2U6UYQO9Lq5r;pd-h zR-eVnvzV+g*(dc2W})JRv7k&)ffECTYCZG-@hFOgTANUd||(FtbwrcTFD7_qaPfIh!lx8a*Gr-~DI4 z#W?oIT_ZM!oBL#3(4lR=l?E-G*91_rkVdnZHYhPZEnc~%4CfoacrX9cfS&JZTAT9w zj)_|~5ab8>kkW*4otabgSG)shva^owD4Z)rEz`h`>|=TG(;!+D{_zRmG4%NR#uT%9|Al{z&*SKK5+i(5P3mfm zKTxNJcc(ip49_W;kfSi&`5}POEo>^33nG535>2Vx=RLZb# zYP5EP@(2a0-?)0X{t%y1NoXy1L8#r(JoN@})oc zW=2zebNhVpSL3*dt!28k)PD3C%P!4s(E$l>_0dk=ug$PfZ2@Z8+8#m8$`RM-izXj`l90TVwcK@i$tzlYAr(u`JwkXtdVUNY52dtN1aUG zj>4fWQfKvr<3qM#3}||u{U~kMYMV7)w;RI_iYeAywP`Yv+_||v5yW9kF=Uc%v>T)G zZ>BWErvCXnzj?99#b8&PFdpA(5VHT%c~zJ$Nsz|_K@X+PnAhD+i7d}Pz>=3|w|rJS zk&)lsY3$EmY*40z^eyE#Df_hUhktt!LydkISa$u#1O;-rc{^%MUKpw$zi`2r zDZO&gH8YyKE(oGbef96@eout#^Mwu-@8<$liM22C3=-MQzDB^JA+#~E5+Mjho(pUM z+x?tkcIDOImQ8mO#s+Ye=dr<0&b@z}XJt3h#?aVs89K63{!HonqKf=&uK-o{##2{> z9Mbj25aqH2bOuu)iD$z7;ca{4k?JLins-jx4?Evc)LEK5z+zOqFT)W&*3LRC<`WdN{JQ|JlMkY?qwgK(8W|(b8+pKA?YnS zqvp*Uj{SS-S_5qcJ&K}y1JCTwyr#;h1uY6dkhRME`5QEsQkKNR>-3RlePfcD4^GBX9c#|a+FEuYEJ-0I#x>)feAkSBau}%f%D5c zU)Q066Zy9%34wi;tQ=5TYl%7BeG_ZR1z^b>dIt0LT`hq(fz0-C-U8aUB^}{g7?Y>| zx;OxksmTkW7(Vj!@^IhKXoxT&=Z(nej1JiJ;8xEEnXI4=y)|((Zvg8tR?W)qAGnh6 z5R<{;jnSN?!_$I51Db4L=Y;en#aP23&^3lWlg^=rl)kH=dx|24%D?#f@8+&rEPKzH zU(jJ@Lj9II&=!X@`y+CCv;51SF+f2v$;etX=bzLea#goLkTgB|y`^-JY(}AoQV54I z1Vj@-Yt7{}?Ly%4tfOfZ&FNYT^glqXpPS4hLUY2e7|?v5>r+^f9VKXfC7@>D`4)5k zrvbp`+43`%3vVBPR5?yx5;XZ8#3~nx2>FPfEgr>A4~sg#VFJI>thC{By5W2Uu!09_ zlPj^H__ikH;a^G$WJu!tw_?COSXhy0QJO>x2p$S)QhqlDk8qaP^&81fguz4}~(uWQnL__?ep0fy@3+|fsa=tM5|HX{Tf3gWh~`P@SGWxmlB z>V-09A;2)n1Y?W(dwZIoTp^E1KqR^BF&4x4ds@(x5JHfeY;o-J(5Yzre4)u{el0Uo zqjJ+KkvMkVEVxkOBeTN+o!>zd&H6&{^5tS>lfgxch5MY)t`f*Lmfb`82HZ8{7iKv< zG3M?fdGopQO5KpG(#W|%@vlK33NpzzSpvg2|MTq8`m-}*=Cg?yh&V8hCkaYXYE?7= zs3c^##YbHj(W9=pm)4fy-(7Oq6VF2<}z z&3jA>r}5K-gm^{iEI0LJtU&qRx4pU8OhY2rfzU*o7wa#FA%O+Ebx~$KIVkFWX4ZeZ z0|9+BZ|0iCh(SqU_jpM{0)ZetJ1F1f(9Bs@_n5^KV|n?hqJfD>5J z9OUlAuPZ!6K3dXl{68&x)h#bFPd}-yaeJN4~ zY(B4fIa-;gNEa|~6BhAvsA?)TkJ$hA_A$mL&+&8k^YNg+$CuI>2MT7Y=l$@S@#nW*O6$aLuEGKb z-hSrT@z}qZH~m1CZ1(Aj=E4$)UiIqv@bcl5oLkL5guRcqOvRWfC`ot=HTO`So@@8tZ0 z&;OY{VPLa#=VE~1R`!Iny0wR91UFV6i+#;s_3fR-<*!FrC8XA0cpe>RdxvxTG57vR z-d*MKJN@=>xBTt?{lD==YF?C@9)n``VU~xhugV*+3BPB0?q2P?&!o0lBrdg&-|&y! zjsjq$@EdW=i(LDCLf{g4pmo0(4HB2gEBs}6$?jqA@QYE>4dw`3^gHe)nj)2P4DB0t z*m=y4){vec>UK-qcM-$7hpp;ejE6rnE#cKZGW+{)<*AF$#b0?nK{QC>{UjBKR_S_; zAC8>-<{yl;Y&{z;{eA!Rk9xN|vJUDrKlhng zrtM(Up8IpSGWLXBVs0+WV&r1BE-+}i|Cd={!lTfFYef-WjDCJ+%zijM;Y^fdd9q5v zWQOI*%M}+y)jqx7GHu>|jr_)hlG_Iv`|qFl!t_PnveYiCO7z3q=iA>&I(;{h9y zcAH7Vt!8ezez>^^yPaFeEf&W=3{GqQ&E`Cza-b?kCveGT^X*&GKD^t&e3>P1ZSftW zLnkMmet7QUyoU_A>~g9XJ702be9wGtZCLEax1#?x7QW@`_62Ht_4VADOHr-*ovpwB kGwNO6=u)Pf#q)!i>x&it?&9_LffrCU6sUksv^nxAIV;TWqri*WU6f13-IG> zq)a8I7m$%W3kabNj5J@NBnvm^b~iwf1SwDiNl=AThDm^UsSmpEyFrvAM~)mha^%Qy zPM|19>Zt$mYx$US9Oo-8s5v5@BPJKkJ#F^j1AO0QribNcTQwRGIf*GDB<{1qHD+FJ z64n#gm97j*=pyk-R3)1c{K;SJhz40P?qeo{%-V^&oeYqk%Y1%n1 zETi#PTYuHEPKHr-9NwRO+I5(e{`vvSExYbw@#u_jC`V%Gp|epctE19R3xP~Etk0wO zeMyE4CBN_8Us$CPASW>)_WrU!PJtGc;vdQM9Dw(k{2T7Gt$C}Q)J0LD(}?t0mL!T# zBIG0{0Em#@w#Giah?og)xOjN6y!p?!!G0i8K29gnfiB|EY-Ec^10W~y6leko1U83H zeB~8)bNgny!XijQBorE1W+TUVGywWmY#&YG)6d#}m-AFoYB_R+Gw6xu0khB=_&M?S%XnLDL)1EX$+RZgG zwk110-}v>n+wW)qBq9+*gNvK1+f@&L-0d`cepy8Q-tXk&1~iY?W=k&u)pFc96!H4( z=Jtc|obg2JlXJ20tA2<1b|h77Diq!zt({u}#SUbr3<4k9P2xp+)G~x(AD>mF8-e;D z{AP3q0~{|OJMu6Xhk%8m>YxM)l9K2KU;IxeAL*O|qmB47(Ob;l3kDyRH@Dm5DC0Ti zYp0zG|62CZL>3B&T&Q5^WSA_kCtk5qcAb(K}ss7hTt*-hfk^e~NB^siEbleF$Xb zA*F=e*r%3ZRG=I96ixOe^@P^kLr;!8Bp5~)#lAK_Q*Cy0>y&dO$547A?5AlsibvuI z7}VQg8d!Xyf!#(`mTn`%TRq3e@!2RQxUC;^DgW$gGFe<(LoACGUy*wOt`}w<4 zB~i#zhs76kh$n7DkxK9WLe+h`b-UL-p_j?ZDJT?s_^63Yl3s}K%?9@A5uE}kJ918O zeC(JZn2oW|YL}@s+(}f>f?e<~3OzuRgB&wt0d;C8~qXcu; z>j`;iWH_y*mvQK5Qsx0$_3kZrWaOJ`PSw4E+hKjAHM_(@lfv4bH20`@2M@p+b^TNH zjp+`6f4`-KNR31o$=C6;1vk6Pz8c62ksPWTx3$y56bqM{=94%xs+1BU+HPP^*3l+6 zX$VNKC&B;@hEP8CZ$}N}QKOx}AXP$!xgep53R94S{;j}+g8g3&DqkblmydSG`IWTJ z-;NZA(}+81jJBk>2wMpx1Z=;r1?ysT@7=$O2h4~2Fudn_Qu%Wlx`eLoAoAhSkZ|to zMA&0*0++DF{o2DuO`DLLjr{2(m0>QHKqe5_3>*&w=ZZw@ue5R*e#xiXaiOQOu?fRr zw8J;5_i

%1cS|LTyNDhL2&)vqUw>oLUpORiz>0T#*>@xQH3i;SsUXn22vwJ;6}B z7WrL149V6cAmjY6Q5Trxso=qg2{w;I&-egt)H5H6%*c%TCAo*)dcS@kinh zRM)Y5475WU9xA~X5K(RaQ9B`;Hf)_sjZhADBakl~UWp481^a`H=hTuotdpni#2th1 zmI&%4JUnywB2I_NAQ5)aGVUHj7+0{mB*wMGBh&O728cul$S*cpY*cu4UZC-=+f|+W z5;Sx_1U3_gX{oi|LQ)sw!d9cfP#xf`9mO1N2p8L`lB+|hBXE6TD2+6}C2%oVwTg-- zD@h3y^6E?4AG~N2hu1;^F)T~c@VE#R43$K&z1+?18L04=MGgep#W#pv`JPfj{r=Z; zp|(j%2dza0>T5I!lrgQ=H-RSWdtS(Ey72Us_I2vwv#p^S@kud~P?TtC_(K;#WSN?= zxJqh0(l0a}4a)b75+i=xwn7&Wik|sA34uZHa}=+)YMw@`HHpLH>wQs-G@d9=tawn! z64{=Ma%PkoyTP!Hpu{?qn0n~PHA854O{fg=s7X8nTLbl$|9xIAZI?1SM3jv0pwaY0;42LN9;ogx{$GA|HZANqR<=284BSRpWlM-kKM!Th$B{7KzT^i6ao?RblMw^nd&N2eLJV z1?EdMU?EIe7@x;x%C8X@^jER*^pmapT|Du--LyS~k!(%8p3K1ZUUh)T>sz1cL)YUB zfuiWR<`Bkp^pNX{jD!~JLWBOBnku}e2!S|Xd&WR`{u8BUC?gm-i9!S#YaNYOiENe# zd_|A$@CKk!Yxt-JPJu`-s+gMiXe|iKniTj*6k#ozLLWIcN}!g3=>0R1UN?_>`Y}9X z9D%;RCgPA`-ib#U!ANDIxZ%)PI}}sn-iC@e1wdw90=!Cmn62)u13j4mw>G30;okkL zJ5d?i-UA~tFj{^VJx5#*Bn{N zw0cKOS#|NS3?o}F`dFi|cxE3p2RXNHujx`Nf%BnhP-+kr8*OF+zlkGI9Nl=3mZ*n= zeViEGa4NRkVNo|4$Xw|XXM?>*RW$xuOAn$jz}sP5+s2e*YT+?3>aqS(r(^u->IylI z3&Pf-wjG=SGA_2;iXEwhQsxaTBD57aCVHkEjsqhjQEPb1m2&Vs!69z&#Z+N!S9|R& zWP-TcmdL>Vtd;-KNqk2ZnoVMA_m}G0!y{ge8vOla#ElB0_1K`1k`NdWAuv5KS#&&! z{VC-57Y?T*#ME1y;(@#EsI-dST5lsTaW3d6k-4C-c4l)k8@o%2MOTPSHJUEP+n-0D z$^UL-|5q65JsqVux=>L^&NXbj&bnwO)I{qx)uKijMCNKS%1?iQtBiGuVO)yEsc=qq zBWrHQ?-!1VK0Ee8Un_=P^tc@;L;DEazA;gc=0tkh^HXemndO?yOr~c*{w>p4;TH~? z9jJ4XJPhYjOQD0fN>?cnMSP?NqB%1h2%jIm%#x$K8^%JSm>Uk-oOM=KGl4s-epa3ISnHtQIrfQ3pTBRrde0AnED7)8{v>n-+&zS zeR_G8hP1A`xu%rhuD!_CQdB9$)ctW`Br;KZY7paV%L4C;m)21xw3GLu*DK>QahIntN>B@42jMFB#i+F)5@q4Jb zzXi6 zLj)GFHNMWY0NA# zU5mjdynz^AG_5g^RWb-lBC-UEMSfnvkXn}^w(GfA-NeK2Sj){)W1~lU{-T{EL3+NF z!2L%i*~5^&3%N;IM79fR96gPQA<>kOZ@@|L1q|{zG_=SxX=q*PgyHuxzW1^VJ$&@6 zLkw)k`rWIdNUb9byrlDRrV6i_j(>w~`;l_tVb@(Wi>uyVrUok#hvq>F&{`|CIdtAu;WJIRkUJs*D$F@m9EFMaFMP2V_LPi6p zJ~q$4EzSdxrS`gdKDP>G5Sb^@)N^b(adAA8H0;Zil&qJOCqTjjBohk-HH->o#~&zz znD}YI#AzmU&KUYK#f>83lTN1`zYB*?1l6d(f^$_xTH z1ZY(A%k>G4D@Pv$smVmGEfNPxMvhKsi;Eb$rKx>`BEy$e}N(Y!Ay0`xjB&zlsYnP8_dBP%*T`H;G{c(I4J|04_3`2 z1f2Qo=m~+m(n03@u9#IlQ}LHceZh%M3@IqF<T97`{**-etiTB$EpXwooIB-%M;BsH;Owx~CQC?oRfm}bp~Hn?SC_L)hj zzldTYNEE-KeWUgw4B;@X9}GO80nnPwW;UtgqOQLPighLdnao1b1uTVvT|@sU{%_hj zDt0!+E*yl{uEoO#9tr;r+21tC~ z-KD2zz`rxBSvJH0K&Fu9uhg||)LT47D_L?3WJxAg6gjV>JuX~W&*=2f%z!_$O;u zJ2NuTP8h_KvGC4`N}f`fBS(%LIdbI4kt0Wr96562$dMyQjvP5gkN*P-WECc-Cnc5u O0000?#OMCTr$@(154?S{cg(?Amd z_?*ak`iv0(5Zl(huW9-cxH-L@eC>PNpPhWDu4a<)@V#3saV)hRZ!;gkf?sA_Ws*|t z;EiicT=W4Z5|-)mM)O$T(5^!#NPc@C>5fU$MJZQ$X2Y@$lQ4~cZX8^?b^H9{CSJYN zX*4^RdZKt~*64|q?ckoi5zaXzdn+<=hb{oF9s=^vx%;sxgnLri--5QwH7dW$P&hmI{h&XonRip6#-`A<^uk1O8 z?QG1n1Z-al2Vuhik*}SMelI&-xV2_tXhb4jha|zvB+M%1H*2KPX<{NBdswYno^&7( z#ERXo96E}m z5Wfe!%Izw3h;{Wjo{j~NuIJ_@{qS0@o?&;0SWpPi9(;ve+Gn6X;4dW=9& z+R}-Iudlh*k{;Rdhjoj;R4);i437tJu1&WDctpY>rV|{D|5oGF$mOJQ`^3J)kUin7 zlI?b~J7uiG%I4aiy3$9tufO~i&a}tO_)7P_<3*UUi(T-3+AH#A<8Ey}MSpGIqA`Je zyhBJZcrOXW1AH$?o3%vj0H;p1Za>}CL!R#REY8IUrdu4*)+oxv#6$rw72Kg?yAln` zHd3dUhMeH@`{hxC@&&D=A9pEXpaK=zU;-57!h#wU<#Lt}dp-CJ(5(~J`E^8sO?@bS zzKcUWRh_bGv{wt@ydTzik3@FL3tSUN{^?|5T_l90AI^qi1pfu-Ujc&jH0cc*iG$PX zTI#H_!L%2^g$dQBH%J8uAL$0{pK%c`gjUHXxGclR%V zz=O|t_v)AvHDVlTYZ#$n#8xEYEhz1AiD6$YPK`*ls;c&V52WEH4nRMk!EJup$)tRJ zPE7qccB(C?QP4TeQ7a6Heu(O_4O|sLu6&4jUPArFq`rbH?9`5A>ni@Ne@?dJva`l; z$Hz`#jtr#;1uD)MUS$VLX zKR3f;Ivoe#^4(t>?M{<0Q#QHSok<}dZhMT2epa|<9SC-MR+}YdMnL-kZ@-b?HjQJK z3=jU|Y?O_vw=}q2;PiNXpJ0dCXlsOH33XLOC?nV|+(6L{d9bmbP*Ix);QA-lNESx_ zR&DpFRrMeD3Br7S$!@5Jd(ur?Lw)LzZmB{agr68>Pw8p|N6ZpD=5*VAE$~OLs+5$JiQ!;!x^r{#Bgrz^5~G} z6V~c-K4Gi6m;e5h#v+3o!0hH3t48-e9l_w69r83>D&Yk$;fpX5 zq?N+N{{H?w%gU~fuZ3rFtTYe`)%8iT$wyxK)Hl3T9T`b~d_eSI%;chDGrke%xO2Rm ze;7H4H&R#iugwCb2FxH!FeaYML>+t)h)T(n-8eMK^$nn@}Q zEqn;q)rlF#MIdQl<_e(kur&oTiQ7FRZyzqxmxTf7CErwsYF9Z6gr4rq!yzXY#!ncy zP1Ex<6o!97*4C>qTVk8}!=JdK`3uX=$^KikoENlpq~tKUr&Wv7wGzM{+B0B`o1kyx5~@9 zP~+9En~vTcE)%PRCVuaz<)o_%@zX*3Y5V^(E5D)z)(&yR1i`ODsa9Mzu4-^@|NOPT_?ef+8#n8l32?+9;%O11@bDuM2R7F`Hd{Y~1B>%@6fTbVZgE0@5#SZ+3dIzW4I}7HWzZ?zpGUS{4O} z>@N+Z#zSMiTvMbMcS-&qhS8yoMT>$>6KF)T7E5Z`Zg({z0kf1Cp_{=w->;%DRLb_c z@`m~CuTUI<)<|W)SwLwAolJtVLaA1zakTxtt%Ghm*bimbR)q zwTs6@56q_eomyL&scE@%4)xROS+?&U1Ay-rVbvG*7^t)k#%9=sOoZ}LY052$FEWCu z-MO>E1O+T!fK?mI54}2BEdf`+A?&xdTSvji@+6NQOtw>}}5^rl+LjgJ>5PxAl|bZ8#n zI!$C~f*uWrNZzJnlLa9__~BYPqRK#Zw}Azbe!wXdy{w z$yNZ79CS1nb4as{J0Zo)nKd*dv?G0eeJ9Tv#eAN}Phe=^?arcUwCc%9p#gc!8f!mY z=Q2-&)6|86uP^8%->9F?aSYOg96_GdV7YxP2}VRt25#09qWzL7ZRL1w!p$c9!7dvyZ{39}^uToUl$B_=a|&sKs-# ziU@V=YsHzD_IBMc`ayK?ms|6SsC|bE<}}xCOtXWG*PI*N3Y27+#ST&dc#rczlzoEO z^g0i48!xEwY|#qK&m+U~dI_TZ)~N*blol26hLa0(H3N-z zM!YBx+TOK~@V<=F6m#tc7UnzN4$z}Da`ECJt_gszrARJ!Z#s!fpeM?m z@Ql~YE13bXoztDfP&b<|-o+wZ9|~u!%l>{M{@)K-2?e0%05@beprTuc;nrItj_W<) z57ar>QMPrZ?)YuDNkSh3FF->a+EG6FQ%*zm6rn}uJOM!YBMTtH^_;Q&DQV`xXyr;G$$!Ue$W-FP7_ z7wLmK2hYQx&2ATN^VT3*@wq?U0_6C8+tIo7N%6qhR^)<*iYGMTb*2;sY)>bf` z69_79;v7mhAd1(zc)I8wFUBOgBS+CC2+KqG>cn29fd>(J=qNuVPqX)45_7>e2k@`* z{7-{xf9<+Azw#pP*04pZcsvBjRGm*zlZfmYFK@wUsq*}MKploc6Qx^z&Lf1@@h3-z zQTwaPGh=>}Ds46E^nX;1i(&^9-sb*rz9OOUd_bxNj@M2=HyRVtO8zvwTsOD|BTdM1UyIO+IezCn*Jq+K>DDT4BpMIu4`6ufWhFrQ5l`Ceb?w|qqFOSojAgMt zGut?EAF>Ah%Iw>zm<`6ZW$qX>+z@GUOeDRV&Nz}%aipZQM|^oU;>Veu;pbBT?5{c(53I z5QXR6pKCS2Hj0rFiA2$oRIvUDi)aEhDvX%4=sfwOS8z=8*W#y^8bODJHrGoh-}O)0 z?TmVB(~?VESKb0I^xx@oP$2U4a_b*J2Ea`0nHw=~2WgU85D`0pMw{b3W*~dUm@u}F zr6%Ry$cEU`h}3=CYxcq6fH3eX5RC#O%6mF8I~5PFrI@$(U34(nHI(S6yJgTzCZRJmy38>xbzmmaA2k5`tLJ&$F-!Or-X?82`?qL^Z5>j8`&&eYC#0#gKio@a^ z5#I;$ul%aF-3!V2S#ky7wX)E2XU}~f)BAEnx$Dri-tq;+AfCT8!P?PFY)kv=LS98A z)?|Pvj<=0RZU=~yT;Kl;XEH!NWrp6O{Veg4cRLx(6hqYkQv5&0FKT~&GM6Fdai!_% zt%~zAX}?DzyoHzAY*1V9g<%aJ9Z)CQ5g*M8Rfu3uz5si#Bj%V5{bJ2Me=`MIbc9aD zryNUMS`f8l;t8+Flck309(X#9zV93?mAoqRF7Af$y~bX z$1C&pcREdV<~lYb1~3zKVv)f%4o)EOIj}(jFp4(AcZI^%bnx4}p*_?_+2s`f`il=W zw-y?l6K}5;>r1ps@%X(D^eZ7xpnagKLXngv_2eYm=EPVK5&UQ%b;JGI-Quk`gT zKg=xL2E_p)?@e*M*(@FydH-A(J7V}UkgBug@rS1?SOROqFKS~UU-Xh+{xKY>s2pm! z+y8Ij?*q3HvQ(K2Uw(_RejW$;j+?pG``?;vuA8sg zZ#6o~ZIM$0s>VDb}3^H&CX^?0QC3Er%BR~CVKVt z8!1lSdE9q-u)3Rfpj~#M1Fxn+`@y1Tk@hBGJSzE;25X7PthvP0Vl{s;QM*YkiAL}8 zTGabSk)xS;7Etla+%p$p?4CfZ3dTFrucsxid_QjGs9R0OB{VU_mB8fg-XGyjv2xMs zBnRMo+(ONn<~X8@ZIr%^`dD%o`B_A4v;NNNeXO?s80c%n z8SqGj!qWkU*vls&vAy_#wBuVBA*VM!38&2NBzmV!TzQ-%T9XOps$iO5BB1!ODvYpC z3(9^N4`#sqEKM=;Ki0RKat4bQn~KCfS|2{VZ&MjVF% z1K}6y>fM-<7JEitk5wGZ$vd#Q=UR#T3EGs{W=T)ZBW39y2&{&SEE!&iRUFg(?8wMd z%h!x9XvN~gF88inn95@d@mYiqoW5=pkutJGK5nQVf-cRC75n|Ei&rC6h%x)gb^2*z zF7Ki14gc>JMX<=yADFvlhP#KW@Mn~W`+Uqd%Gj%YHHr3-Ml>D1m{Z9V@KFCLME*h*p_< zlMcSuC&<4~g|^-bg>aK*99xE^sO%JNvB=!A&P9WFmKK{|EGJw@QFn!1mHbcgf+te= zcBN}!?NTUSFuKMO)pLS2O?2FSQR}RgtX_o7fb6I|NLMWko=F|ncx48%`(gUafVcz6 z%Isto_5Pk&ate<(T+2by^YC#{sA6fuH`y9wc_L-|-F)a&b=pM=?H;Bad~d((z;aOK zIEt8|z#laToyy|${a=&)8L2lSC9SE}N9B>lRj$qEa41 z%a{6}ji6h+`6mz0i`W}$JZe7+Y+aR!aXgS2obNYr;2+*aVx_G)p3OY`0$C@_2FwsC z!D91-LNCWSY>KJv&hA=JRkY)yPh@b9#)3ItpRj#gYKcH2t~SBd!=kk3_41Q#YIA#N zE$OgzkpX`ax;JLfu;@x9AruOd_mG#Q0Z1)$r z4-z;q91YTiQsMzN?M0hA1TU?OeBRm{zjOdqG~ zUCQ|8_AHZ`qGp!W-KTASe)bW@O*7F!En<;@0|AWi=wSYeROCJO?KT6c_Xbtjns=XL zUv4IZ+!(C3sd3fz)4xYe;Hi7OcG@C`m(TS-rZgBLX5F%*{$1}!{$L{xYBlG*b-M{_BU(2{qPw%glTzedSuP02`U3j}$Mf0d}VIl4CNtNO0 zZsWT~j)CeekIaR6Q|tDS;3aE}t`%HgZ8n>D=8VxKd=pfErn;4N)E=FnduE+~$Lv0- z`PldPszEclajG)ND8ouoEV0VJ=+sKIoz2Fq-TGoTTcYDEqOY%LuQT%%McD?jVu!Wd zLdh+vEgKL*Qm=z;GCp6c{gEI(+&7!Nz`~}`dWhOmo&M+Q^rJ23#Yx{^gN^X$yLKgZ zjjK|Mvq?eJd-{y^Q+59O$h~iuo*slh&9icEYE6%&8rQY4eA(RPYr@R>lG+w$6lZp; zT73Tjzg$%FMt-e%O$?F)Q)8Ro>2rMLd5-4s2uifJn2r1SfPfx-*dC19>6>6r|x2VGV&p=`rPcm=vXI#qC&eAlbaL*@8Nkb^P#6H>2z`;|aNI**mZhqK} zm|uOUW}v9wZ1~mxyGtN;>tq|1vgnF*bk1`75>an8!zhzK)=5pu(f1FtcsG%LhZuq! zT&*)wszj8@FYT^uvvO2`6tB-Z^tT7;3C`pf;h zehaF)6>)gBndzdbKXEU7!9hFH>jETL^(18EK&OA~DrArl-xqMVTniHTN1^qVGA{6D z6~^v|XqwowTqbwRo*b3Vfa$tQl+0>b1**{SE3!cdUzcFSNAEtnfh6)3x%p%P3N4ch z)dv2ItA)rUSMj7ndQKDvbrJLIu?D?Yi#z{(lpt>Q*4fC|P4(H48oXUrLxMHA(SX-}w@J!Zn|Y&XSHF3@&- zGgJvo=}4>R7vKJ;NUfV)cKB1xA66eCTcS&-CB%h3kv?8dc)rOMzPBbE`0+5{=hLq> z6i$oxpU(^zC{Mcst3)KLxNP!>=*l2!`NI1rTTv?ESRcXyAAG%{BWa1Ri3fELM!m|dewCb(QIaCt<}O4Y z5AtxpG`XCCNdlh;2CWUc?qzW-LXHPU8v%o|cR)$6h9!3Qd-tLnZEUtFl}bDdk(u5n zNM&)qp?BqGM(xXko_9Q_bjb5w7RN-pfyBB1zsZEHV%_ET}I;lk=Z*vM8OK^S%eWwQt!iH}q#WidfatQ6IIK z;Nx~Gy=7n~NDkp?J*=}uB}s;}VfkAIGW}Yg+wDQPRfn>{M{IVzcR)I7`IWCiY@%|p zw*&boMnzk*X~TD$yN<7cCz;Ru_qe%9nQsEWOK+Qu_ifoyeynjidXlVn(d(x&B^5Vu z1^IGEeiiEzJ7b?@RC)N58q_*{23gOVH&6V99Caz>-`)wi_V)GvOk$!U@a|H`DSxBn zwiR2v+O4_1>nd-8lBLUi+g`lr*00jBeGtubV35^LaJXO%wVAA4rJ^DVAG4h3_1ERa zC#3H1qc_Yjm37h_)8I8Z{QQ`B@+*$vCh`pxkR|xe`HyRI>m1(YEnVGG-DH2`WG4HvgQ((t%J<&cVceIk$EuCcsaYiM1Cn8edT5U>35lMk z8&Kd0A#HGk*QTW|-yhP;dYZu&7(x2zetja1Q=0TW6{&wOuN1FbODvqdwI5kId$8X% zliKOOxl+7%Ibz^hg3G?pvG+zsonJKR%oOc9(xYL7F1#AtHBDFGH}2^&I~p*4+3N|5 zx7ZWJ!O*W_1EFg(3!4wGlcv*lcS9T##;9iCAY?*#mk@I5iPhdW!Cv2lVy&!lljR2i zR*zm-{H7ad*;?w>J<%IuOVVpr;o`PkYyX`cw)^mIpJC3SCSe~jxkq*QWAN{0f3+3N zYoB?L)2wm7$iqtMK+h)m(Cm8cj()huF*2dTg-$j>t0<{~36cA7Q2+zHl{NI8!sUSFVj^r~rYdRac*uHjh?e z>co3_H19|46ZOpfM|KHq)Wbc&fZxjTSprxtYKJe1xI70C-pwY z^R=D)kLKg-t?l4G4~5En9HmV#OVNeQ3-`Sq(|*5EtQ=#zKna+Z3lW-d37Tyl5+V$5{idV={-p)JE(P#Z5lE)9LomKfe!tZ$|2H`Q1ZxgQgtqd#cd^I-guk44c}E zL>sEb8wcbrJk+0H{ppu{m9ziS8^!VO`4?6<2*|I%%NaM-H@DXnV;(3I%^dfnL(}{f zhWB>YMjNQ_raYt*xA2I0)f?k43ymG?s2{}&#Z+w0EQl!${5@ElF z%?zbaY2SiB5qy%>?X}0kx;hP9J^#Ut%oH88zx`S1S|1_Ks-1@|h0O!z znw9y_eYmHO$%Q_GKeBmg0e5jP#zAL^BgV!3$nnScShHfN)NVLy%;lMP3(P%30nc}j z_EVtx$CaqYD;9Q&AI@L1wZ*8N?#~#E%Y@9VD7)IK&L)U^5DZ)&2C7E;dCS$DdInC| z|7tcPe3IMea`!Rw1{Z3XqU83yPt%a#o4@nEai?Zb+G~KYH_BMl{~7(s3a9XNT9vROXK=!aIzZP+`QV}_FJU}1KwL z>cwrwtRTf>4RKts*3}>$?5GP-Zc!;WdAm1WZB0yCWAjQcQNJv+LHACdlc_Vr(n6dB znh88D5w^3Rea<=2t;JQV$Y~TReOBBTfQz%we<5nfAH2wx}|OgqpMYGwdyYGssFS{PM4ZuKiT|O2^;yQAS2Lt(YA-z+=I8i_aRL z??&Y(-IYVSi!Hch9pN@CJOnU&7Fs1;xDHK|TwP|krwL)+RpuG_tBzya+G6R0r3piG z_sQjoB$K@8SoKCX)SSV`lbK+jT~gYf5}VZ(|F%|PyZHVD74ElphZ67R;42QEKW1qz_!nvG5Y48|jTTr}N>3qYk`nlL22V`Z5oeU;Zgjvk8p*!@(VuR-2Bz11AWgv8+aw!*UYlbUV$;2f;)OW zxpUR6z3jpky;Oca1$FgyK@a(X>Gvz`>T>8eb68t3^~}owbdnqh?17Hbs^fqmB97x3 zBfO#vsIBfWH(eZ9h%46hYnCegbN@*lesgGXlNnwiR!H8&u&V4jC!b!=Xu)6o-~Ipi dZ+6Gm6FW6O(T;rJ>xsWH-3NyEEAPP~{vSG0O3wfQ diff --git a/docs/static/index/github.png b/docs/static/index/github.png deleted file mode 100644 index 5be61967d65789f8423db2a2ead7f10ad5d8936e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2529 zcmV<72_E)|P)VoOAlzSq06- z4~*LyJB`DP>l)WL_BXC>{KmM%IN$iW@oi(b@oD2z#sz2+RzYLI_O~>SHg09y$oM1F z1}f}X;{(QrjE`a^#6aUw#*d848^k%r3C0pu0*o@=W&9G2Q;3q&Z?f^%s3Yn%l{PR= zH2&ZC!-VXIaBIe!Wqty;rYwegNUk+&8Gmm4g>ggUNVv5q1}f=l;|azdv>4wv9*Xz~ z;loZf9%Eb&wS$ZGIOBArM-b&2y?*B6cg7jW@8B%sk;b2){wNs_f!ncS_x}}|j4{S} zIURbl-08K^cyOEF8~F<2;=c=;gp+bQK^x-;*kK?>CN!r%((u*H43U894N*V4D5h z%?9AR>Gwss)1Mh%GrnYe9_}S^3haioRr@uZS3lMJZF}P_=sne@m;3TH3W8?7zJ-$m za^pmXMqg;b^KG4P5P#cwTpYQ!2$eR>!GZpUcGj^dh?4OuseA`|YdXVaB6p5)HRH~3 zBW`(v`7&JKYYpQ?a&1zezuc-HLZ!k}4(bYAAr-$we$3-GksT-3cSqcxsg?ADbn!ey zWF{kwU!rn!U7|v2n01yP#>c71cjH%+@**i;12@_@4LM}FHc-)AJGhh4uX+b=ig8`x zH0x*l3EbMlb!fP8vvOUw&yjHoTjraIQ@TZuJ)%U(Bh4>(N`2-H0!A zlSTf}h%6UaK4dMSP1a!`D(0Aum%iK@k>%8zgUAT;*NqN1yT!*D^Tb30mx2c$1F zIW|tz8G_1Z*uoc>AQxvMjhRyCJ-Dr~RiIP*F?#puu>$p;mO2ysi&Ny{F^R_hhDuHE zh1P@HuJ)@dQRftqe=xhiMMn*u5sAjG6#3WC9^mGo7Wc)8Iwg_kQO*kcZnRwZDA5q( zAL_=BLHmHgZ@+JJK4S|p?KGW&+i_RgLI;QeJW7)mXrrk zF$3hscM-=xDbGRsL0|5V6Y_JVygn793)82FL!H2RXg|!Ca>@!;`L(GSbdw>#POk&JJEj74Wc_C zuQ@$SsTkE}Kg98}ls8BFK~0Jm6Y@G37Em#|tEBQ-GQTPV&t?XreL$~&lvie7Q%0-y ze<2m~zWmsMI5=Xa$ZtoWeQ=HCwTJ)ggx(I+`vn@JU7#6NjBBic)`9na0loV~=WsxQy6O}&L19Gzc!^vXSfH`7 zsEmCu9MOuTK;HAJKz~wkC1NltBY%Uki_gi$@hFC=fzI53^p;;w^ef*}`;pp-GVCk$ zp7bxCg`6l>0wuTuGwoEj@gHbB7@vJ9=l-RFZ9FV>FYqsJEf;z2QZ(z5vDOa)og;Y? zxe4kI=EmFzXR+!%=c0E~Qw-c_G|6CFUx5D7d=WSh#lan^8)4;;bnE$T0gqa1Nf4xE zx+&ZgeGW|t{AxtJUGrvt2S!zF(M3gOV%y+pPE<55s zs04HS&VxHPeHF#NO}Gm2lTZa;(CdgyqYNVhH`C-{Kb@5w-XqWs6MMRmevpiz1joNu zQJ)dV9Z)^?478Pd{PPGZS7?7^93E(FQchbt!i|jBLtTe_+?lBRWcePohCOt&Oo<6| zQPag}fA9J(Ih|wzQJ(!m@B0LVcaHwZ2W^Q^QXwb0A9lQ@hZ8$E_*74OeKqs+V8sK@2 z8f&KpZbVyx=@nd?xV7IOXs22%kTyo4dN6LJnV+<;)~U839^5aJhL}?_%#)!VezqPSwZQ9X3@c2!Y#IKp<-QRoA> z1z(Hs!}*Ct^rp{6{)bepQcNepO1T_$0DXsPfyOyBHx*gVSLWZ+l;^h33@iPxsu_ym z3#CmqKhW=KsAJ5C)8)RvO?dQje4=g_nyVPTNPcq=O0mBe)%jC}zy?Djhoo&#%Bk0d z<|(Gbcn`vTWZGFHzUW;64#CTFZp^85Y@kjT8m}1oOFXFvO&uC=SEETGSK23`AW`<< z{Ny(aLU*BYisA7v9otzsePOtQ+3HP=LvMw6c!AfL0l6pYub4gHtk}0A_vf|IXc1^3 zAB*%BU2nz2$f?7iTajavT-0lb(cGR0_eU>M;BLXc{lX|dy<8p^%Hf>%_aFL7jt<6&_{~v6veBFI zrfrv%H(|8olR42e!FJ@&_u&i<+y%|1@$mun1!Q9&iRSPM@Qv#)5g rL?-%NTz`08)dLsjxf_+)Tn)pwu*Uom1q-y|?1_|jHWN^q~ z9J=rEeSde|b^p76-&t!0&OZC>v(MS*dA^_LJ8Kg4^)x6*?~;N*APUW=YK9;Xjt}sz zAtnS`5(Ta8fIlQYPtE;7AO@~~ZyZg-2N<9c$KOyx1ynQ6vJGtDyDIA{gFs)QWLNeC zAdu*=rkb)b1ZRK6+M8wWJvXMjt)SG#BPr8Enm>7fYqKBc_k6t+VYGDAQhoseVeZcx z5U18^=U~dp9m1rAdsUx0^VNS8;|kOdCUdzdM(IwhqMwekt#&?@G-a9Kc*63Pm1prm zwi+MoY1qCv%2wDR!%ecPO3&o|=a7h)yZj7->A!wGD~i}DoCeX*(7(egPMGPRJ-h3( zxb4u<6)isATrT(FI|KK?!0Yz%0j1k3>ANf9<7=aVkLi>t)M&m~$)7Lg_sY{*4EM}j zFx3cjF1WqEW1hoc8@8613_F!Xwff5&0(4AAwj>xIO%W0(}lVE!N?nKbFVS`Q0jJy}#d8<{jnz!m&DZH4kQqZxF>Qy)r%R-%OK(9{dEhZTESn;AK~WT2<`ed{JT( z3cZ#-{!Lui0kb4Jc&YY9yZBv%)fz@#(Jw57lAi8pIelW6k`qH=qqG{H-bgA6=O5i| zZI>>ujn*u!aJz0fTDA_DknxF=txd<1{~pnSWkS%;O#6>Wzjah-2w9|V?4YLWbfvI zvbDqe;**VaY*4q5XrE3L(LB@)7q6q#Z?|>BJo9{mbD+HXz&GP$IfjYChrRJ_$$?B5 z3M)~O$%NqgHjNFkTv&ykTHbiQ`wOD_3KkY5gd~0m>kj<>^08OxBLffr6la#^Po}`) z%M|NtUw4!-AlPU5KJ|fwah5{SEm-ux^j0g8rUG0_r({)Of=yUa=#y~Es;7;CSg@C6 z`^$nk>`ULJMLz^_$A^xWuYU$6_R<(XHob=Y^w+k2cOE?EgWsim;Tr&zi%YLoGky)T`|0C`DQ-L@+V25^yz# zY_zpw*P$+MEG4;{Ta3Lm`IFraQI=6SUL>x81S!ynRCE_f05~{P`F@6l^$#r3fx~eI zm9p_Ga-WeIFJVahn-)eST_XPJaO3IMkbT$H+QiSm!Ig{U-7yVYM6Gv=K%Ui465o-+;8J>Y&cHKSR*dv0;q0CW>$&x9{?u{JL8#xH1n-AHFw}8jB>7{ws(Z;c?2k_Xd4T-?{;Xjrx$P)Ky#HxxWcc9!jZMaseTa8 z#?q|cxtxI{6C&W*l&AZGKdo~Tj^1fvgQs9lH?1=i$XDtsO{O8}&B*hQr`ytMnSzAU z6yH{w8uwZgu>+v;ryLQfAo0HOf5zPo+xIzbj|jp z=BIZaXJBciStm!hXw&$cs=y8Bbu2nKaw28dOTL>3z1xatEvxf3Kg1`V^U2 zYBs8TqX&k^OPOHBhM*T=MDZ5~PEB*WC)%>&>Gpd($WmG2{510^YaVZLat&lJ|p<3|TCCEMNtz#4|GsmN$ z8cvP)gVqgFH~Q^CX5_E$AReu!Wd!JuKQFK_xtcogT0>5wRzGkx4q8>H)R8KaqxIzr zy5j;VW+Xiv#Pe49kN=26p#+dqW^OdD&Nsf8XaV{H&IWPjp zU?(b7o9O&vUD$8YUQ-mFNXYc+yuG?NV!6tsMW9;(nC233bFiSFk9WWG`DQPOXR%mB zA;`*JDFR_H5pgW)o`nlcn*c~iw&y=imO8S>=DoZuEq%y+K6lg~E&HxmX4mr5pKSE^ zg37fgF*yF=dD&8XC*rCA12yh6VIrp<#UAgA=hLFL1B!9e#G z&*~K>gmK7TEA5VT_U+QHkK!fXAmt-{Gy``~>j#N|SCQ8fa-m&+1>>4Y>IH5^8Fz_p z@pnQ688hVwA*uhQnUP!L8Gl7^nzEY#4$tX?5P0!=&YRVcl;`*s9hHo6%Mu-}wefiG zn1)Kifn~%FeVwC2kMc|JUq%>Js_P5LR!qpv`)|SbumNWWPi*sGHbI!;n@(*ue>#g@ z$Cmn6>qt?3Ar}2I2JU;hu@^6|M_dlNi_Urvue+?G!&tJ-&$g_ezZh3OMSkDg`K4?d zaOyDP$!qmJ`|#P&7Tb0JYllQg*aaW2I5{Ty>BNfqX(66#{n~*yEyA>304}%dld2@v zE+I7<`3>vhu4Q{&eoPA|VBX*zoiX$Iaefj&lqpLH87@T;qPN=c1;g8#S#=MX5L(b5 zta48_Uf}~e+j#hBFCUg`oop>r-L>GRofb3d_w&AtVmM(YA0gzemrknAFZ~Q3pP&ni zRCtO12bU-Xo(%jT;YyPoQqOBAyBznljTPxvivWp04%oTKF?1et{#3mnsFU6tmupDkoJ6imcQjK_&-3vugyepV7@WjvFbUj+AUT1Nq@1t|M$1go(!F8YT zEbZOVG7$csgnY-3YCe3Ux%?YY2ZqLLZ^zety;|{rlz#?|V7XeFg#c~a;-40`%Jzn# z`^#GdA|WiVHK>o44RctOwfJOli~JER6YW6IS!S>G%UlcO<1^D9Z3wPFyYcQO;3!53 zM4IX6?G>bJMvc5!PL$Fv%XtlgHeRimeleNqv2b=J$0+LPkg=FPdV;w52FGjA_RLHZ z7@0GHixRflQ!b`(SvjPwL0QijfMo4hFz3y?IoM66*XBC1D3|M+3-P+ZbBLdg{)oy99>M=Z8=2CJ-=CK7t;EI!%-gxsOHa~8bNFL zZBY5oq1gud0d9fJBjAptU(XcP=z(k~VBFpbLXCqvsa>n<#Yxav?gx&?ixVBTW`M2? z8jkOfmYNzA-B7M*P^l;!QEx>U=KK#S7*hZp{A|*0+xje=lhFGG&o2=1&%1+JO#m(A zBd#<-rcvjJ)VigG{%=aCkOlOjXC~m&Zvi3WdT+i zxJ>yA-;PjqF20cjCP+F;hIQ}e^;BB@CB+SZg4H%Glo}X^#A9cGYmltw^E7V!zZ0g5 z9c}EjmH>%4YqT@s>JP=2-G~m}#i2j+aQIq%%)5;FB#$>NL%iaG$jN<*Nz*L)zgB>4FZboM6(P;Qh z7{6iZdE}AZpGMM9uLfXb`pn~|MynebB`~vC=dYgo_4j@K6@z5|?aG;LYBX5|Xq4QC z1Dm_pls&T6&I_RX*}J_J>6T!+`E8dzm-%k&Ayt; zUhoHi4<_#FL|zhKq!M&nQnB4!_464Whh^;b+3$fb+8%Jwgu#Y?yvPaiuwp-$+WNY? zz0Eaqb&|}FBT2MlAK0yYy&l7KIpx2qB*o0Eeu{8hwVZi^x~s9FCP9rSuWFc+tnl|y z^472u8!PhvA_SV_e$d3=Jnh6b;^s5sZ-Ytz52oYIs{?K!vGB_*4YVO1FVgIQG_nCJ6-LI~~; zAnip*SnV&2Sj1MSFd;Xb&C#ex_1loP734&eL4mS)3C zaL;1QJY((`PN~nr*3VOf^*`ec_}W*-LRVuuXQHG1PvCog+iM)Ae>|5~ zk_4M;d@OF^1Nt9}mZH?dlf6$1F4SHf-71w7N#wLq&TRS4^2G?~NFM#HLkCR0|J)O- z8ZCF#sf*(!qm5^^&DaNMxuUpedrJaog;knnnp7J^^%j|7mv)a+xd!iQ(}K}#0lN0= zp9H*@ZPR5%)b2+9TE2Khj+rkN%JcuJ@JO^hgr44T0R$8?)jmkER3?_#ybHeZniTtp zHzmHc|2U%L)l;IgmkK+IO)GO^aL07m<#)~oZU2f7!_dcOYaWfq<`9!xKU}9&^gv*4 zQTo$bH>C*GwGUq$Ps?DTKj;48$D9eJ)pNF6P!6p9WaaKFU9(R@+wt_c^H9h5|4^)1 z8+cAij^C)t<7nA!>Y%kGR!9Sp&Kq9kOgQiVdANtwA?lPk`l^0IX&vG<+Pe6>XJnUi zE3*}Ry8EvEqRo7*33cCT3lwxjRyype22+g+RcNl!9ye_SPQ(6-2oRlreNjMdTz}N2 zm;XFtLk4$kwxeWiNOx4o8rQQI1ZLYBaDHG?c9^MUeJ5*%??M6dss3og-nw{gwkUPV zpX!}7Jj>$Eh7)$)g%+{b8U#cI4lwTW?m!lLfb4k{5NNF3Lak8=Kt)TafvD_$TF!## z`Suv~=6+z%<0S_+^=}tJ)n%VIa4&uT;YgLVNoGVxki`c|9}+Y-xqWOeEc7{fF+2Si z@I<2!^OJe#QinjKz2U2dWqLq__4?|Jajmx9;i8cgxm1`Pfk>%Docl4&`9_*++pO~O zaYsTU&6VJ_t=5k*mhbg|2^&t*w7f&OOt4b*@!)7*{_Ra}Yo^6N*SwyI;F+*c4?zpH z>3PDjK^bBsKB`i(6!_AiUU&ej9nJ6wVdctWsnFFS<*bs~N4Mj=gQ1~btO)O|v^+IJ zJ_+2~Bj7Ng>o%MBP<_}qSW2MJ)BV!(@NF{k4nkA|`4@M&hyh~vs**x18prGTk4CNS zCo4;up-YQnOC7VNLcZx=??B=F&13GSKuRR>3W%E?g}^98J3XssxOQ8_eG2BjAXyNq z6oedv(&YqY$vyAHVx}qi#TQdKw(GA_V4-4RBA_cwgRa4oEAazC)FE_0Az%V5EAEbw80XnK^?fXNDv-l7@st`dB9B{ zhJWAV&_ZtH@1hj+XlK2w z2Bu8juLmir8qmGOM~9)r8v1Z4(&!6+(f>0eju$+?D23=#O=u|h?e4&BLJq`cy={mL z7DVBK8(}9z5=WCOQaH@%;)5jyRU^-DFays!PZSeA11nRxxKAJ*bVEGLCuceNv9q|( zQl+p$iwVKdZdMfrxCnvO#rH#qzbS{Hv#27!x4q#@0(_zWf(;iORxmgK5~lv&IAf8f ztRhSO?;uhJBxCer{f^x7@6ckzD@^wO zkKM6g7d5S**<76kv&`?fz2^!k;OtQd zIeVDQa>F-^`0(nG(5=}kikB3?C%qji@OQuuFjzW7bVklyFDnO7$?vU1R%~BV{g0vX zU$xa0hF?yZXaOXnA9_-JvwdJ*^|J|pCg$`ur@h45?KWiPQ`_1fr;fVd@!x`kG62T# z3`ZR;m&kT>7G$YnqXFLdVzQ+OfF9lPZmT1S8+~4e?d=%P>dhmye~mM*loG5bSXR3J z;}^7D2G8;#?E$}n?%(Zuplh#${>3Nd4O$7*KL$XRTyx5QU~}J0T(KvjqZpV6iKedD z$tTq49GAiI(FWki%<|C#X2M&|5cF4`B8`nT)@b^GwA%2$BYOjYVXG%my1IE_KFehg ze0>&k*}-*gSgjun*b{L-&z9M}XWm0B>=t(dPd5Hzc(9!Uit>MWK?^Xn{>R4v46%7= z4mDn31=T-~6)iY|YHIuhm?Vq6^MGNiSq)?k?bkSZbo0IibM@;@jj@3BG|aC(!8+IJ z>Fyo=_F}An9E)ooW%Owis1!}zZaHtR0D+F}0o)A01*#3BZ44)RtkB}=`B4O3X=u3g zr_lr0U~8Uvjg9&+xnGZ8=|h$-Z=K#2j_|j4-=GfL z#0<#2@vb7Qs>5%z%^sWDMOpTqFO3M7EGgi8`lGBl7Gw|^e7lF8T#3A%wg2R!QZ6&$ zu9B)4ug5414ZDnFAu#X3G1s{MHQ7XoHCvaat^3)lanP&hru^|g>icX|NXYAY<(HJD zXLx@%rOj!d#m@uRuB8h5MH$6T;(9`1C+$kpY4sJEYSk+yfzUz$a-Yq|V_Q<0t~@R| zvN2s4bbq#`nRNBDia^vam0^S2%38#i!~Jc_LD`NdK}P&@XI7+;KyZfusn2#RYIeOa z-09UzuiRUo&C6=e3x*rZi#KhELp%P14hL3OjhpEsmO+aSA2OBqZhG6)k=EeW$cP+P zoEip}8K49`<$6%}&+BlbzgDY5`cL|mPgMEW`~J2nKGXt2k}-J>%DHx5Y7GQl_8(wx z)KXo4bgeD(`J&Kn!ul~e26wm}Jn<0T*?b%l0C6Y>9*(ys8>_*d0vq_D=->8HjJX$8 zE=FrEZE1l@FyAO`xYaL((@Nl5ftC;AiXUr|Crgb)&^6{ssC!Z1hL zm18B1yMZn>L#29GUPM~Etkfn3PGbQ6Bm9})jjnoIgGw> zrtJ@1twRYt1nk~pK$^*w#vc-#b)Z|HIRUl2$+wsB^;g!_|1QSQAu6Yk(r;x$wl;rn zV2rTk{N-zN;_S`t&yg=TUl9?P5u|pAtq^80;C0~Y%<;wH%0Iv9LK&w=x95DCru-x% zQyYpFFPyNB0J7%a=#KTR`*SZjdK`-Yb456^*@POI;}3f^xOqVT9aQ_FbefVD|EvQK zr85QK{BRXewBEBWbh{eZHrM|Ic*p|irp7)Zh-fHU`rmhC7L3)&P1T#&Q^40Wvnq@T zO8{8V5=!b`K4k&$edeX_r(5r)uhB+@49`8ueX!&Wo}O<&!q!0S?W{HCe@-Qf`{$5< zBf%?_Oq_yJQDVY?GG+MjzqB3z7W!1K8|{2Pv11GC@fjHi?2>lH+T3{Yvv!Er3)tTJ z&F@o|3d!_s9?Vc_Aih=aU;Gv1#LNIrWFD-D!k*UOo4i<$j$UyO-+WlH`K8I6!u0sA%WLYHf-g?9nIgx zzj+H2c;$MwLZ9SK;?vMLtHz^S#89LOd}V~iS1hk(H0a#vJ~I(YZw@&{jV${zs?kp| z9Tm7a+N*n{ZaB#bi2YUYzUFMTd+=z3t$6;`-Pi3GTint$?5fI;=JdT#v_b?)oK8N2 zvf9_IG8K~EOXl#xIOf9(>CYN#yOGBJBo45N9)lEl12uXKOq@| zU|yi6^W0-q*gGj%vWHkcRw6X&ar|nOHgaJ5$MK0oRJ|13nQAN8S&!Ct#*>kHnFp$KH>uevzA9=*OX=GXK23Qatna(&hqif$hfd7*-)p2UBFcG?Weat*+ z#xG)@50*%pS~S_mb<`DtXbfJ{J9Cx2X->}7^$?&(`%d8iO^@ta?T!RP3tL1=L_!NI z*cCrSS*|h|bq~zP4H#0KuQNG!-ozEUW#yCo&is*7%lpp!!>HSlkaxBlr~S;%6+i#70}m3MgZ3S#bHdxKAG1{JC@ zar$6j*8N!+`R(0I&dFof3>=h7#EnS&{WWY+EUHJ2=VD?j{K`-PJWB}i;jk2GR`~KJ z=w9}MU(cYrpAJ}ft0ZxqkMkh06smo3_WH(OF5$+WZVZv2eY5&Nj!chzLGBClnXr>@ zW4cagxv>5{utfc|V;CVbKBQ<@uXo$>0wQ~MTo>pJxxNP797{3Ym^8%)7yb0waWn)! zV_$8hHj8o81K-Lc4=2g9rUmFWC!~X)P zgK0=AkV>JnBo%L7oi%Kx4ItF%_fa04>TZXDj1kYuOeIe9=LY%_2lIdLj;c@J zFyy`~^RLaev1$UmIAyD|^{j=!dD>Mht{>$|*z@3fH|-mV(x{%dAgV535=kR*y)w@{ z-(pEc5A)uoSXQKbC?mySg5df-hlXWMt2`08T>7ql*MPCc24H+a$3*8{wZOM;m?j36 z@j6hwN<`wT)9c}P7>Fa9F#A=4wPzn6hFcM@IvDw3kqulO<#s)%chWnI(hnYGQLl+% zy?Sl1(Q-$JlW{aHLu8RQVBtPog^~|sbveo7t01mB^0Q(f-}DmLUBlktCY(MtAw%N7 z0Q9AIrEnfpcc3;zP;7I+#0^NK8HWVnH41u2>(~HjlTtborb;8^RiMj`@~;>b4jai& z3V8Bc=xv}LlF(gzM`lnE9{Pl=OjdTurI2n@@QaVKCh{(|!fBe$ue4xanm5vSc9_GL zt|(iIFrldum=P{`Xozga2r@%pQ?d*dJ49d-n-rGU5uYi=7{N%6moKsuT7?s@iG@7L zPsusA;eELL?VHOF&BqZ<#a$_%q8}pKJ9+albNq$cbwLSN@kS!5de}n6)t(=Oex1G7 zjy6%Z(z&@zn9n!oOLEs&!5%Koe%{{hE`kHcRs;O^nNHORgrJz5_s%u-oN}s^BlU?kC@P$R)6q; zv@6d9cSTV{?)Rz3?+(vvoAHN{AnH^3;hOyBO!EQbWJRl6DOY2$L>s28?%n_kE>4sm zVk3Qs*eb8+x6?-Xj!w_smgN%_o_EWI6yuY=t-1DFXvO|1%kZ(DT2AD1_#ZbFwH2}s z5`U>JO*3Szg94L(4GO{?ltt%uR2Qz7b!~VAI5@yWi3K*~V7} zdEzrSPlv(RHU@vKV{nEe3-kH9(0WRt4AaDY1cKc2k{b_@y~}^pD?MM|TgE8hX%!?R zYT%J9o4E_?-1Bu-xy<3^!PTts-JYLG28?l}Y{=Sn!FCBjRF=2IXGNsnss(VDXY^i} zj!7>#RM7N-DiuDe^pGLQc?I7|lt&|As;ryX zRNUDGl%_~3Lz^oPr(XGf-OKO$U#}S#b0ht+l*{*if5r`p@5}r+eP3E2e$iQl z4M2F7>f{A4CkJ$>hD~^86^7wk&2&$EC#P_C=Z!-L;`PDj46H$vi=4<(+b7>;d z#V;iNH-)Z$y==}i&b)DvBqJSSF9b}(ua7)CA`00HRUlJ(HcRGL(7SvBDvG`ae=j&9 z;qoIfOzlk(qhZ1$n26zim#3;&mTo0J%t+7~NOXE&erJyEij7KXav&d&M);;@Mxg}1 z(gi;R^Ujpl%)cgK6^AS9m*b1#nvsAbR?;Wyn(is(@x-IwlYB2;^cMBd$F&M%MRF4# zTH$sQv$j-0n={{ouOFo%gXWCDt^0PAnZ~Bjk4eaPaWC#@PGlsl!@LcbatkBZsz`3F zW5cW*o;Mt&j*D-PB~c@a6WNvAcsYgMwMVmTr%%pWQ@n(*I`yfGnNbMy-n}@`@bk(P zCUv9Bi_TG3*zT=0cR|DlGi{B1(?Go>YOxYH6j5auDf}kzT~`3ACuaLAlae>_%acZ- zcSD!4i{FDoev6%1$SwGcZk<7WJ~?(!Rbj^CrHM{-f`pBe#)~D{%UQK^DzVHXre&g7kV= zf*&p-NT$yJz&USaWQ>GyLDxh!@GTWf4-sEA%zckB%L`A z=*PGTNy`W$VKBBxnpbaH-$0Nd(mAp6J(f%v;zr{=wo`AD^Oz`dpWkOK__z<^y~p~% zD2@igC;l>;?cj*y(ELpJt~p@{Ee)g4pQbxTiXeIy$rc*yFl^wcm-5|+oV>0F(c^_J zHfAYQ+vNAgRpsH3J}a>Bc4VV1c%rvEp2rB(_xM@HUc=P#c9Ng97C1~Tzw#4O=!$w` z&(q=OHVu>;4~~s2u`BU8hgl=pq@QJcC~kR+#D;XlpFfV@&5060l_P8vUq+qKaQpD*w|&u^WF@o z&V#Jid5y%w+SH$HOc2I@Ig{{x=S-)F=`jC#&%TZmHdK%=CeBrrQmd0pT6!@2EgsDr z-f%0FU5XdSjcuLPAshAcE_U1%NX7KO%=$e=QUsB)-$>GXOL9e#h^8Cjh8 zt^*9S2p)35POp?BBA>hcRupOxvxrL`oWOtP-N*RtK~s2P zHKh-YK(o|&@T@ZWH*=Gqj^*=Wfc5wwu%WIOhdCZ@eW-dhJ-seF&Agr52!~44{k%5 zu2Nc_N^+`sDL#PJ(1n4ym7Y|WKwXbdT>{*-uEU3k=c*LbU@~Wcx-ERwv{%yGMX8t_9whh_XW6AiGU4gx<%_1+%C6@9p!P%T8 z?RM_n-+wk0dXnCf&v2((5#8wIM?Us>=G7kGjuLw(oHp>J_wvyLf3EpRXULsGK4zn7 zoXt>~VyNYCAwOU2!u^YhLgQZ<6$i#Qj_GRu>Zml%^92)`e%QlLS2T6ohhnu*#|z@M z_Z~rm2hGP{KfK%BaYSsbT##~Q_WE6N8|LN}cR$NBeBmd0XR;|e=sx!W?ZRWih_6Mv zJ;dfAKC~uUgZ)3^sn?>{hzEIZRWK7p__sC@wog{e&{EfjAJ#lRJa%gB#O~73`fJB7 zTeS$*=EkiNj5s_9PReu1QbDF)s$Ja8mfc{cE1JpIc#j0r334`wU!0NU+OoJhxc0vv zJXLC#rdaMxhO74Irtphs*pS%-nbVPw_a_Fv|WG2vR;jtV&acSml}5EBZ&@`!JG?pJmAZ zGUu*55GL~z8rKMB=XZ)KuUl5kf;y4?`MMPLcQ8X?mDQ_W@3pEGY=9@4RWpTD#R?Wv zR(@8-^+k^(yjz|1b@|ujRdvz=O@b__)#IjL4{aa?kan#TUb}?&xpi{G;#-ek^w4z; zNAh>)F>PcJI+nsGT}3y2SfieRU1+UbkbN8@^cCTo^(-#$b?2!)^737%xdxG-u_zZ) zVzWKUPHXV=fw=jNNAA6rSW2ICU~wLnV$wZ&P9GLcrK1LU!=~g%FhWu!ukC!ZhW7J- zk&&4K`a@XRJrdug)sKURhGc;;RZf@yLd5U>c@b@}4J?y!NqS<$5FdhxDS+j1gv!ZhJV_wp z#3?&+a8?m2%Qaz50P*8_f6D{ZWKZehD%oK_R0k~ zF%pU^idlIN*|)Ec*cqY8smR}^BB3yj!Cc*MATT+u)l!{Uk4$|E?uz5I&V|v z<*sA509@geSr2FIWtE7=pM(*c#mX7V9(4uAFCtmG6{+@)w+Q>} z_&CP*%hVSd;iHtk{!fKIKY;pA`A;i%y)X4-_5B?E6DtV@3LG)Nv7s{K&P&sQ))O_N z8}8=^^9=J#ByL%$im5~~j~4K?VM)Svj)yh&Ic}#4JQpo)^}VCcjF0Z>oVSlYT37lA zi@x=O?;n18k2s^hLckomb$tou-dp)UBZ|LwmA!--2@T@SWF=F2GE~#eHfnu_QcE3( v;k-JvJciK*D~3#HcU^LE{J*{YZzxn_>w}*fE4~MQ Date: Wed, 17 May 2023 16:43:58 +0200 Subject: [PATCH 04/62] Update module github.com/kairos-io/kairos-sdk to v0.0.3 (#1411) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- profile-build/go.mod | 54 ++++++++- profile-build/go.sum | 271 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 323 insertions(+), 2 deletions(-) diff --git a/profile-build/go.mod b/profile-build/go.mod index 05d95e50e..bfc867f45 100644 --- a/profile-build/go.mod +++ b/profile-build/go.mod @@ -3,15 +3,65 @@ module main go 1.20 require ( - github.com/kairos-io/kairos-sdk v0.0.1 + github.com/kairos-io/kairos-sdk v0.0.3 github.com/urfave/cli v1.22.13 ) require ( + atomicgo.dev/cursor v0.1.1 // indirect + atomicgo.dev/keyboard v0.2.9 // indirect + atomicgo.dev/schedule v0.0.2 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/hcsshim v0.10.0-rc.8 // indirect + github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect + github.com/containerd/cgroups v1.1.0 // indirect + github.com/containerd/console v1.0.3 // indirect + github.com/containerd/containerd v1.7.1 // indirect + github.com/containerd/continuity v0.3.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect + github.com/docker/cli v23.0.5+incompatible // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/docker v23.0.5+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-containerregistry v0.15.2 // indirect + github.com/gookit/color v1.5.3 // indirect github.com/joho/godotenv v1.5.1 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/lithammer/fuzzysearch v1.1.7 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc3 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pterm/pterm v0.12.61 // indirect + github.com/qeesung/image2ascii v1.0.1 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + github.com/vbatts/tar-split v0.11.3 // indirect + github.com/wayneashleyberry/terminal-dimensions v1.1.0 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/tools v0.9.1 // indirect + google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/grpc v1.53.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/profile-build/go.sum b/profile-build/go.sum index 2ef4a0d7b..227b5233f 100644 --- a/profile-build/go.sum +++ b/profile-build/go.sum @@ -1,4 +1,39 @@ +atomicgo.dev/cursor v0.1.1 h1:0t9sxQomCTRh5ug+hAMCs59x/UmC9QL6Ci5uosINKD4= +atomicgo.dev/cursor v0.1.1/go.mod h1:Lr4ZJB3U7DfPPOkbH7/6TOtJ4vFGHlgj1nc+n900IpU= +atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= +atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ= +atomicgo.dev/schedule v0.0.2 h1:2e/4KY6t3wokja01Cyty6qgkQM8MotJzjtqCH70oX2Q= +atomicgo.dev/schedule v0.0.2/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= +github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= +github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= +github.com/MarvinJWendt/testza v0.2.10/go.mod h1:pd+VWsoGUiFtq+hRKSU1Bktnn+DMCSrDrXDpX2bG66k= +github.com/MarvinJWendt/testza v0.2.12/go.mod h1:JOIegYyV7rX+7VZ9r77L/eH6CfJHHzXjB69adAhzZkI= +github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/2oUqKc6bF2c= +github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.10.0-rc.8 h1:YSZVvlIIDD1UxQpJp0h+dnpLUw+TrY0cx8obKsp3bek= +github.com/Microsoft/hcsshim v0.10.0-rc.8/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= +github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/containerd v1.7.1 h1:k8DbDkSOwt5rgxQ3uCI4WMKIJxIndSCBUaGm5oRn+Go= +github.com/containerd/containerd v1.7.1/go.mod h1:gA+nJUADRBm98QS5j5RPROnt0POQSMK+r7P7EGMC/Qc= +github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -6,28 +41,264 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= +github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE= +github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v23.0.5+incompatible h1:DaxtlTJjFSnLOXVNUBU1+6kXGz2lpDoEAH6QoxaSg8k= +github.com/docker/docker v23.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE= +github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= +github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= +github.com/gookit/color v1.5.3 h1:twfIhZs4QLCtimkP7MOxlF3A0U/5cDPseRT9M/+2SCE= +github.com/gookit/color v1.5.3/go.mod h1:NUzwzeehUfl7GIb36pqId+UGmRfQcU/WiiyTTeNjHtE= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kairos-io/kairos-sdk v0.0.1 h1:obJw0/5amn+/wWNuDTVq81HcPuX5TwHbGS4xxEy9Ju4= github.com/kairos-io/kairos-sdk v0.0.1/go.mod h1:E70cYgGQpu1MXI8ddhH4CHVIvNi3w7l6MQlxLTeBTXY= +github.com/kairos-io/kairos-sdk v0.0.3 h1:V/GJHFZuR7QWvWdxUZ76RXJdrZO8dL1aBJM74oJ0rU4= +github.com/kairos-io/kairos-sdk v0.0.3/go.mod h1:wAO6aJy/ek/Y/SE/iq5x7M9d8XYUR/jfDXA5DDG/IRU= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lithammer/fuzzysearch v1.1.7 h1:q8rZNmBIUkqxsxb/IlwsXVbCoPIH/0juxjFHY0UIwhU= +github.com/lithammer/fuzzysearch v1.1.7/go.mod h1:ZhIlfRGxnD8qa9car/yplC6GmnM14CS07BYAKJJBK2I= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI= +github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg= +github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE= +github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEejaWgXU= +github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE= +github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8= +github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s= +github.com/pterm/pterm v0.12.61 h1:cZFweZ0C4zbBsusyThfgqg0KU0PTnq5xupnGN3Ytxzc= +github.com/pterm/pterm v0.12.61/go.mod h1:07yyGZKQr8BpKKBaOZI1qKzzngqUisHdSYR4fQ9Nb4g= +github.com/qeesung/image2ascii v1.0.1 h1:Fe5zTnX/v/qNC3OC4P/cfASOXS501Xyw2UUcgrLgtp4= +github.com/qeesung/image2ascii v1.0.1/go.mod h1:kZKhyX0h2g/YXa/zdJR3JnLnJ8avHjZ3LrvEKSYyAyU= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/urfave/cli v1.22.13 h1:wsLILXG8qCJNse/qAgLNf23737Cx05GflHg/PJGe1Ok= github.com/urfave/cli v1.22.13/go.mod h1:VufqObjsMTF2BBwKawpx9R8eAneNEWhoO0yx8Vd+FkE= +github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= +github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= +github.com/wayneashleyberry/terminal-dimensions v1.1.0 h1:EB7cIzBdsOzAgmhTUtTTQXBByuPheP/Zv1zL2BRPY6g= +github.com/wayneashleyberry/terminal-dimensions v1.1.0/go.mod h1:2lc/0eWCObmhRczn2SdGSQtgBooLUzIotkkEGXqghyg= +github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From afd650e5db3f9048e006e277212877579096d519 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Thu, 18 May 2023 13:58:32 +0200 Subject: [PATCH 05/62] :bug: Merge grub yip config (#1426) With yip 1.x things can run at the same level as its now using a DAG. Problem is with the grub stuff it was 2 files trying to mount and unmount the state dir. That worked as expected before as it run one after the other but now they can run in parallel as they dont depend on each other. This PR merges both configs into one to avoid mount/umount several times and because both deal with grub stuff. Signed-off-by: Itxaka --- .../{08_boot_assessment.yaml => 08_grub.yaml} | 13 ++++++++++--- overlay/files/system/oem/21_grub.yaml | 18 ------------------ 2 files changed, 10 insertions(+), 21 deletions(-) rename overlay/files/system/oem/{08_boot_assessment.yaml => 08_grub.yaml} (93%) delete mode 100644 overlay/files/system/oem/21_grub.yaml diff --git a/overlay/files/system/oem/08_boot_assessment.yaml b/overlay/files/system/oem/08_grub.yaml similarity index 93% rename from overlay/files/system/oem/08_boot_assessment.yaml rename to overlay/files/system/oem/08_grub.yaml index aa1a9a881..5aee73828 100644 --- a/overlay/files/system/oem/08_boot_assessment.yaml +++ b/overlay/files/system/oem/08_grub.yaml @@ -9,7 +9,7 @@ # - If we get back again at the GRUB menu, a failure must have occurred and we select the fallback entry, creating also # sentinels files and a specific cmdline option indicating we failed booting after an upgrade -name: "Boot assessment" +name: "Boot assessment and branding" stages: boot.before: # Remove any grub sentinel and add @@ -40,6 +40,7 @@ stages: group: 0 after-install: # After install, reset, and upgrade, we install additional GRUB configuration for boot assessment into COS_STATE. + # We also add any branding that exists - &statemount name: "Mount state" @@ -97,11 +98,15 @@ stages: fi fi fi + - &grubinstall + name: "Grub branding" + if: '[ -e "/etc/kairos/branding/grubmenu.cfg" ]' + commands: + - cp -rfv /etc/kairos/branding/grubmenu.cfg /tmp/mnt/STATE/grubmenu - &stateumount name: "umount state" commands: - - | - umount /tmp/mnt/STATE + - umount /tmp/mnt/STATE # Here we do enable boot assessment for the next bootup. # Similarly, we could trigger boot assessment in other cases @@ -113,6 +118,7 @@ stages: grub2-editenv /tmp/mnt/STATE/boot_assessment set enable_boot_assessment=yes # We do re-install hooks here if needed to track upgrades of boot assessment - <<: *customhook + - <<: *grubinstall - <<: *bootgrub - <<: *stateumount @@ -120,5 +126,6 @@ stages: - <<: *statemount # Reset completely restores COS_STATE, so we re-inject ourselves - <<: *customhook + - <<: *grubinstall - <<: *bootgrub - <<: *stateumount diff --git a/overlay/files/system/oem/21_grub.yaml b/overlay/files/system/oem/21_grub.yaml deleted file mode 100644 index abe7114b6..000000000 --- a/overlay/files/system/oem/21_grub.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: "Additional grub menu entries" -stages: - after-install: - - &grubinstall - name: "Mount state" - if: '[ -e "/etc/kairos/branding/grubmenu.cfg" ]' - commands: - - | - STATEDIR=/tmp/mnt/STATE - STATE=$(blkid -L COS_STATE || true) - mkdir -p $STATEDIR || true - mount ${STATE} $STATEDIR - cp -rfv /etc/kairos/branding/grubmenu.cfg /tmp/mnt/STATE/grubmenu - umount /tmp/mnt/STATE - after-upgrade: - - <<: *grubinstall - after-reset: - - <<: *grubinstall From fc8daba82c5c60916f819651f6d5b25ce366a145 Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Fri, 19 May 2023 13:57:03 +0200 Subject: [PATCH 06/62] :robot: lint yaml grub file (#1434) * lint yaml grub file Signed-off-by: Mauro Morales * fix for good Signed-off-by: Mauro Morales --------- Signed-off-by: Mauro Morales --- overlay/files/system/oem/08_grub.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overlay/files/system/oem/08_grub.yaml b/overlay/files/system/oem/08_grub.yaml index 5aee73828..1caf5a282 100644 --- a/overlay/files/system/oem/08_grub.yaml +++ b/overlay/files/system/oem/08_grub.yaml @@ -102,7 +102,7 @@ stages: name: "Grub branding" if: '[ -e "/etc/kairos/branding/grubmenu.cfg" ]' commands: - - cp -rfv /etc/kairos/branding/grubmenu.cfg /tmp/mnt/STATE/grubmenu + - cp -rfv /etc/kairos/branding/grubmenu.cfg /tmp/mnt/STATE/grubmenu - &stateumount name: "umount state" commands: From 3ea41d51553f2932d405e3bd6ce99aa90aa63c05 Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Fri, 19 May 2023 17:50:11 +0200 Subject: [PATCH 07/62] :penguin: Use latest for fedora images (#1432) * Use latest for fedora images Signed-off-by: Mauro Morales * update zfs-release to 2.3 for fedora Signed-off-by: Mauro Morales --------- Signed-off-by: Mauro Morales --- images/Dockerfile.fedora | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/images/Dockerfile.fedora b/images/Dockerfile.fedora index 903f3c618..303eb6689 100644 --- a/images/Dockerfile.fedora +++ b/images/Dockerfile.fedora @@ -1,9 +1,10 @@ -ARG BASE_IMAGE=fedora:36 +# latest is the last stable release, does not include rawhide +ARG BASE_IMAGE=fedora:latest FROM $BASE_IMAGE RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf -RUN dnf install -y "https://zfsonlinux.org/fedora/zfs-release-2-2$(rpm --eval "%{dist}").noarch.rpm" && dnf clean all +RUN dnf install -y "https://zfsonlinux.org/fedora/zfs-release-2-3$(rpm --eval "%{dist}").noarch.rpm" && dnf clean all RUN dnf install -y \ NetworkManager \ squashfs-tools \ From 1dc54eee8a2e303bacde6518752e7d4704d11850 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 11:45:52 +0200 Subject: [PATCH 08/62] Update earthly/earthly Docker tag to v0.7.5 (#1398) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- earthly.ps1 | 2 +- earthly.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/earthly.ps1 b/earthly.ps1 index b2b02705d..df6d6c835 100644 --- a/earthly.ps1 +++ b/earthly.ps1 @@ -1 +1 @@ -docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.4 --allow-privileged @args +docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged @args diff --git a/earthly.sh b/earthly.sh index cfb4bce68..8793b0644 100755 --- a/earthly.sh +++ b/earthly.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.4 --allow-privileged "$@" +docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged "$@" From 67b0df071cc4d6eda06986e05accd33c9d61864c Mon Sep 17 00:00:00 2001 From: Itxaka Date: Tue, 23 May 2023 13:19:38 +0200 Subject: [PATCH 09/62] Revert ":robot: Run jobs on ubuntu-latest instead of self-hosted" (#1382) --- .github/workflows/image.yaml | 6 +++--- .github/workflows/release.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index addd7a2b2..90944b01e 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -157,7 +157,7 @@ jobs: build-framework: needs: - get-matrix - runs-on: ubuntu-latest + runs-on: self-hosted permissions: id-token: write strategy: @@ -336,7 +336,7 @@ jobs: qemu-bundles-tests: needs: - build - runs-on: ubuntu-latest + runs-on: self-hosted strategy: fail-fast: false matrix: @@ -376,7 +376,7 @@ jobs: qemu-reset-tests: needs: - build - runs-on: ubuntu-latest + runs-on: self-hosted strategy: fail-fast: false matrix: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fe02b9f92..9b7fa3d93 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,7 +24,7 @@ jobs: echo "::set-output name=matrix::{\"include\": $content }" build-framework: - runs-on: ubuntu-latest + runs-on: self-hosted needs: - get-matrix permissions: From 25533c4ac9a98dc5e092539e20c6613ffab4b2ae Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 24 May 2023 00:47:28 +0200 Subject: [PATCH 10/62] Update quay.io/kairos/osbuilder-tools Docker tag to v0.6.7 (#1412) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index cff89d682..f615a69cd 100644 --- a/Earthfile +++ b/Earthfile @@ -20,7 +20,7 @@ END ARG COSIGN_EXPERIMENTAL=0 ARG CGO_ENABLED=0 # renovate: datasource=docker depName=quay.io/kairos/osbuilder-tools versioning=semver-coerced -ARG OSBUILDER_VERSION=v0.6.4 +ARG OSBUILDER_VERSION=v0.6.7 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION ARG GOLINT_VERSION=1.52.2 # renovate: datasource=docker depName=golang From 3d32a21ea42357fc65787330dab93718002430cf Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 24 May 2023 00:48:30 +0200 Subject: [PATCH 11/62] Update earthly/earthly Docker tag to v0.7.6 (#1444) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- earthly.ps1 | 2 +- earthly.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/earthly.ps1 b/earthly.ps1 index df6d6c835..c250f6fc9 100644 --- a/earthly.ps1 +++ b/earthly.ps1 @@ -1 +1 @@ -docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged @args +docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.6 --allow-privileged @args diff --git a/earthly.sh b/earthly.sh index 8793b0644..538eb5d3b 100755 --- a/earthly.sh +++ b/earthly.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged "$@" +docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.6 --allow-privileged "$@" From ffd9f67353afdcd33b2cc165b934922f5708e01e Mon Sep 17 00:00:00 2001 From: Itxaka Date: Thu, 25 May 2023 08:05:30 +0000 Subject: [PATCH 12/62] :sparkles: Directly blacklist the vc4 module everywhere (#1443) --- Earthfile | 5 ----- overlay/files-rpi/etc/cos/bootargs.cfg | 18 ------------------ .../files/etc/modprobe.d/vc4-blacklist.conf | 9 +++++++++ 3 files changed, 9 insertions(+), 23 deletions(-) delete mode 100644 overlay/files-rpi/etc/cos/bootargs.cfg create mode 100644 overlay/files/etc/modprobe.d/vc4-blacklist.conf diff --git a/Earthfile b/Earthfile index f615a69cd..b95cb8544 100644 --- a/Earthfile +++ b/Earthfile @@ -202,11 +202,6 @@ framework: # TODO: Make this also a package? COPY overlay/files /framework - # Copy common overlay files for Raspberry Pi - IF [ "$MODEL" = "rpi64" ] - COPY overlay/files-rpi/ /framework - END - # Copy flavor-specific overlay files IF [[ "$FLAVOR" =~ ^alpine* ]] COPY overlay/files-alpine/ /framework diff --git a/overlay/files-rpi/etc/cos/bootargs.cfg b/overlay/files-rpi/etc/cos/bootargs.cfg deleted file mode 100644 index 8391be319..000000000 --- a/overlay/files-rpi/etc/cos/bootargs.cfg +++ /dev/null @@ -1,18 +0,0 @@ -set kernel=/boot/vmlinuz - -# Note on RPI bootargs -# We additionally set modprobe.blacklist=vc4 as certain Displays are not supported by vc4. -# As kairos main target is cloud and not graphics usage, we blacklist it to avoid -# that the HDMI output goes off due to drivers kicking during boot. vc4 is required where graphics -# or video playback is needed, which is not the case in this example here. -# A similar workaround could be applied at config.txt level, by diabling the vc4 overlay. -# See also: https://en.opensuse.org/HCL:Raspberry_Pi3#I_see_HDMI_output_in_U-Boot.2C_but_not_in_Linux , -# https://en.opensuse.org/HCL:Raspberry_Pi3#DSI_output_not_supported_by_VC4_driver, -# https://bugzilla.opensuse.org/show_bug.cgi?id=1181683 and https://github.com/raspberrypi/linux/issues/4020 -if [ -n "$recoverylabel" ]; then - set kernelcmd="console=tty1 console=ttyS0,115200 root=live:LABEL=$recoverylabel net.ifnames=1 rd.live.dir=/ rd.live.squashimg=$img panic=5 modprobe.blacklist=vc4 rd.cos.oemtimeout=10" -else - set kernelcmd="console=tty1 console=ttyS0,115200 root=LABEL=$label net.ifnames=1 cos-img/filename=$img panic=5 security=selinux selinux=1 modprobe.blacklist=vc4 rd.cos.oemtimeout=10 rd.cos.oemlabel=COS_OEM" -fi - -set initramfs=/boot/initrd \ No newline at end of file diff --git a/overlay/files/etc/modprobe.d/vc4-blacklist.conf b/overlay/files/etc/modprobe.d/vc4-blacklist.conf new file mode 100644 index 000000000..8b278baf8 --- /dev/null +++ b/overlay/files/etc/modprobe.d/vc4-blacklist.conf @@ -0,0 +1,9 @@ +# We additionally blacklist=vc4 as certain Displays are not supported by vc4. +# As kairos main target is cloud and not graphics usage, we blacklist it to avoid +# that the HDMI output goes off due to drivers kicking during boot. vc4 is required where graphics +# or video playback is needed, which is not the case in this example here. +# A similar workaround could be applied at config.txt level, by diabling the vc4 overlay. +# See also: https://en.opensuse.org/HCL:Raspberry_Pi3#I_see_HDMI_output_in_U-Boot.2C_but_not_in_Linux , +# https://en.opensuse.org/HCL:Raspberry_Pi3#DSI_output_not_supported_by_VC4_driver, +# https://bugzilla.opensuse.org/show_bug.cgi?id=1181683 and https://github.com/raspberrypi/linux/issues/4020 +blacklist vc4 \ No newline at end of file From e485cd451be597e3ed756b624ba9ddc3bafb8ad1 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 25 May 2023 13:31:50 +0200 Subject: [PATCH 13/62] :penguin: Add Jetson agx orin image (#1440) * WIP image Signed-off-by: mudler * Add nvidia overlay files Signed-off-by: mudler * fixups, add to pipelines Signed-off-by: mudler * lint fixes Signed-off-by: mudler * ci: workaround Earthly parallelism Signed-off-by: mudler * ci: nuke more space from public workers Signed-off-by: mudler * ci: try to run on self-hosted Signed-off-by: mudler * ci: add comment Signed-off-by: mudler * ci: use mirror in ARM jobs Signed-off-by: mudler * Run with docker Signed-off-by: mudler * run some jobs on self-hosted Signed-off-by: mudler * cleanup if on public runners Signed-off-by: mudler --------- Signed-off-by: mudler --- .github/flavors-arm.json | 23 +++- .github/workflows/image-arm.yaml | 62 +++++++++- .github/workflows/release-arm.yaml | 22 ++++ Earthfile | 49 +++++++- framework-profile.yaml | 3 + ...e.ubuntu-20-lts-arm-nvidia-jetson-agx-orin | 111 ++++++++++++++++++ overlay/files-nvidia/etc/cos/bootargs.cfg | 9 ++ .../files-nvidia/etc/dracut.conf.d/iscsi.conf | 1 + 8 files changed, 269 insertions(+), 11 deletions(-) create mode 100644 images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin create mode 100644 overlay/files-nvidia/etc/cos/bootargs.cfg create mode 100644 overlay/files-nvidia/etc/dracut.conf.d/iscsi.conf diff --git a/.github/flavors-arm.json b/.github/flavors-arm.json index 186f2c470..ae2c54c56 100644 --- a/.github/flavors-arm.json +++ b/.github/flavors-arm.json @@ -1,26 +1,37 @@ [ { "flavor": "opensuse-leap-arm-rpi", - "model": "rpi64" + "model": "rpi64", + "worker": "ubuntu-latest" }, { "flavor": "opensuse-tumbleweed-arm-rpi", - "model": "rpi64" + "model": "rpi64", + "worker": "ubuntu-latest" }, { "flavor": "alpine-arm-rpi", - "model": "rpi64" + "model": "rpi64", + "worker": "ubuntu-latest" }, { "flavor": "ubuntu-arm-rpi", - "model": "rpi64" + "model": "rpi64", + "worker": "ubuntu-latest" }, { "flavor": "ubuntu-20-lts-arm-rpi", - "model": "rpi64" + "model": "rpi64", + "worker": "ubuntu-latest" }, { "flavor": "ubuntu-22-lts-arm-rpi", - "model": "rpi64" + "model": "rpi64", + "worker": "ubuntu-latest" + }, + { + "flavor": "ubuntu-20-lts-arm-nvidia-jetson-agx-orin", + "model": "none", + "worker": "self-hosted" } ] diff --git a/.github/workflows/image-arm.yaml b/.github/workflows/image-arm.yaml index 38c659b54..00e954252 100644 --- a/.github/workflows/image-arm.yaml +++ b/.github/workflows/image-arm.yaml @@ -36,7 +36,7 @@ jobs: docker: needs: - get-matrix - runs-on: ubuntu-latest + runs-on: ${{ matrix.worker }} permissions: id-token: write # OIDC support contents: write @@ -47,9 +47,38 @@ jobs: matrix: ${{fromJson(needs.get-matrix.outputs.matrix)}} steps: - name: Release space from worker + if: ${{ matrix.worker != 'self-hosted' }} run: | - sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android - sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET + echo "Listing top largest packages" + pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr) + head -n 30 <<< "${pkgs}" + echo + df -h + echo + sudo apt-get remove -y '^llvm-.*|^libllvm.*' || true + sudo apt-get remove --auto-remove android-sdk-platform-tools || true + sudo apt-get purge --auto-remove android-sdk-platform-tools || true + sudo rm -rf /usr/local/lib/android + sudo apt-get remove -y '^dotnet-.*|^aspnetcore-.*' || true + sudo rm -rf /usr/share/dotnet + sudo apt-get remove -y '^mono-.*' || true + sudo apt-get remove -y '^ghc-.*' || true + sudo apt-get remove -y '.*jdk.*|.*jre.*' || true + sudo apt-get remove -y 'php.*' || true + sudo apt-get remove -y hhvm powershell firefox monodoc-manual msbuild || true + sudo apt-get remove -y '^google-.*' || true + sudo apt-get remove -y azure-cli || true + sudo apt-get remove -y '^mongo.*-.*|^postgresql-.*|^mysql-.*|^mssql-.*' || true + sudo apt-get remove -y '^gfortran-.*' || true + sudo apt-get autoremove -y + sudo apt-get clean + echo + echo "Listing top largest packages" + pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr) + head -n 30 <<< "${pkgs}" + echo + sudo rm -rfv build || true + df -h - uses: actions/checkout@v3 - run: | git fetch --prune --unshallow @@ -65,12 +94,37 @@ jobs: - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@master - - name: Build 🔧 + - name: Install earthly + uses: Luet-lab/luet-install-action@v1 + with: + repository: quay.io/kairos/packages + packages: utils/earthly + - name: Standard Build 🔧 + if: ${{ matrix.worker != 'self-hosted' }} env: FLAVOR: ${{ matrix.flavor }} MODEL: ${{ matrix.model }} run: | ./earthly.sh +all-arm --IMAGE_NAME=kairos-$FLAVOR-latest.img --IMAGE=quay.io/kairos/core-$FLAVOR:latest --MODEL=$MODEL --FLAVOR=$FLAVOR + - name: Selfhosted Build 🔧 + if: ${{ matrix.worker == 'self-hosted' }} + env: + FLAVOR: ${{ matrix.flavor }} + MODEL: ${{ matrix.model }} + run: | + # Configure earthly to use the docker mirror in CI + # https://docs.earthly.dev/ci-integration/pull-through-cache#configuring-earthly-to-use-the-cache + mkdir -p ~/.earthly/ + cat << EOF > ~/.earthly/config.yml + global: + buildkit_additional_config: | + [registry."docker.io"] + mirrors = ["registry.docker-mirror.svc.cluster.local:5000"] + [registry."registry.docker-mirror.svc.cluster.local:5000"] + insecure = true + http = true + EOF + docker run --privileged -v $HOME/.earthly/config.yml:/etc/.earthly/config.yml -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged +all-arm --IMAGE_NAME=kairos-$FLAVOR-latest.img --IMAGE=quay.io/kairos/core-$FLAVOR:latest --MODEL=$MODEL --FLAVOR=$FLAVOR - name: Push 🔧 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} env: diff --git a/.github/workflows/release-arm.yaml b/.github/workflows/release-arm.yaml index 2c3fea249..63a7c2bfd 100644 --- a/.github/workflows/release-arm.yaml +++ b/.github/workflows/release-arm.yaml @@ -57,12 +57,34 @@ jobs: username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - name: Build 🔧 + if: ${{ matrix.worker != 'self-hosted' }} env: FLAVOR: ${{ matrix.flavor }} MODEL: ${{ matrix.model }} run: | export TAG=${GITHUB_REF##*/} ./earthly.sh +all-arm --IMAGE_NAME=kairos-$FLAVOR-$TAG.img --IMAGE=quay.io/kairos/core-$FLAVOR:$TAG --MODEL=$MODEL --FLAVOR=$FLAVOR + - name: Selfhosted Build 🔧 + if: ${{ matrix.worker == 'self-hosted' }} + env: + FLAVOR: ${{ matrix.flavor }} + MODEL: ${{ matrix.model }} + run: | + # Configure earthly to use the docker mirror in CI + # https://docs.earthly.dev/ci-integration/pull-through-cache#configuring-earthly-to-use-the-cache + mkdir -p ~/.earthly/ + cat << EOF > ~/.earthly/config.yml + global: + buildkit_additional_config: | + [registry."docker.io"] + mirrors = ["registry.docker-mirror.svc.cluster.local:5000"] + [registry."registry.docker-mirror.svc.cluster.local:5000"] + insecure = true + http = true + EOF + export TAG=${GITHUB_REF##*/} + docker run --privileged -v $HOME/.earthly/config.yml:/etc/.earthly/config.yml -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged +all-arm --IMAGE_NAME=kairos-$FLAVOR-$TAG.img --IMAGE=quay.io/kairos/core-$FLAVOR:$TAG --MODEL=$MODEL --FLAVOR=$FLAVOR + - name: Push 🔧 env: FLAVOR: ${{ matrix.flavor }} diff --git a/Earthfile b/Earthfile index b95cb8544..c2f7ac968 100644 --- a/Earthfile +++ b/Earthfile @@ -55,7 +55,17 @@ all-arm: BUILD --platform=linux/arm64 +trivy-scan --MODEL=rpi64 BUILD --platform=linux/arm64 +grype-scan --MODEL=rpi64 END - BUILD +arm-image --MODEL=rpi64 + + IF [[ "$FLAVOR" = "ubuntu-20-lts-arm-nvidia-jetson-agx-orin" ]] + BUILD +prepare-arm-image --MODEL=rpi64 --FLAVOR=${FLAVOR} + + ELSE + BUILD +arm-image --MODEL=rpi64 + END + +arm-container-image: + ARG MODEL + BUILD --platform=linux/arm64 +image --MODEL=$MODEL all-arm-generic: BUILD --platform=linux/arm64 +image --MODEL=generic @@ -213,6 +223,10 @@ framework: COPY overlay/files-ubuntu-arm-rpi/ /framework END + IF [[ "$FLAVOR" = "ubuntu-20-lts-arm-nvidia-jetson-agx-orin" ]] + COPY overlay/files-nvidia/ /framework + END + SAVE ARTIFACT --keep-own /framework/ framework build-framework-image: @@ -472,6 +486,39 @@ arm-image: END SAVE ARTIFACT /build/$IMAGE_NAME.sha256 img-sha256 AS LOCAL build/$IMAGE_NAME.sha256 +prepare-arm-image: + ARG OSBUILDER_IMAGE + ARG COMPRESS_IMG=true + FROM $OSBUILDER_IMAGE + ARG MODEL=rpi64 + ARG IMAGE_NAME=${FLAVOR}.img + WORKDIR /build + # These sizes are in MB + ENV SIZE="15200" + IF [[ "$FLAVOR" =~ ^ubuntu* ]] + ENV STATE_SIZE="6900" + ENV RECOVERY_SIZE="4600" + ENV DEFAULT_ACTIVE_SIZE="2500" + ELSE + ENV STATE_SIZE="6200" + ENV RECOVERY_SIZE="4200" + ENV DEFAULT_ACTIVE_SIZE="2000" + END + COPY --platform=linux/arm64 +image-rootfs/rootfs /build/image + + ENV directory=/build/image + RUN mkdir bootloader + # With docker is required for loop devices + WITH DOCKER --allow-privileged + RUN /prepare_arm_images.sh + END + + SAVE ARTIFACT /build/bootloader/efi.img efi.img AS LOCAL build/efi.img + SAVE ARTIFACT /build/bootloader/oem.img oem.img AS LOCAL build/oem.img + SAVE ARTIFACT /build/bootloader/persistent.img persistent.img AS LOCAL build/persistent.img + SAVE ARTIFACT /build/bootloader/recovery_partition.img recovery_partition.img AS LOCAL build/recovery_partition.img + SAVE ARTIFACT /build/bootloader/state_partition.img state_partition.img AS LOCAL build/state_partition.img + ipxe-iso: FROM ubuntu ARG ipxe_script diff --git a/framework-profile.yaml b/framework-profile.yaml index 27646ee12..14df727a0 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -18,6 +18,9 @@ flavors: ubuntu-arm-rpi: - systemd-base - dracut-network-legacy + ubuntu-20-lts-arm-nvidia-jetson-agx-orin: + - systemd-base + - dracut-network-legacy-compat ubuntu-20-lts-arm-rpi: - systemd-base - dracut-network-legacy-compat diff --git a/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin b/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin new file mode 100644 index 000000000..80a379baa --- /dev/null +++ b/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin @@ -0,0 +1,111 @@ +FROM ubuntu:20.04 as base + +RUN apt-get update +RUN apt-get install -y ca-certificates + +RUN apt-get install -y sudo +RUN apt-get install -y ssh +RUN apt-get install -y netplan.io + +# resizerootfs +RUN apt-get install -y udev +RUN apt-get install -y parted + +# ifconfig +RUN apt-get install -y net-tools + +# needed by knod-static-nodes to create a list of static device nodes +RUN apt-get install -y kmod + +RUN systemctl enable ssh +RUN systemctl enable systemd-networkd + +RUN mkdir -p /opt/nvidia/l4t-packages +RUN touch /opt/nvidia/l4t-packages/.nv-l4t-disable-boot-fw-update-in-preinstall +#jetson orin +ARG BOARD_MODEL=t234 +ARG FRAMEWORK_VERSION=35.3 + +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common + +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub +RUN apt-key adv --fetch-key https://repo.download.nvidia.com/jetson/jetson-ota-public.asc +RUN add-apt-repository "deb https://repo.download.nvidia.com/jetson/common r$FRAMEWORK_VERSION main" +RUN add-apt-repository "deb https://repo.download.nvidia.com/jetson/$BOARD_MODEL r$FRAMEWORK_VERSION main" + +RUN apt-get update + +# nv-l4t-usb-device-mode +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + bridge-utils \ + conntrack \ + console-data \ + coreutils \ + cryptsetup \ + curl \ + debianutils \ + dmsetup \ + dosfstools \ + dracut \ + dracut-network \ + e2fsprogs \ + efibootmgr \ + file \ + fuse \ + gawk \ + grub-efi-arm64-bin \ + grub2-common \ + haveged \ + iproute2 \ + iptables \ + isc-dhcp-common \ + jq \ + kbd \ + krb5-locales \ + lldpd \ + lvm2 \ + mdadm \ + nbd-client \ + ncurses-term \ + networkd-dispatcher \ + nfs-common \ + open-iscsi \ + open-vm-tools \ + openssh-server \ + os-prober \ + packagekit-tools \ + parted \ + policykit-1 \ + publicsuffix \ + rsync \ + shared-mime-info \ + snmpd \ + squashfs-tools \ + sudo \ + systemd \ + systemd-timesyncd \ + xdg-user-dirs \ + xxd \ + xz-utils + +# https://docs.nvidia.com/jetson/l4t/index.html#page/Tegra%20Linux%20Driver%20Package%20Development%20Guide/updating_jetson_and_host.html +RUN apt-get install -y -o Dpkg::Options::="--force-overwrite" \ + nvidia-l4t-core \ + nvidia-l4t-init \ + nvidia-l4t-bootloader \ + nvidia-l4t-camera \ + nvidia-l4t-initrd \ + nvidia-l4t-xusb-firmware \ + nvidia-l4t-kernel \ + nvidia-l4t-kernel-dtbs \ + nvidia-l4t-kernel-headers \ + nvidia-l4t-cuda \ + jetson-gpio-common \ + python3-jetson-gpio + +# RUN rm -rf /opt/nvidia/l4t-packages +# RUN rm -rf /var/lib/apt/lists/* +# RUN useradd -ms /bin/bash jetson +# RUN echo 'jetson:jetson' | chpasswd +# RUN usermod -a -G sudo jetson \ No newline at end of file diff --git a/overlay/files-nvidia/etc/cos/bootargs.cfg b/overlay/files-nvidia/etc/cos/bootargs.cfg new file mode 100644 index 000000000..84c947e29 --- /dev/null +++ b/overlay/files-nvidia/etc/cos/bootargs.cfg @@ -0,0 +1,9 @@ +set kernel=/boot/vmlinuz + +if [ -n "$recoverylabel" ]; then + set kernelcmd="console=tty1 console=ttyTCU0,115200 root=live:LABEL=$recoverylabel net.ifnames=1 rd.live.dir=/ rd.live.squashimg=$img panic=5 rd.cos.oemtimeout=10" +else + set kernelcmd="console=tty1 console=ttyTCU0,115200 root=LABEL=$label net.ifnames=1 cos-img/filename=$img panic=5 security=selinux selinux=1 rd.cos.oemtimeout=10 rd.cos.oemlabel=COS_OEM" +fi + +set initramfs=/boot/initrd diff --git a/overlay/files-nvidia/etc/dracut.conf.d/iscsi.conf b/overlay/files-nvidia/etc/dracut.conf.d/iscsi.conf new file mode 100644 index 000000000..fbfcfdd56 --- /dev/null +++ b/overlay/files-nvidia/etc/dracut.conf.d/iscsi.conf @@ -0,0 +1 @@ +omit_dracutmodules+=" iscsi " From 22d7a418b598db7535049a0df932e9f77988018e Mon Sep 17 00:00:00 2001 From: Itxaka Date: Thu, 25 May 2023 13:25:47 +0000 Subject: [PATCH 14/62] Fully drop elemental from kairos (#1441) --- .github/encryption-tests.sh | 94 ++++++++++---------- .github/workflows/image.yaml | 2 +- Earthfile | 13 +-- framework-profile.yaml | 3 +- images/Dockerfile.opensuse-tumbleweed | 1 + overlay/files-alpine/etc/init.d/kairos-agent | 16 ++-- overlay/files-alpine/etc/init.d/kairos-webui | 2 +- overlay/files/system/oem/00_datasource.yaml | 11 ++- overlay/files/system/oem/00_rootfs.yaml | 10 +++ overlay/files/system/oem/02_agent.yaml | 4 +- overlay/files/system/oem/06_recovery.yaml | 4 +- overlay/files/system/oem/11_persistency.yaml | 1 + tests/assets/autoinstall.yaml | 2 +- tests/assets/config.yaml | 5 ++ tests/encryption_test.go | 67 ++++++++------ tests/go.mod | 4 +- tests/go.sum | 4 +- tests/tests_suite_test.go | 22 ++--- tests/upgrade_cli_test.go | 50 +++++------ tests/upgrade_latest_cli_test.go | 37 ++++---- 20 files changed, 196 insertions(+), 156 deletions(-) diff --git a/.github/encryption-tests.sh b/.github/encryption-tests.sh index aeb75c8f0..87f131b96 100755 --- a/.github/encryption-tests.sh +++ b/.github/encryption-tests.sh @@ -7,53 +7,57 @@ set -ex GINKGO_NODES="${GINKGO_NODES:-1}" K3S_IMAGE="rancher/k3s:v1.26.1-k3s1" - SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) CLUSTER_NAME=$(echo $RANDOM | md5sum | head -c 10; echo;) -KUBECONFIG=$(mktemp) -export KUBECONFIG - -cleanup() { - echo "Cleaning up $CLUSTER_NAME" - k3d cluster delete "$CLUSTER_NAME" || true - rm -rf "$KUBECONFIG" -} -trap cleanup EXIT - -# Create a cluster and bind ports 80 and 443 on the host -# This will allow us to access challenger server on 10.0.2.2 which is the IP -# on which qemu "sees" the host. -# We change the CIDR because k3s creates iptables rules that block DNS traffic to this CIDR -# (something like that). If you run k3d inside a k3s cluster (inside a Pod), DNS won't work -# inside the k3d server container unless you use a different CIDR. -# Here we are avoiding CIDR "10.43.x.x" -k3d cluster create "$CLUSTER_NAME" --k3s-arg "--cluster-cidr=10.49.0.1/16@server:0" --k3s-arg "--service-cidr=10.48.0.1/16@server:0" -p '80:80@server:0' -p '443:443@server:0' --image "$K3S_IMAGE" -k3d kubeconfig get "$CLUSTER_NAME" > "$KUBECONFIG" - -# Import the image to the cluster -#docker pull quay.io/kairos/kcrypt-challenger:latest -#k3d image import -c "$CLUSTER_NAME" quay.io/kairos/kcrypt-challenger:latest - -# Install cert manager -kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml -kubectl wait --for=condition=Available deployment --timeout=2m -n cert-manager --all - -# Replace the CLUSTER_IP in the kustomize resource -# Only needed for debugging so that we can access the server from the host -# (the 10.0.2.2 IP address is only useful from within qemu) -CLUSTER_IP=$(docker inspect "k3d-${CLUSTER_NAME}-server-0" | jq -r '.[0].NetworkSettings.Networks[].IPAddress') -export CLUSTER_IP - -envsubst \ - < "$SCRIPT_DIR/../tests/assets/encryption/challenger-server-ingress.template.yaml" \ - > "$SCRIPT_DIR/../tests/assets/encryption/challenger-server-ingress.yaml" - -# Install the challenger server kustomization -kubectl apply -k "$SCRIPT_DIR/../tests/assets/encryption/" - -# 10.0.2.2 is where the vm sees the host -# https://stackoverflow.com/a/6752280 -export KMS_ADDRESS="10.0.2.2.challenger.sslip.io" + + +if [ "$LABEL" != "local-encryption" ]; then + KUBECONFIG=$(mktemp) + export KUBECONFIG + + cleanup() { + echo "Cleaning up $CLUSTER_NAME" + k3d cluster delete "$CLUSTER_NAME" || true + rm -rf "$KUBECONFIG" + } + trap cleanup EXIT + + # Create a cluster and bind ports 80 and 443 on the host + # This will allow us to access challenger server on 10.0.2.2 which is the IP + # on which qemu "sees" the host. + # We change the CIDR because k3s creates iptables rules that block DNS traffic to this CIDR + # (something like that). If you run k3d inside a k3s cluster (inside a Pod), DNS won't work + # inside the k3d server container unless you use a different CIDR. + # Here we are avoiding CIDR "10.43.x.x" + k3d cluster create "$CLUSTER_NAME" --k3s-arg "--cluster-cidr=10.49.0.1/16@server:0" --k3s-arg "--service-cidr=10.48.0.1/16@server:0" -p '80:80@server:0' -p '443:443@server:0' --image "$K3S_IMAGE" + k3d kubeconfig get "$CLUSTER_NAME" > "$KUBECONFIG" + + # Import the image to the cluster + #docker pull quay.io/kairos/kcrypt-challenger:latest + #k3d image import -c "$CLUSTER_NAME" quay.io/kairos/kcrypt-challenger:latest + + # Install cert manager + kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml + kubectl wait --for=condition=Available deployment --timeout=2m -n cert-manager --all + + # Replace the CLUSTER_IP in the kustomize resource + # Only needed for debugging so that we can access the server from the host + # (the 10.0.2.2 IP address is only useful from within qemu) + CLUSTER_IP=$(docker inspect "k3d-${CLUSTER_NAME}-server-0" | jq -r '.[0].NetworkSettings.Networks[].IPAddress') + export CLUSTER_IP + + envsubst \ + < "$SCRIPT_DIR/../tests/assets/encryption/challenger-server-ingress.template.yaml" \ + > "$SCRIPT_DIR/../tests/assets/encryption/challenger-server-ingress.yaml" + + # Install the challenger server kustomization + kubectl apply -k "$SCRIPT_DIR/../tests/assets/encryption/" + + # 10.0.2.2 is where the vm sees the host + # https://stackoverflow.com/a/6752280 + export KMS_ADDRESS="10.0.2.2.challenger.sslip.io" +fi + pushd "$SCRIPT_DIR/../tests/" go run github.com/onsi/ginkgo/v2/ginkgo -v --nodes "$GINKGO_NODES" --label-filter "$LABEL" --fail-fast -r ./... diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 90944b01e..1b4102bba 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -439,7 +439,7 @@ jobs: upgrade-with-cli-test: needs: - build - runs-on: ubuntu-latest + runs-on: self-hosted strategy: fail-fast: false matrix: diff --git a/Earthfile b/Earthfile index c2f7ac968..95a27cf11 100644 --- a/Earthfile +++ b/Earthfile @@ -267,14 +267,13 @@ base-image: # Includes overlay/files COPY (+framework/framework --FLAVOR=$FLAVOR --VERSION=$OS_VERSION --MODEL=$MODEL) / - - RUN rm -rf /etc/machine-id && touch /etc/machine-id && chmod 444 /etc/machine-id - # Avoid to accidentally push keys generated by package managers RUN rm -rf /etc/ssh/ssh_host_* # Enable services IF [ -f /sbin/openrc ] + # Fully remove machine-id, it will be generated + RUN rm -rf /etc/machine-id RUN mkdir -p /etc/runlevels/default && \ ln -sf /etc/init.d/cos-setup-boot /etc/runlevels/default/cos-setup-boot && \ ln -sf /etc/init.d/cos-setup-network /etc/runlevels/default/cos-setup-network && \ @@ -282,6 +281,8 @@ base-image: ln -sf /etc/init.d/kairos-agent /etc/runlevels/default/kairos-agent # Otherwise we assume systemd ELSE + # Empty machine-id so we dont accidentally run systemd-firstboot ¬_¬ + RUN rm -rf /etc/machine-id && touch /etc/machine-id && chmod 444 /etc/machine-id RUN ls -liah /etc/systemd/system RUN systemctl enable cos-setup-reconcile.timer && \ systemctl enable cos-setup-fs.service && \ @@ -343,13 +344,13 @@ base-image: RUN --no-cache kernel=$(ls /lib/modules | head -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd END + + # Set /boot/vmlinuz pointing to our kernel so kairos-agent can use it + # https://github.com/kairos-io/kairos-agent/blob/0288fb111bc568a1bfca59cb09f39302220475b6/pkg/elemental/elemental.go#L548 q IF [ "$FLAVOR" = "fedora" ] || [ "$FLAVOR" = "rockylinux" ] - # https://github.com/kairos-io/elemental-cli/blob/23ca64435fedb9f521c95e798d2c98d2714c53bd/pkg/elemental/elemental.go#L553 RUN rm -rf /boot/initramfs-* END - # Set /boot/vmlinuz pointing to our kernel so elemental-cli can use it - # https://github.com/kairos-io/elemental-cli/blob/23ca64435fedb9f521c95e798d2c98d2714c53bd/pkg/elemental/elemental.go#L553 IF [ ! -e "/boot/vmlinuz" ] # If it's an ARM flavor, we want a symlink here from zImage/Image # Check that its not a symlink already or grub will fail! diff --git a/framework-profile.yaml b/framework-profile.yaml index 14df727a0..6790cae4b 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -5,7 +5,6 @@ common: - system/kcrypt-challenger - system/suc-upgrade - system/grub2-efi - - system/elemental-cli - system/immucore - system/kairos-agent flavors: @@ -100,7 +99,7 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230512113938-repository.yaml + reference: 20230524091830-repository.yaml - !!merge <<: *kairos arch: arm64 urls: diff --git a/images/Dockerfile.opensuse-tumbleweed b/images/Dockerfile.opensuse-tumbleweed index 7f8442457..6cf73a1e4 100644 --- a/images/Dockerfile.opensuse-tumbleweed +++ b/images/Dockerfile.opensuse-tumbleweed @@ -2,6 +2,7 @@ ARG BASE_IMAGE=opensuse/tumbleweed FROM $BASE_IMAGE +RUN zypper removerepo repo-openh264 RUN zypper ar -G https://download.opensuse.org/repositories/utilities/openSUSE_Factory/utilities.repo && \ zypper ref diff --git a/overlay/files-alpine/etc/init.d/kairos-agent b/overlay/files-alpine/etc/init.d/kairos-agent index 69e3a1063..99c8fb144 100755 --- a/overlay/files-alpine/etc/init.d/kairos-agent +++ b/overlay/files-alpine/etc/init.d/kairos-agent @@ -2,15 +2,11 @@ depend() { provide kairos-agent + after cos-setup-network + use net } -supervisor=supervise-daemon -name="kairos-agent" -command="kairos-agent start" -command_args="--restart" -supervise_daemon_args="--stdout /var/log/kairos/agent.log --stderr /var/log/kairos/agent.log" -pidfile="/run/kairos-agent.pid" -respawn_delay=5 -set -o allexport -if [ -f /etc/environment ]; then source /etc/environment; fi -set +o allexport \ No newline at end of file +start() { + kairos-agent start + eend $? +} \ No newline at end of file diff --git a/overlay/files-alpine/etc/init.d/kairos-webui b/overlay/files-alpine/etc/init.d/kairos-webui index f0cd0d6a0..c59d96c31 100755 --- a/overlay/files-alpine/etc/init.d/kairos-webui +++ b/overlay/files-alpine/etc/init.d/kairos-webui @@ -8,7 +8,7 @@ supervisor=supervise-daemon name="kairos-webui" command="kairos-agent webui" supervise_daemon_args="--stdout /var/log/kairos/webui.log --stderr /var/log/kairos/webui.log" -pidfile="/run/kairos-agent.pid" +pidfile="/run/${RC_SVCNAME}.pid" respawn_delay=5 set -o allexport if [ -f /etc/environment ]; then source /etc/environment; fi diff --git a/overlay/files/system/oem/00_datasource.yaml b/overlay/files/system/oem/00_datasource.yaml index 7844e84da..d87edc4d8 100644 --- a/overlay/files/system/oem/00_datasource.yaml +++ b/overlay/files/system/oem/00_datasource.yaml @@ -5,24 +5,27 @@ stages: if: '[ ! -f /oem/userdata ]' name: "Pull data from provider" datasource: - providers: ["cdrom", "gcp", "openstack", "aws", "azure", "hetzner", "packet", "scaleway", "vultr", "digitalocean", "metaldata"] + providers: ["cdrom", "gcp", "openstack", "aws", "azure", "hetzner", "packet", "vultr", "digitalocean", "metaldata"] path: "/oem" - if: '[ ! -f /oem/userdata ]' + name: "Sentinel file for userdata" files: - path: /run/.userdata_load initramfs.before: - <<: *datasource - if: '[ ! -f /oem/userdata ]' files: - - path: /oem/.userdata_load + - path: /run/.userdata_load # After network, if no datasource could be pulled, we stop trying network: - <<: *datasource - if: '[ -f /oem/userdata ] && [ -f /run/.userdata_load ]' + name: "Run stages if userdata is found" commands: - - elemental cloud-init -s initramfs /oem/userdata - - elemental cloud-init -s boot /oem/userdata + - kairos-agent run-stage initramfs + - kairos-agent run-stage boot - rm -rf /run/.userdata_load - if: '[ ! -f /oem/userdata ] && [ -f /run/.userdata_load ]' + name: "Remove userdata sentinel" commands: - rm -rf /run/.userdata_load diff --git a/overlay/files/system/oem/00_rootfs.yaml b/overlay/files/system/oem/00_rootfs.yaml index 748dcd08f..e78bcec4e 100644 --- a/overlay/files/system/oem/00_rootfs.yaml +++ b/overlay/files/system/oem/00_rootfs.yaml @@ -31,6 +31,10 @@ stages: name: "Restore /etc/machine-id" commands: - cat /usr/local/etc/machine-id > /etc/machine-id + - if: '[ ! -f "/run/cos/recovery_mode" ] && [ -s /var/lib/dbus/machine-id ]' + name: "Restore /etc/machine-id for openrc systems" + commands: + - cat /var/lib/dbus/machine-id > /etc/machine-id fs: - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -s /usr/local/etc/machine-id ] ' name: "Save /etc/machine-id" @@ -38,3 +42,9 @@ stages: - | mkdir -p /usr/local/etc cp /etc/machine-id /usr/local/etc + - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -s /var/lib/dbus/machine-id ] ' + name: "Save /etc/machine-id for openrc systems" + commands: + - | + mkdir -p /var/lib/dbus/ + cp /etc/machine-id /var/lib/dbus/ diff --git a/overlay/files/system/oem/02_agent.yaml b/overlay/files/system/oem/02_agent.yaml index b9ec6ecc7..652ab3463 100644 --- a/overlay/files/system/oem/02_agent.yaml +++ b/overlay/files/system/oem/02_agent.yaml @@ -1,11 +1,11 @@ name: "Start agent" stages: boot: - - if: '[ ! -f "/run/cos/recovery_mode" ]' + - if: '[ ! -f "/run/cos/recovery_mode" ] && [ -d "/usr/share/systemd" ]' commands: - systemctl start kairos-agent - systemctl enable kairos-agent initramfs: - - if: '[ ! -f "/run/cos/recovery_mode" ]' + - if: '[ ! -f "/run/cos/recovery_mode" ] && [ -d "/usr/share/systemd" ]' commands: - systemctl enable kairos-agent diff --git a/overlay/files/system/oem/06_recovery.yaml b/overlay/files/system/oem/06_recovery.yaml index cada93eb6..ba9676c2a 100644 --- a/overlay/files/system/oem/06_recovery.yaml +++ b/overlay/files/system/oem/06_recovery.yaml @@ -25,6 +25,6 @@ stages: - | source /etc/os-release echo >> /etc/issue - echo "You are booting from recovery mode. Run 'cos-reset' to reset the system to $VERSION" >> /etc/issue - echo " or elemental upgrade to upgrade the active partition" >> /etc/issue + echo "You are booting from recovery mode. Run 'kairos-agent reset' to reset the system to $VERSION" >> /etc/issue + echo " or 'kairos-agent upgrade' to upgrade the active partition" >> /etc/issue echo >> /etc/issue diff --git a/overlay/files/system/oem/11_persistency.yaml b/overlay/files/system/oem/11_persistency.yaml index 8095cda36..846050d46 100644 --- a/overlay/files/system/oem/11_persistency.yaml +++ b/overlay/files/system/oem/11_persistency.yaml @@ -33,6 +33,7 @@ stages: /var/lib/wicked /var/lib/longhorn /var/lib/cni + /var/lib/dbus /usr/share/pki/trust /usr/share/pki/trust/anchors /var/lib/ca-certificates diff --git a/tests/assets/autoinstall.yaml b/tests/assets/autoinstall.yaml index 62617b461..7bf9566cf 100644 --- a/tests/assets/autoinstall.yaml +++ b/tests/assets/autoinstall.yaml @@ -5,7 +5,7 @@ install: reboot: true device: /dev/vda grub_options: - extra_cmdline: "foobarzz" + extra_cmdline: "rd.immucore.debug foobarzz" stages: initramfs: diff --git a/tests/assets/config.yaml b/tests/assets/config.yaml index 9cfcd4ad6..38eee09df 100644 --- a/tests/assets/config.yaml +++ b/tests/assets/config.yaml @@ -1,9 +1,14 @@ #cloud-config +install: + grub_options: + extra_cmdline: "rd.immucore.debug" + stages: initramfs: - name: "Set user and password" users: kairos: passwd: "kairos" + - name: "Set hostname" hostname: kairos-{{ trunc 4 .Random }} diff --git a/tests/encryption_test.go b/tests/encryption_test.go index c15bafabb..2d42f7520 100644 --- a/tests/encryption_test.go +++ b/tests/encryption_test.go @@ -2,17 +2,17 @@ package mos_test import ( "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/spectrocloud/peg/matcher" + "gopkg.in/yaml.v3" "os" "os/exec" "path" + "path/filepath" "strconv" "strings" "syscall" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/spectrocloud/peg/matcher" - "gopkg.in/yaml.v3" ) var installationOutput string @@ -37,14 +37,21 @@ var _ = Describe("kcrypt encryption", func() { err = os.WriteFile(configFile.Name(), []byte(config), 0744) Expect(err).ToNot(HaveOccurred()) - err = vm.Scp(configFile.Name(), "config.yaml", "0744") + err = vm.Scp(configFile.Name(), "/tmp/config.yaml", "0744") Expect(err).ToNot(HaveOccurred()) - - installationOutput, err = vm.Sudo("/bin/bash -c 'set -o pipefail && kairos-agent manual-install --device auto config.yaml 2>&1 | tee manual-install.txt'") + By("Manually installing") + installationOutput, err = vm.Sudo("kairos-agent --debug manual-install --device auto /tmp/config.yaml") Expect(err).ToNot(HaveOccurred(), installationOutput) }) AfterEach(func() { + if CurrentSpecReport().Failed() { + gatherLogs(vm) + serial, _ := os.ReadFile(filepath.Join(vm.StateDir, "serial.log")) + _ = os.MkdirAll("logs", os.ModePerm|os.ModeDir) + _ = os.WriteFile(filepath.Join("logs", "serial.log"), serial, os.ModePerm) + fmt.Println(string(serial)) + } err := vm.Destroy(func(vm VM) { // Stop TPM emulator tpmPID, err := os.ReadFile(path.Join(vm.StateDir, "tpm", "pid")) @@ -66,20 +73,27 @@ var _ = Describe("kcrypt encryption", func() { config = `#cloud-config install: + grub_options: + extra_cmdline: "rd.immucore.debug" encrypted_partitions: - - COS_PERSISTENT + - COS_PERSISTENT reboot: false # we will reboot manually -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - passwd: kairos +stages: + initramfs: + - name: "Set user and password" + users: + kairos: + passwd: "kairos" + hostname: kairos-{{ trunc 4 .Random }} ` }) It("boots and has an encrypted partition", func() { + By("Rebooting") vm.Reboot() vm.EventuallyConnects(1200) + By("Checking the partition") out, err := vm.Sudo("blkid") Expect(err).ToNot(HaveOccurred(), out) Expect(out).To(MatchRegexp("TYPE=\"crypto_LUKS\" PARTLABEL=\"persistent\""), out) @@ -195,10 +209,13 @@ spec: config = fmt.Sprintf(`#cloud-config -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - passwd: kairos +stages: + initramfs: + - name: "Set user and password" + users: + kairos: + passwd: "kairos" + hostname: kairos-{{ trunc 4 .Random }} install: encrypted_partitions: @@ -268,11 +285,13 @@ spec: Expect(err).ToNot(HaveOccurred()) config = fmt.Sprintf(`#cloud-config -hostname: metal-{{ trunc 4 .MachineID }} -users: -- name: kairos - passwd: kairos - +stages: + initramfs: + - name: "Set user and password" + users: + kairos: + passwd: "kairos" + hostname: kairos-{{ trunc 4 .Random }} install: encrypted_partitions: - COS_PERSISTENT @@ -321,9 +340,7 @@ kcrypt: }) It("fails to talk to the server", func() { - out, err := vm.Sudo("cat manual-install.txt") - Expect(err).ToNot(HaveOccurred(), out) - Expect(out).To(MatchRegexp("could not encrypt partition.*x509: certificate signed by unknown authority")) + Expect(installationOutput).To(MatchRegexp("could not encrypt partition.*x509: certificate signed by unknown authority")) }) }) }) diff --git a/tests/go.mod b/tests/go.mod index 9dea23123..59634509e 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -7,7 +7,8 @@ require ( github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.6 - github.com/spectrocloud/peg v0.0.0-20230407121159-2e15270c4a46 + github.com/spectrocloud/peg v0.0.0-20230517145826-35016677d163 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -37,5 +38,4 @@ require ( golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.7.0 // indirect google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tests/go.sum b/tests/go.sum index b4f598694..da1aa070d 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -87,8 +87,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/spectrocloud/peg v0.0.0-20230407121159-2e15270c4a46 h1:q2T2RnISqPdZWvUpBQw0n7QWtF4cNo5RpCDTZmV732M= -github.com/spectrocloud/peg v0.0.0-20230407121159-2e15270c4a46/go.mod h1:L2fIdtZqbQEagjOOXwkwH3t7MjJUd7fbt52cLSQGDBg= +github.com/spectrocloud/peg v0.0.0-20230517145826-35016677d163 h1:5RlgJFagRV81CIDBC5peiWuYdhpG0jSdlx32sIyOFJw= +github.com/spectrocloud/peg v0.0.0-20230517145826-35016677d163/go.mod h1:L2fIdtZqbQEagjOOXwkwH3t7MjJUd7fbt52cLSQGDBg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/tests/tests_suite_test.go b/tests/tests_suite_test.go index c45744045..d33527ddc 100644 --- a/tests/tests_suite_test.go +++ b/tests/tests_suite_test.go @@ -26,11 +26,6 @@ func TestSuite(t *testing.T) { RunSpecs(t, "kairos Test Suite") } -var tempDir string -var sshPort string - -var machineID string = os.Getenv("MACHINE_ID") - var getVersionCmd = ". /etc/os-release; [ ! -z \"$KAIROS_VERSION\" ] && echo $KAIROS_VERSION || echo $VERSION" // https://gist.github.com/sevkin/96bdae9274465b2d09191384f86ef39d @@ -48,20 +43,20 @@ func getFreePort() (port int, err error) { } func user() string { - user := os.Getenv("SSH_USER") - if user == "" { - user = "kairos" + u := os.Getenv("SSH_USER") + if u == "" { + u = "kairos" } - return user + return u } func pass() string { - pass := os.Getenv("SSH_PASS") - if pass == "" { - pass = "kairos" + p := os.Getenv("SSH_PASS") + if p == "" { + p = "kairos" } - return pass + return p } func gatherLogs(vm VM) { @@ -117,6 +112,7 @@ func startVM() (context.Context, VM) { sshPort, err = getFreePort() Expect(err).ToNot(HaveOccurred()) + fmt.Printf("Using ssh port: %d\n", sshPort) memory := os.Getenv("MEMORY") if memory == "" { diff --git a/tests/upgrade_cli_test.go b/tests/upgrade_cli_test.go index 126c3ed3f..04e63a2c2 100644 --- a/tests/upgrade_cli_test.go +++ b/tests/upgrade_cli_test.go @@ -3,6 +3,7 @@ package mos_test import ( "fmt" "os" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -10,36 +11,44 @@ import ( . "github.com/spectrocloud/peg/matcher" ) -// test ci var _ = Describe("k3s upgrade manual test", Label("upgrade-with-cli"), func() { - + var vm VM containerImage := os.Getenv("CONTAINER_IMAGE") - var vm VM BeforeEach(func() { + if containerImage == "" { + Fail("CONTAINER_IMAGE needs to be set") + } _, vm = startVM() vm.EventuallyConnects(1200) }) AfterEach(func() { + if CurrentSpecReport().Failed() { + gatherLogs(vm) + serial, _ := os.ReadFile(filepath.Join(vm.StateDir, "serial.log")) + _ = os.MkdirAll("logs", os.ModePerm|os.ModeDir) + _ = os.WriteFile(filepath.Join("logs", "serial.log"), serial, os.ModePerm) + fmt.Println(string(serial)) + } Expect(vm.Destroy(nil)).ToNot(HaveOccurred()) }) Context("upgrades", func() { BeforeEach(func() { - if containerImage == "" { - Fail("CONTAINER_IMAGE needs to be set") - } - expectDefaultService(vm) By("Copying config file") err := vm.Scp("assets/config.yaml", "/tmp/config.yaml", "0770") Expect(err).ToNot(HaveOccurred()) By("Manually installing") - out, err := vm.Sudo("kairos-agent manual-install --device auto /tmp/config.yaml") - Expect(err).ToNot(HaveOccurred()) + out, err := vm.Sudo("/bin/bash -c 'set -o pipefail && kairos-agent --debug manual-install --device auto /tmp/config.yaml 2>&1 | tee manual-install.txt'") + Expect(err).ToNot(HaveOccurred(), out) + Expect(out).Should(ContainSubstring("Running after-install hook")) vm.Sudo("sync") + + err = vm.DetachCD() + Expect(err).ToNot(HaveOccurred()) By("Rebooting") vm.Reboot() }) @@ -49,22 +58,13 @@ var _ = Describe("k3s upgrade manual test", Label("upgrade-with-cli"), func() { Expect(err).ToNot(HaveOccurred()) By(fmt.Sprintf("Checking current version: %s", currentVersion)) Expect(currentVersion).To(ContainSubstring("v")) - _, err = vm.Sudo("kairos-agent") - if err == nil { - By(fmt.Sprintf("Upgrading to: %s", containerImage)) - out, err := vm.Sudo("kairos-agent upgrade --force --image " + containerImage) - Expect(err).ToNot(HaveOccurred(), string(out)) - Expect(out).To(ContainSubstring("Upgrade completed")) - Expect(out).To(ContainSubstring(containerImage)) - fmt.Println(out) - } else { - By(fmt.Sprintf("Upgrading to: %s", containerImage)) - out, err := vm.Sudo("kairos upgrade --force --image " + containerImage) - Expect(err).ToNot(HaveOccurred(), string(out)) - Expect(out).To(ContainSubstring("Upgrade completed")) - Expect(out).To(ContainSubstring(containerImage)) - fmt.Println(out) - } + + By(fmt.Sprintf("Upgrading to: %s", containerImage)) + out, err := vm.Sudo("kairos-agent --debug upgrade --force --image " + containerImage) + Expect(err).ToNot(HaveOccurred(), string(out)) + Expect(out).To(ContainSubstring("Upgrade completed")) + Expect(out).To(ContainSubstring(containerImage)) + fmt.Println(out) vm.Reboot() diff --git a/tests/upgrade_latest_cli_test.go b/tests/upgrade_latest_cli_test.go index 4e4ff979a..72613cb65 100644 --- a/tests/upgrade_latest_cli_test.go +++ b/tests/upgrade_latest_cli_test.go @@ -3,6 +3,7 @@ package mos_test import ( "fmt" "os" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -15,21 +16,31 @@ var _ = Describe("k3s upgrade manual test", Label("upgrade-latest-with-cli"), fu containerImage := os.Getenv("CONTAINER_IMAGE") BeforeEach(func() { + if containerImage == "" { + Fail("CONTAINER_IMAGE needs to be set") + } _, vm = startVM() vm.EventuallyConnects(1200) }) AfterEach(func() { + if CurrentSpecReport().Failed() { + gatherLogs(vm) + serial, _ := os.ReadFile(filepath.Join(vm.StateDir, "serial.log")) + _ = os.MkdirAll("logs", os.ModePerm|os.ModeDir) + _ = os.WriteFile(filepath.Join("logs", "serial.log"), serial, os.ModePerm) + fmt.Println(string(serial)) + } Expect(vm.Destroy(nil)).ToNot(HaveOccurred()) }) Context("upgrades", func() { BeforeEach(func() { expectDefaultService(vm) - + By("Copying config file") err := vm.Scp("assets/config.yaml", "/tmp/config.yaml", "0770") Expect(err).ToNot(HaveOccurred()) - + By("Manually installing") out, err := vm.Sudo("/bin/bash -c 'set -o pipefail && kairos-agent manual-install --device auto /tmp/config.yaml 2>&1 | tee manual-install.txt'") Expect(err).ToNot(HaveOccurred(), out) @@ -38,26 +49,22 @@ var _ = Describe("k3s upgrade manual test", Label("upgrade-latest-with-cli"), fu err = vm.DetachCD() Expect(err).ToNot(HaveOccurred()) + By("Rebooting") vm.Reboot() }) It("can upgrade to current image", func() { currentVersion, err := vm.Sudo(getVersionCmd) Expect(err).ToNot(HaveOccurred()) + By(fmt.Sprintf("Checking current version: %s", currentVersion)) Expect(currentVersion).To(ContainSubstring("v")) - _, err = vm.Sudo("kairos-agent") - if err == nil { - out, err := vm.Sudo("kairos-agent upgrade --force --image " + containerImage) - Expect(err).ToNot(HaveOccurred(), string(out)) - Expect(out).To(ContainSubstring("Upgrade completed")) - Expect(out).To(ContainSubstring(containerImage)) - fmt.Println(out) - } else { - out, err := vm.Sudo("kairos upgrade --force --image " + containerImage) - Expect(err).ToNot(HaveOccurred(), string(out)) - Expect(out).To(ContainSubstring("Upgrade completed")) - Expect(out).To(ContainSubstring(containerImage)) - } + + By(fmt.Sprintf("Upgrading to: %s", containerImage)) + out, err := vm.Sudo("kairos-agent upgrade --force --image " + containerImage) + Expect(err).ToNot(HaveOccurred(), string(out)) + Expect(out).To(ContainSubstring("Upgrade completed")) + Expect(out).To(ContainSubstring(containerImage)) + fmt.Println(out) vm.Reboot() From 2bfcc338b657c1e52d6fea237f1f012f5e5c0ee2 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Thu, 25 May 2023 13:38:00 +0000 Subject: [PATCH 15/62] Fix lint (#1452) --- overlay/files/system/oem/00_rootfs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overlay/files/system/oem/00_rootfs.yaml b/overlay/files/system/oem/00_rootfs.yaml index e78bcec4e..a2a9e7272 100644 --- a/overlay/files/system/oem/00_rootfs.yaml +++ b/overlay/files/system/oem/00_rootfs.yaml @@ -47,4 +47,4 @@ stages: commands: - | mkdir -p /var/lib/dbus/ - cp /etc/machine-id /var/lib/dbus/ + cp /etc/machine-id /var/lib/dbus/ From b374bf12f8b0941fa73d0ae97cc36370d00cfbad Mon Sep 17 00:00:00 2001 From: "ci-robbot [bot]" <105103991+ci-robbot@users.noreply.github.com> Date: Fri, 26 May 2023 18:02:57 +0200 Subject: [PATCH 16/62] :arrow_up: Update repositories (#1442) Co-authored-by: Itxaka --- framework-profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework-profile.yaml b/framework-profile.yaml index 6790cae4b..c0f1b6d39 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -99,9 +99,9 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230524091830-repository.yaml + reference: 20230526140327-repository.yaml - !!merge <<: *kairos arch: arm64 urls: - "quay.io/kairos/packages-arm64" - reference: 20230512115044-repository.yaml + reference: 20230524170431-repository.yaml From 156c79d2571104e729254b315c682b1a0509b9d6 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 26 May 2023 22:06:46 +0200 Subject: [PATCH 17/62] :penguin: Fixes to the orin image (#1455) * :penguin: add grub2 symlink to the orin image Signed-off-by: mudler * :bug: Copy ubuntu arm artifacts into orin images Signed-off-by: mudler * Update Earthfile Co-authored-by: Mauro Morales Signed-off-by: Ettore Di Giacinto Signed-off-by: mudler * :seedling: move framework preparation to a script Signed-off-by: mudler * :seedling: run dracut only when its present in the image Signed-off-by: mudler * Revert ":seedling: move framework preparation to a script" This reverts commit 025ffa248434bc49baf1f91c7105e04c6d3ee0b4. Signed-off-by: mudler * try to keep things on earthly Signed-off-by: mudler --------- Signed-off-by: mudler Signed-off-by: Ettore Di Giacinto Co-authored-by: Mauro Morales --- Earthfile | 7 ++----- images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin | 4 ++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Earthfile b/Earthfile index 95a27cf11..8bde70201 100644 --- a/Earthfile +++ b/Earthfile @@ -219,7 +219,7 @@ framework: COPY overlay/files-fedora/ /framework ELSE IF [ "$FLAVOR" = "debian" ] || [ "$FLAVOR" = "ubuntu" ] || [ "$FLAVOR" = "ubuntu-20-lts" ] || [ "$FLAVOR" = "ubuntu-22-lts" ] COPY overlay/files-ubuntu/ /framework - ELSE IF [[ "$FLAVOR" =~ ^ubuntu-arm* ]] + ELSE IF [[ "$FLAVOR" =~ ^ubuntu-.*-lts-arm-.*$ ]] COPY overlay/files-ubuntu-arm-rpi/ /framework END @@ -336,15 +336,12 @@ base-image: END - IF [[ "$FLAVOR" =~ ^alpine.* ]] - # no dracut on those flavors, do nothing - ELSE + IF [ -e "/usr/bin/dracut" ] # Regenerate initrd if necessary RUN --no-cache kernel=$(ls /lib/modules | head -n1) && depmod -a "${kernel}" RUN --no-cache kernel=$(ls /lib/modules | head -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd END - # Set /boot/vmlinuz pointing to our kernel so kairos-agent can use it # https://github.com/kairos-io/kairos-agent/blob/0288fb111bc568a1bfca59cb09f39302220475b6/pkg/elemental/elemental.go#L548 q IF [ "$FLAVOR" = "fedora" ] || [ "$FLAVOR" = "rockylinux" ] diff --git a/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin b/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin index 80a379baa..0ed8452cd 100644 --- a/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin +++ b/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin @@ -104,6 +104,10 @@ RUN apt-get install -y -o Dpkg::Options::="--force-overwrite" \ jetson-gpio-common \ python3-jetson-gpio +# Symlinks to make installer work +RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install && \ + ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv + # RUN rm -rf /opt/nvidia/l4t-packages # RUN rm -rf /var/lib/apt/lists/* # RUN useradd -ms /bin/bash jetson From 2a4253db31fbf5dc5e16a1c9cb2b00d9dd1da16d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 30 May 2023 01:00:54 +0200 Subject: [PATCH 18/62] :robot: disable parallelism on arm self-hosted release builds (#1461) :robot: disable parallelism Signed-off-by: mudler --- .github/workflows/release-arm.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release-arm.yaml b/.github/workflows/release-arm.yaml index 63a7c2bfd..15edc0218 100644 --- a/.github/workflows/release-arm.yaml +++ b/.github/workflows/release-arm.yaml @@ -47,6 +47,11 @@ jobs: platforms: all - name: Install Cosign uses: sigstore/cosign-installer@main + - name: Install earthly + uses: Luet-lab/luet-install-action@v1 + with: + repository: quay.io/kairos/packages + packages: utils/earthly - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@master @@ -82,6 +87,9 @@ jobs: insecure = true http = true EOF + ## Otherwise parallel jobs might kill small workers + earthly config "global.conversion_parallelism" "1" + earthly config "global.buildkit_max_parallelism" "1" export TAG=${GITHUB_REF##*/} docker run --privileged -v $HOME/.earthly/config.yml:/etc/.earthly/config.yml -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged +all-arm --IMAGE_NAME=kairos-$FLAVOR-$TAG.img --IMAGE=quay.io/kairos/core-$FLAVOR:$TAG --MODEL=$MODEL --FLAVOR=$FLAVOR From cfef66a7665fe62028107feff86c8e8679600b23 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Tue, 30 May 2023 09:23:41 +0000 Subject: [PATCH 19/62] Build and publish UKI images in our pipeline (#800) --- .github/workflows/release.yaml | 21 +++ Earthfile | 141 +++++++++++++++++-- UKI-experimental.md | 92 ++++++++++++ overlay/files/system/oem/00_rootfs.yaml | 2 +- overlay/files/system/oem/10_accounting.yaml | 4 +- overlay/files/system/oem/11_persistency.yaml | 42 +++++- 6 files changed, 287 insertions(+), 15 deletions(-) create mode 100644 UKI-experimental.md diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9b7fa3d93..90d00d060 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -157,6 +157,27 @@ jobs: with: sarif_file: 'sarif' category: ${{ matrix.flavor }} + build-uki: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: | + git fetch --prune --unshallow + - name: Install earthly + uses: Luet-lab/luet-install-action@v1 + with: + repository: quay.io/kairos/packages + packages: utils/earthly + - name: Build uki image 🔧 + run: | + # Do fedora as its the smaller uki possible + earthly +uki --FLAVOR=fedora + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + build/efi # build-vm-images: # needs: build # runs-on: macos-12 diff --git a/Earthfile b/Earthfile index 8bde70201..1ca0c2e34 100644 --- a/Earthfile +++ b/Earthfile @@ -247,6 +247,7 @@ base-image: ARG MODEL ARG FLAVOR ARG VARIANT + ARG BUILD_INITRD="true" IF [ "$BASE_IMAGE" = "" ] # Source the flavor-provided docker file FROM DOCKERFILE --build-arg MODEL=$MODEL -f images/Dockerfile.$FLAVOR . @@ -330,16 +331,17 @@ base-image: RUN find /usr/lib/modules -type f -name "*.ko" -execdir zstd --rm -9 {} \+ END - - IF [ "$FLAVOR" = "debian" ] - RUN rm -rf /boot/initrd.img-* - END + IF [ "$BUILD_INITRD" = "true" ] + IF [ "$FLAVOR" = "debian" ] + RUN rm -rf /boot/initrd.img-* + END - IF [ -e "/usr/bin/dracut" ] - # Regenerate initrd if necessary - RUN --no-cache kernel=$(ls /lib/modules | head -n1) && depmod -a "${kernel}" - RUN --no-cache kernel=$(ls /lib/modules | head -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd + IF [ -e "/usr/bin/dracut" ] + # Regenerate initrd if necessary + RUN --no-cache kernel=$(ls /lib/modules | head -n1) && depmod -a "${kernel}" + RUN --no-cache kernel=$(ls /lib/modules | head -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd + END END # Set /boot/vmlinuz pointing to our kernel so kairos-agent can use it @@ -366,10 +368,12 @@ base-image: END END + RUN rm -rf /tmp/* image: - FROM +base-image + ARG BUILD_INITRD="true" + FROM +base-image --BUILD_INITRD=$BUILD_INITRD ARG FLAVOR ARG VARIANT ARG MODEL @@ -395,6 +399,125 @@ image-rootfs: FROM +image SAVE ARTIFACT --keep-own /. rootfs +uki-artifacts: + FROM +image --BUILD_INITRD=false + RUN /usr/bin/immucore version + RUN ln -s /usr/bin/immucore /init + RUN find . \( -path ./sys -prune -o -path ./run -prune -o -path ./dev -prune -o -path ./tmp -prune -o -path ./proc -prune \) -o -print | cpio -R root:root -H newc -o | gzip -2 > /tmp/initramfs.cpio.gz + RUN echo "console=tty1 console=ttyS0 net.ifnames=1 rd.immucore.debug rd.immucore.uki selinux=0" > /tmp/Cmdline + RUN basename $(ls /boot/vmlinuz-* |grep -v rescue | head -n1)| sed --expression "s/vmlinuz-//g" > /tmp/Uname + SAVE ARTIFACT /boot/vmlinuz Kernel + SAVE ARTIFACT /etc/os-release Osrelease + SAVE ARTIFACT /tmp/Cmdline Cmdline + SAVE ARTIFACT /tmp/Uname Uname + SAVE ARTIFACT /tmp/initramfs.cpio.gz Initrd + +# Base image for uki operations so we only run the install once +uki-tools-image: + FROM fedora:38 + # objcopy from binutils and systemd-stub from systemd + RUN dnf install -y binutils systemd-boot mtools efitools sbsigntools shim openssl + +uki: + FROM +uki-tools-image + WORKDIR build + COPY +uki-artifacts/Kernel Kernel + COPY +uki-artifacts/Initrd Initrd + COPY +uki-artifacts/Osrelease Osrelease + COPY +uki-artifacts/Uname Uname + COPY +uki-artifacts/Cmdline Cmdline + ARG KVERSION=$(cat Uname) + RUN objcopy /usr/lib/systemd/boot/efi/linuxx64.efi.stub \ + --add-section .osrel=Osrelease --set-section-flags .osrel=data,readonly \ + --add-section .cmdline=Cmdline --set-section-flags .cmdline=data,readonly \ + --add-section .initrd=Initrd --set-section-flags .initrd=data,readonly \ + --add-section .uname=Uname --set-section-flags .uname=data,readonly \ + --add-section .linux=Kernel --set-section-flags .linux=code,readonly \ + $ISO_NAME.unsigned.efi \ + --change-section-vma .osrel=0x17000 \ + --change-section-vma .cmdline=0x18000 \ + --change-section-vma .initrd=0x19000 \ + --change-section-vma .uname=0x5a0ed000 \ + --change-section-vma .linux=0x5a0ee000 + SAVE ARTIFACT Uname Uname + SAVE ARTIFACT $ISO_NAME.unsigned.efi uki.efi AS LOCAL build/$ISO_NAME.unsigned-$KVERSION.efi + + +uki-signed: + FROM +uki-tools-image + # Platform key + RUN openssl req -new -x509 -subj "/CN=Kairos PK/" -days 3650 -nodes -newkey rsa:2048 -sha256 -keyout PK.key -out PK.crt + # CER keys are for FW install + RUN openssl x509 -in PK.crt -out PK.cer -outform DER + # Key exchange + RUN openssl req -new -x509 -subj "/CN=Kairos KEK/" -days 3650 -nodes -newkey rsa:2048 -sha256 -keyout KEK.key -out KEK.crt + # CER keys are for FW install + RUN openssl x509 -in KEK.crt -out KEK.cer -outform DER + # Signature DB + RUN openssl req -new -x509 -subj "/CN=Kairos DB/" -days 3650 -nodes -newkey rsa:2048 -sha256 -keyout DB.key -out DB.crt + # CER keys are for FW install + RUN openssl x509 -in DB.crt -out DB.cer -outform DER + COPY +uki/uki.efi uki.efi + COPY +uki/Uname Uname + ARG KVERSION=$(cat Uname) + + RUN sbsign --key DB.key --cert DB.crt --output uki.signed.efi uki.efi + + + SAVE ARTIFACT /boot/efi/EFI/fedora/mmx64.efi MokManager.efi + SAVE ARTIFACT PK.key PK.key AS LOCAL build/PK.key + SAVE ARTIFACT PK.crt PK.crt AS LOCAL build/PK.crt + SAVE ARTIFACT PK.cer PK.cer AS LOCAL build/PK.cer + SAVE ARTIFACT KEK.key KEK.key AS LOCAL build/KEK.key + SAVE ARTIFACT KEK.crt KEK.crt AS LOCAL build/KEK.crt + SAVE ARTIFACT KEK.cer KEK.cer AS LOCAL build/KEK.cer + SAVE ARTIFACT DB.key DB.key AS LOCAL build/DB.key + SAVE ARTIFACT DB.crt DB.crt AS LOCAL build/DB.crt + SAVE ARTIFACT DB.cer DB.cer AS LOCAL build/DB.cer + SAVE ARTIFACT uki.signed.efi uki.efi AS LOCAL build/$ISO_NAME.signed-$KVERSION.efi + +# This target will prepare a disk.img ready with the uki artifact on it for qemu. Just attach it to qemu and mark you vm to boot from that disk +# here we take advantage of the uefi fallback method, which will load an efi binary in /EFI/BOOT/BOOTX64.efi if there is nothing +# else that it can boot from :D Just make sure to have your disk.img set as boot device in qemu. +prepare-uki-disk-image: + FROM +uki-tools-image + ARG SIGNED_EFI=false + IF [ "$SIGNED_EFI" = "true" ] + COPY +uki-signed/uki.efi . + COPY +uki-signed/PK.key . + COPY +uki-signed/PK.crt . + COPY +uki-signed/PK.cer . + COPY +uki-signed/KEK.key . + COPY +uki-signed/KEK.crt . + COPY +uki-signed/KEK.cer . + COPY +uki-signed/DB.key . + COPY +uki-signed/DB.crt . + COPY +uki-signed/DB.cer . + COPY +uki-signed/MokManager.efi . + ELSE + COPY +uki/uki.efi . + END + RUN dd if=/dev/zero of=disk.img bs=1G count=1 + RUN mformat -i disk.img -F :: + RUN mmd -i disk.img ::/EFI + RUN mmd -i disk.img ::/EFI/BOOT + RUN mcopy -i disk.img uki.efi ::/EFI/BOOT/BOOTX64.efi + IF [ "$SIGNED_EFI" = "true" ] + RUN mcopy -i disk.img PK.key ::/EFI/BOOT/PK.key + RUN mcopy -i disk.img PK.crt ::/EFI/BOOT/PK.crt + RUN mcopy -i disk.img PK.cer ::/EFI/BOOT/PK.cer + RUN mcopy -i disk.img KEK.key ::/EFI/BOOT/KEK.key + RUN mcopy -i disk.img KEK.crt ::/EFI/BOOT/KEK.crt + RUN mcopy -i disk.img KEK.cer ::/EFI/BOOT/KEK.cer + RUN mcopy -i disk.img DB.key ::/EFI/BOOT/DB.key + RUN mcopy -i disk.img DB.crt ::/EFI/BOOT/DB.crt + RUN mcopy -i disk.img DB.cer ::/EFI/BOOT/DB.cer + RUN mcopy -i disk.img MokManager.efi ::/EFI/BOOT/mmx64.efi + END + RUN mdir -i disk.img ::/EFI/BOOT + SAVE ARTIFACT disk.img AS LOCAL build/disk.img + + ### ### Artifacts targets (ISO, netboot, ARM) ### diff --git a/UKI-experimental.md b/UKI-experimental.md new file mode 100644 index 000000000..ab50a68fd --- /dev/null +++ b/UKI-experimental.md @@ -0,0 +1,92 @@ +# UKI: Unified Kernel Image + + +It's basically a kernel, initrd and cmdline for the kernel all lumped up together in an efi binary. Mixing it with something like systemd-stub +means that you can boot from the EFI shell directly into the system. + +You can add more stuff to it like the os-release info, the kernel version (uname), splash image, Devicetree , etc... + +This way you got everything in one nice package and can sign the whole thing for secureboot or calculate the hashes for measured boot. + + +Usually under secureboot the initrd is not signed (as its generated locally), so once the kernel is run initrd signature is not verified. Nor you can measure it with TPM PCRs + +UKI bundles the kernel with initrd and everything else, so you can sign the whole thing AND pre-calculate the hashes for TPM PCRs in advance. + + +Good writeup: https://0pointer.net/blog/brave-new-trusted-boot-world.html + + +### So why not a bit more? + +So why not store the whole system in the initramfs? + +In this branch on the earthfile there is a new target called uki. This will generate an efi with the whole kairos system under the initramfs. +This uses immucore to mount and set up the whole system. + +There is an extra target called `prepare-uki-disk-image` which will generate a disk.img with the efi file inside in the proper place, so you +can just attach that image to a qemu vm and boot from there. An extra arg `SIGNED_EFI` will provide the same image but with a signed efi and all the keys needed +to insert hem into the uefo firmware and test secureboot. + +The only special thing the target does is use objcopy to add sections to the systemd-stub pointing to the correct data: + +```bash +RUN objcopy /usr/lib/systemd/boot/efi/linuxx64.efi.stub \ + --add-section .osrel=Osrelease --set-section-flags .osrel=data,readonly \ + --add-section .cmdline=Cmdline --set-section-flags .cmdline=data,readonly \ + --add-section .initrd=Initrd --set-section-flags .initrd=data,readonly \ + --add-section .uname=Uname --set-section-flags .uname=data,readonly \ + --add-section .linux=Kernel --set-section-flags .linux=code,readonly \ + $ISO_NAME.unsigned.efi \ + --change-section-vma .osrel=0x17000 \ + --change-section-vma .cmdline=0x18000 \ + --change-section-vma .initrd=0x19000 \ + --change-section-vma .uname=0x5a0ed000 \ + --change-section-vma .linux=0x5a0ee000 +``` + +Where: +* Kernel is the kernel that will be booted. + * Initrd is the initramfs that will be booted by the kernel. Currently, a dump of the docker-rootfs...rootfs + * Uname the output of `uname -r` (Optional content) + * Osrelease is the /etc/os-release file from the kairos rootfs (Optional content) + * Cmdline is the line to be passed to the kernel (Optional content, but needed in our case) + + +# Running the efi locally with qemu + +For ease of use there is a target in the earthly file that will generate a disk.img with the efi inside. +Run `earthly +prepare-disk-image` and you will get a `build/disk.img` ready to be consumed + +To run it under qemu use the following arguments: + +```bash +qemu-system-x86_64 -bios $EFI_FIRMWARE -accel kvm -cpu host -m $MEMORY -machine pc \ +-drive file=disk.img,if=none,index=0,media=disk,format=raw,id=disk1 -device virtio-blk-pci,drive=disk1,bootindex=0 \ +-boot menu=on +``` + +Where `$EFI_FIRMWARE` is the OVMF efi firmware and `$MEMORY` is at least 4000. + + +Note that you can also build the uki image signed by passing the `--SIGNED_EFI=true` to earthly. That would produce the same +`build/disk.img` but with some extra files inside, like the certificates needed to be added to the firmware and the MokManager util to install those certificates. + +With those certs and MokManager is possible to install the generated certs to test booting with SecureBoot enabled. + +Note that the `$EFI_FIRMWARE` needs to be set to the OVMF SecureBoot enabled file to test SecureBoot. + +For example under Fedora, the normal firmware with no SecureBoot is found at `/usr/share/edk2/ovmf/OVMF_CODE.fd` while +the SecureBoot enabled one is `/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd` + +Good links: + + - https://man.archlinux.org/man/systemd-stub.7 + - https://wiki.osdev.org/UEFI#UEFI_applications_in_detail + - https://github.com/uapi-group/specifications/blob/main/specs/unified_kernel_image.md + - https://man.archlinux.org/man/systemd-measure.1.en + - https://manuais.iessanclemente.net/images/a/a6/EFI-ShellCommandManual.pdf + - https://0pointer.net/blog/brave-new-trusted-boot-world.html + + + diff --git a/overlay/files/system/oem/00_rootfs.yaml b/overlay/files/system/oem/00_rootfs.yaml index a2a9e7272..72d5568a1 100644 --- a/overlay/files/system/oem/00_rootfs.yaml +++ b/overlay/files/system/oem/00_rootfs.yaml @@ -13,7 +13,7 @@ stages: providers: ["aws", "gcp", "openstack", "cdrom"] path: "/oem" rootfs: - - if: '[ ! -f "/run/cos/recovery_mode" ]' + - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -e "/run/cos/uki_mode" ]' name: "Layout configuration" environment_file: /run/cos/cos-layout.env environment: diff --git a/overlay/files/system/oem/10_accounting.yaml b/overlay/files/system/oem/10_accounting.yaml index 301101589..b0dcd93ba 100644 --- a/overlay/files/system/oem/10_accounting.yaml +++ b/overlay/files/system/oem/10_accounting.yaml @@ -16,8 +16,8 @@ stages: homedir: "/home/kairos" groups: - "admin" - - name: "Set user password if running in live" - if: "[ -e /run/cos/live_mode ]" + - name: "Set user password if running in live or uki" + if: "[ -e /run/cos/live_mode ] || [ -e /run/cos/uki_mode ]" users: kairos: passwd: "kairos" diff --git a/overlay/files/system/oem/11_persistency.yaml b/overlay/files/system/oem/11_persistency.yaml index 846050d46..a7b48f82d 100644 --- a/overlay/files/system/oem/11_persistency.yaml +++ b/overlay/files/system/oem/11_persistency.yaml @@ -1,8 +1,44 @@ name: "Configure persistent dirs bind-mounts" stages: rootfs.after: - - if: '[ ! -f "/run/cos/recovery_mode" ]' - name: "Layout configuration" + - if: '[ -e "/run/cos/uki_mode" ]' + # omit the persistent partition on uki mode + # And mount all persistent mounts under the overlay + name: "Layout configuration for UKI" + environment_file: /run/cos/cos-layout.env + environment: + RW_PATHS: "/var /etc /srv /usr" + OVERLAY: "tmpfs:25%" + PERSISTENT_STATE_PATHS: >- + /var + /etc + /etc/systemd + /etc/modprobe.d + /etc/rancher + /etc/sysconfig + /etc/runlevels + /etc/ssh + /etc/ssl/certs + /etc/iscsi + /etc/cni + /etc/kubernetes + /home + /opt + /root + /var/snap + /usr/libexec + /var/log + /var/lib/rancher + /var/lib/kubelet + /var/lib/snapd + /var/lib/wicked + /var/lib/longhorn + /var/lib/cni + /usr/share/pki/trust + /usr/share/pki/trust/anchors + /var/lib/ca-certificates + - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -e "/run/cos/uki_mode" ]' + name: "Layout configuration for active/passive" environment_file: /run/cos/cos-layout.env environment: VOLUMES: "LABEL=COS_OEM:/oem LABEL=COS_PERSISTENT:/usr/local" @@ -56,7 +92,7 @@ stages: echo PERSISTENT_STATE_PATHS=\"${PERSISTENT_STATE_PATHS}\" >> /run/cos/cos-layout.env - if: | cat /proc/cmdline | grep -q "kairos.boot_live_mode" - name: "Layout configuration" + name: "Layout configuration for boot_live_mode" environment_file: /run/cos/cos-layout.env environment: VOLUMES: "LABEL=COS_OEM:/oem LABEL=COS_PERSISTENT:/usr/local" From 9a0c7385c925f09629c78c0f68f136187e6492c5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 30 May 2023 11:24:30 +0200 Subject: [PATCH 20/62] :robot: run release-arm build on self-hosted (#1462) Signed-off-by: mudler --- .github/workflows/release-arm.yaml | 41 ++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-arm.yaml b/.github/workflows/release-arm.yaml index 15edc0218..b92269edb 100644 --- a/.github/workflows/release-arm.yaml +++ b/.github/workflows/release-arm.yaml @@ -22,7 +22,7 @@ jobs: # end of optional handling for multi line json echo "::set-output name=matrix::{\"include\": $content }" docker: - runs-on: ubuntu-latest + runs-on: ${{ matrix.worker }} needs: - get-matrix permissions: @@ -35,9 +35,38 @@ jobs: matrix: ${{fromJson(needs.get-matrix.outputs.matrix)}} steps: - name: Release space from worker + if: ${{ matrix.worker != 'self-hosted' }} run: | - sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android - sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET + echo "Listing top largest packages" + pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr) + head -n 30 <<< "${pkgs}" + echo + df -h + echo + sudo apt-get remove -y '^llvm-.*|^libllvm.*' || true + sudo apt-get remove --auto-remove android-sdk-platform-tools || true + sudo apt-get purge --auto-remove android-sdk-platform-tools || true + sudo rm -rf /usr/local/lib/android + sudo apt-get remove -y '^dotnet-.*|^aspnetcore-.*' || true + sudo rm -rf /usr/share/dotnet + sudo apt-get remove -y '^mono-.*' || true + sudo apt-get remove -y '^ghc-.*' || true + sudo apt-get remove -y '.*jdk.*|.*jre.*' || true + sudo apt-get remove -y 'php.*' || true + sudo apt-get remove -y hhvm powershell firefox monodoc-manual msbuild || true + sudo apt-get remove -y '^google-.*' || true + sudo apt-get remove -y azure-cli || true + sudo apt-get remove -y '^mongo.*-.*|^postgresql-.*|^mysql-.*|^mssql-.*' || true + sudo apt-get remove -y '^gfortran-.*' || true + sudo apt-get autoremove -y + sudo apt-get clean + echo + echo "Listing top largest packages" + pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr) + head -n 30 <<< "${pkgs}" + echo + sudo rm -rfv build || true + df -h - uses: actions/checkout@v3 - run: | git fetch --prune --unshallow @@ -61,7 +90,7 @@ jobs: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - - name: Build 🔧 + - name: Standard Build 🔧 if: ${{ matrix.worker != 'self-hosted' }} env: FLAVOR: ${{ matrix.flavor }} @@ -87,12 +116,8 @@ jobs: insecure = true http = true EOF - ## Otherwise parallel jobs might kill small workers - earthly config "global.conversion_parallelism" "1" - earthly config "global.buildkit_max_parallelism" "1" export TAG=${GITHUB_REF##*/} docker run --privileged -v $HOME/.earthly/config.yml:/etc/.earthly/config.yml -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged +all-arm --IMAGE_NAME=kairos-$FLAVOR-$TAG.img --IMAGE=quay.io/kairos/core-$FLAVOR:$TAG --MODEL=$MODEL --FLAVOR=$FLAVOR - - name: Push 🔧 env: FLAVOR: ${{ matrix.flavor }} From 4e555cd35ca1558987885cd07b10321b3dbadc95 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 16:15:44 +0200 Subject: [PATCH 21/62] Update robinraju/release-downloader action to v1.8 (#1376) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 1b4102bba..281da62ff 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -263,7 +263,7 @@ jobs: latest-release: runs-on: ubuntu-latest steps: - - uses: robinraju/release-downloader@v1.7 + - uses: robinraju/release-downloader@v1.8 with: # A flag to set the download target as latest release # The default value is 'false' From 62c67e3e61d49435c362014522e5c6696335376f Mon Sep 17 00:00:00 2001 From: Itxaka Date: Wed, 31 May 2023 06:25:57 +0000 Subject: [PATCH 22/62] Yaml cleanup (#1453) --- Earthfile | 4 +- .../files-alpine/system/oem/23_rootfs.yaml | 6 - .../etc/elemental/config.yaml | 21 --- .../system/oem/03-setupcon.yaml | 7 - .../files-ubuntu/etc/default/console-setup | 7 - .../files-ubuntu/system/oem/03-setupcon.yaml | 7 - .../etc/default/console-setup | 0 overlay/files/system/oem/00_rootfs.yaml | 154 +++++++++++++++++- overlay/files/system/oem/03-setupcon.yaml | 7 + overlay/files/system/oem/03_branding.yaml | 29 ---- overlay/files/system/oem/06_recovery.yaml | 30 ---- overlay/files/system/oem/07_live.yaml | 18 -- overlay/files/system/oem/09_services.yaml | 35 +++- overlay/files/system/oem/10_accounting.yaml | 12 +- overlay/files/system/oem/11_persistency.yaml | 142 ---------------- overlay/files/system/oem/21_kcrypt.yaml | 5 - .../files/system/oem/25_default_paths.yaml | 10 -- overlay/files/system/oem/27_harderning.yaml | 13 -- .../system/oem/32_journal_persistency.yaml | 7 - ...20_recovery_mode.yaml => 50_recovery.yaml} | 11 ++ .../oem/{20_reset_mode.yaml => 51_reset.yaml} | 0 .../{04_installer.yaml => 52_installer.yaml} | 0 22 files changed, 206 insertions(+), 319 deletions(-) delete mode 100644 overlay/files-alpine/system/oem/23_rootfs.yaml delete mode 100644 overlay/files-ubuntu-arm-rpi/etc/elemental/config.yaml delete mode 100644 overlay/files-ubuntu-arm-rpi/system/oem/03-setupcon.yaml delete mode 100644 overlay/files-ubuntu/etc/default/console-setup delete mode 100644 overlay/files-ubuntu/system/oem/03-setupcon.yaml rename overlay/{files-ubuntu-arm-rpi => files}/etc/default/console-setup (100%) create mode 100644 overlay/files/system/oem/03-setupcon.yaml delete mode 100644 overlay/files/system/oem/03_branding.yaml delete mode 100644 overlay/files/system/oem/06_recovery.yaml delete mode 100644 overlay/files/system/oem/07_live.yaml delete mode 100644 overlay/files/system/oem/11_persistency.yaml delete mode 100644 overlay/files/system/oem/25_default_paths.yaml delete mode 100644 overlay/files/system/oem/27_harderning.yaml delete mode 100644 overlay/files/system/oem/32_journal_persistency.yaml rename overlay/files/system/oem/{20_recovery_mode.yaml => 50_recovery.yaml} (60%) rename overlay/files/system/oem/{20_reset_mode.yaml => 51_reset.yaml} (100%) rename overlay/files/system/oem/{04_installer.yaml => 52_installer.yaml} (100%) diff --git a/Earthfile b/Earthfile index 1ca0c2e34..232f7923e 100644 --- a/Earthfile +++ b/Earthfile @@ -217,10 +217,8 @@ framework: COPY overlay/files-alpine/ /framework ELSE IF [ "$FLAVOR" = "fedora" ] || [ "$FLAVOR" = "rockylinux" ] COPY overlay/files-fedora/ /framework - ELSE IF [ "$FLAVOR" = "debian" ] || [ "$FLAVOR" = "ubuntu" ] || [ "$FLAVOR" = "ubuntu-20-lts" ] || [ "$FLAVOR" = "ubuntu-22-lts" ] + ELSE IF [ "$FLAVOR" = "debian" ] || [ "$FLAVOR" = "ubuntu" ] || [ "$FLAVOR" = "ubuntu-20-lts" ] || [ "$FLAVOR" = "ubuntu-22-lts" ] || [[ "$FLAVOR" =~ ^ubuntu-.*-lts-arm-.*$ ]] COPY overlay/files-ubuntu/ /framework - ELSE IF [[ "$FLAVOR" =~ ^ubuntu-.*-lts-arm-.*$ ]] - COPY overlay/files-ubuntu-arm-rpi/ /framework END IF [[ "$FLAVOR" = "ubuntu-20-lts-arm-nvidia-jetson-agx-orin" ]] diff --git a/overlay/files-alpine/system/oem/23_rootfs.yaml b/overlay/files-alpine/system/oem/23_rootfs.yaml deleted file mode 100644 index a5ae0f845..000000000 --- a/overlay/files-alpine/system/oem/23_rootfs.yaml +++ /dev/null @@ -1,6 +0,0 @@ -name: "FS Layout setup" -stages: - boot.before: - - commands: - - mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp - - mount --make-rshared / diff --git a/overlay/files-ubuntu-arm-rpi/etc/elemental/config.yaml b/overlay/files-ubuntu-arm-rpi/etc/elemental/config.yaml deleted file mode 100644 index 5ac3b31c4..000000000 --- a/overlay/files-ubuntu-arm-rpi/etc/elemental/config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -cosign: false -verify: false -install: - grub-entry-name: "kairos" - system: - size: 2300 - recovery-system: - size: 2300 -upgrade: - grub-entry-name: "kairos" - recovery-system: - size: 2300 - system: - size: 2300 -reset: - grub-entry-name: "kairos" - system: - size: 2300 - -cloud-init-paths: -- /run/initramfs/cos-state diff --git a/overlay/files-ubuntu-arm-rpi/system/oem/03-setupcon.yaml b/overlay/files-ubuntu-arm-rpi/system/oem/03-setupcon.yaml deleted file mode 100644 index dba44cfbf..000000000 --- a/overlay/files-ubuntu-arm-rpi/system/oem/03-setupcon.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: "Start installer on tty1" -stages: - initramfs.after: - - name: "setupcon initramfs.after" - commands: - - echo "Run setupcon to have fonts working on Ubuntu" - - setupcon diff --git a/overlay/files-ubuntu/etc/default/console-setup b/overlay/files-ubuntu/etc/default/console-setup deleted file mode 100644 index ebe02a396..000000000 --- a/overlay/files-ubuntu/etc/default/console-setup +++ /dev/null @@ -1,7 +0,0 @@ -ACTIVE_CONSOLES="/dev/tty[1-6]" -FONT="default8x16.psf.gz" -CHARMAP="UTF-8" -CODESET="guess" -FONTFACE="default" -FONTSIZE="8x16" -SCREEN_WIDTH="100" diff --git a/overlay/files-ubuntu/system/oem/03-setupcon.yaml b/overlay/files-ubuntu/system/oem/03-setupcon.yaml deleted file mode 100644 index dba44cfbf..000000000 --- a/overlay/files-ubuntu/system/oem/03-setupcon.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: "Start installer on tty1" -stages: - initramfs.after: - - name: "setupcon initramfs.after" - commands: - - echo "Run setupcon to have fonts working on Ubuntu" - - setupcon diff --git a/overlay/files-ubuntu-arm-rpi/etc/default/console-setup b/overlay/files/etc/default/console-setup similarity index 100% rename from overlay/files-ubuntu-arm-rpi/etc/default/console-setup rename to overlay/files/etc/default/console-setup diff --git a/overlay/files/system/oem/00_rootfs.yaml b/overlay/files/system/oem/00_rootfs.yaml index 72d5568a1..d3557c8d1 100644 --- a/overlay/files/system/oem/00_rootfs.yaml +++ b/overlay/files/system/oem/00_rootfs.yaml @@ -14,19 +14,153 @@ stages: path: "/oem" rootfs: - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -e "/run/cos/uki_mode" ]' - name: "Layout configuration" + name: "Layout configuration for active/passive mode" environment_file: /run/cos/cos-layout.env environment: - VOLUMES: "LABEL=COS_OEM:/oem LABEL=COS_PERSISTENT:/usr/local" + VOLUMES: "LABEL=COS_PERSISTENT:/usr/local" OVERLAY: "tmpfs:25%" + RW_PATHS: "/var /etc /srv" + PERSISTENT_STATE_PATHS: >- + /etc/systemd + /etc/modprobe.d + /etc/rancher + /etc/sysconfig + /etc/runlevels + /etc/ssh + /etc/ssl/certs + /etc/iscsi + /etc/zfs + /etc/cni + /etc/kubernetes + /home + /opt + /root + /snap + /var/snap + /usr/libexec + /var/log + /var/lib/rancher + /var/lib/kubelet + /var/lib/snapd + /var/lib/wicked + /var/lib/longhorn + /var/lib/cni + /var/lib/dbus + /usr/share/pki/trust + /usr/share/pki/trust/anchors + /var/lib/ca-certificates + PERSISTENT_STATE_BIND: "true" - if: '[ -f "/run/cos/recovery_mode" ]' # omit the persistent partition on recovery mode - name: "Layout configuration" + name: "Layout configuration for recovery mode" environment_file: /run/cos/cos-layout.env environment: - VOLUMES: "LABEL=COS_OEM:/oem" OVERLAY: "tmpfs:25%" + - if: grep -q "kairos.boot_live_mode" /proc/cmdline + name: "Layout configuration for booting local node from livecd" + environment_file: /run/cos/cos-layout.env + environment: + VOLUMES: "LABEL=COS_PERSISTENT:/usr/local" + OVERLAY: "tmpfs:25%" + RW_PATHS: "/var /etc /srv" + PERSISTENT_STATE_PATHS: >- + /etc/systemd + /etc/modprobe.d + /etc/rancher + /etc/sysconfig + /etc/runlevels + /etc/ssh + /etc/ssl/certs + /etc/iscsi + /etc/cni + /etc/kubernetes + /home + /opt + /root + /snap + /var/snap + /usr/libexec + /var/log + /var/lib/rancher + /var/lib/kubelet + /var/lib/snapd + /var/lib/wicked + /var/lib/longhorn + /var/lib/cni + /usr/share/pki/trust + /usr/share/pki/trust/anchors + /var/lib/ca-certificates + PERSISTENT_STATE_BIND: "true" + - if: '[ -e "/run/cos/uki_mode" ]' + # omit the persistent partition on uki mode + # And mount all persistent mounts under the overlay + name: "Layout configuration for UKI" + environment_file: /run/cos/cos-layout.env + environment: + RW_PATHS: "/var /etc /srv /usr" + OVERLAY: "tmpfs:25%" + PERSISTENT_STATE_PATHS: >- + /var + /etc + /etc/systemd + /etc/modprobe.d + /etc/rancher + /etc/sysconfig + /etc/runlevels + /etc/ssh + /etc/ssl/certs + /etc/iscsi + /etc/cni + /etc/kubernetes + /home + /opt + /root + /var/snap + /usr/libexec + /var/log + /var/lib/rancher + /var/lib/kubelet + /var/lib/snapd + /var/lib/wicked + /var/lib/longhorn + /var/lib/cni + /usr/share/pki/trust + /usr/share/pki/trust/anchors + /var/lib/ca-certificates + rootfs.after: + - if: '[ -r /run/cos/custom-layout.env ] && [ ! -f "/run/cos/recovery_mode" ] && [ ! -f /run/cos/live_mode ]' + name: "add custom bind and ephemeral mounts to /run/cos/cos-layout.env" + commands: + - | + source /run/cos/cos-layout.env + source /run/cos/custom-layout.env + PERSISTENT_STATE_PATHS="${PERSISTENT_STATE_PATHS} ${CUSTOM_BIND_MOUNTS} " + RW_PATHS="${RW_PATHS} ${CUSTOM_EPHEMERAL_MOUNTS}" + # Remove the existing lines that we are gonna rewrite + sed -i "/RW_PATHS/d" /run/cos/cos-layout.env + sed -i "/PERSISTENT_STATE_PATHS/d" /run/cos/cos-layout.env + # Add the new lines + echo "# rw paths with user bind mounts" >> /run/cos/cos-layout.env + echo RW_PATHS=\"${RW_PATHS}\" >> /run/cos/cos-layout.env + echo "# persistent state paths with user ephemeral mounts" >> /run/cos/cos-layout.env + echo PERSISTENT_STATE_PATHS=\"${PERSISTENT_STATE_PATHS}\" >> /run/cos/cos-layout.env + - if: '[ ! -f /run/cos/recovery_mode ] && [ ! -f /run/cos/live_mode ]' + name: "Grow persistent" + layout: + device: + label: COS_PERSISTENT + expand_partition: + # Size 0 is required to specify all remaining space + size: 0 initramfs: + - name: "Mount BPF" + if: '[ ! -e "/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ] && [ ! -e "/usr/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ]' + commands: + - mount bpffs -o rw,nosuid,nodev,noexec,relatime,mode=700 /sys/fs/bpf -t bpf + - name: "Create journalctl /var/log/journal dir" + if: '[ -e "/sbin/systemctl" ] || [ -e "/bin/systemctl" ] || [ -e "/usr/sbin/systemctl" ] || [ -e "/usr/bin/systemctl" ]' + directories: + - path: /var/log/journal - if: '[ ! -f "/run/cos/recovery_mode" ] && [ -s /usr/local/etc/machine-id ]' name: "Restore /etc/machine-id" commands: @@ -48,3 +182,15 @@ stages: - | mkdir -p /var/lib/dbus/ cp /etc/machine-id /var/lib/dbus/ + fs.after: + - if: "[ ! -d /usr/local/cloud-config ]" + name: "Ensure /usr/local/cloud-config exists" + commands: + - mkdir /usr/local/cloud-config + - chmod 600 /usr/local/cloud-config + boot.before: + - name: "Mount tmp on alpine" + if: "[[ $(kairos-agent state get kairos.flavor) =~ ^alpine ]]" + commands: + - mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp + - mount --make-rshared / diff --git a/overlay/files/system/oem/03-setupcon.yaml b/overlay/files/system/oem/03-setupcon.yaml new file mode 100644 index 000000000..09cf2e078 --- /dev/null +++ b/overlay/files/system/oem/03-setupcon.yaml @@ -0,0 +1,7 @@ +name: "Fonts fix for ubuntu" +stages: + initramfs.after: + - if: "[[ $(kairos-agent state get kairos.flavor) =~ ^ubuntu ]]" + name: "setupcon initramfs.after ubuntu" + commands: + - setupcon diff --git a/overlay/files/system/oem/03_branding.yaml b/overlay/files/system/oem/03_branding.yaml deleted file mode 100644 index 53c3a1af5..000000000 --- a/overlay/files/system/oem/03_branding.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: "Branding" -stages: - boot: - - name: "Default sysctl settings" - sysctl: - net.core.rmem_max: 2500000 - vm.max_map_count: 262144 - initramfs: - - name: "Default mounts" - if: '[ ! -e "/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ] && [ ! -e "/usr/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ]' - commands: - - mount bpffs -o rw,nosuid,nodev,noexec,relatime,mode=700 /sys/fs/bpf -t bpf - - name: "Default systemd config" - if: '[ -e "/sbin/systemctl" ] || [ -e "/usr/bin/systemctl" ] || [ -e "/usr/sbin/systemctl" ] || [ -e "/usr/bin/systemctl" ]' - systemctl: - enable: - - multi-user.target - - getty@tty1 - - iscsid - - systemd-timesyncd - - nohang - - nohang-desktop - - fail2ban - - logrotate.timer - - sshd - mask: - - purge-kernels - commands: - - systemctl set-default multi-user.target diff --git a/overlay/files/system/oem/06_recovery.yaml b/overlay/files/system/oem/06_recovery.yaml deleted file mode 100644 index ba9676c2a..000000000 --- a/overlay/files/system/oem/06_recovery.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Default Kairos OEM configuration file -# -# This file is part of Kairos and will get reset during upgrades. -# -# Before you change this file manually, -# consider copying this file to /usr/local/cloud-config or -# copy the file with a prefix starting by 90, e.g. /oem/91_custom.yaml -name: "Recovery partition boot setup" -stages: - rootfs.before: - - if: | - [ -n "$(blkid -L COS_SYSTEM || true)" ] || cat /proc/cmdline | grep -q "COS_RECOVERY" - name: "Identify recovery mode" - files: - - path: /run/cos/recovery_mode - content: "1" - permissions: 0600 - owner: 0 - group: 0 - boot: - - name: "Recovery" - if: '[ -f "/run/cos/recovery_mode" ]' - hostname: "cos-recovery" - commands: - - | - source /etc/os-release - echo >> /etc/issue - echo "You are booting from recovery mode. Run 'kairos-agent reset' to reset the system to $VERSION" >> /etc/issue - echo " or 'kairos-agent upgrade' to upgrade the active partition" >> /etc/issue - echo >> /etc/issue diff --git a/overlay/files/system/oem/07_live.yaml b/overlay/files/system/oem/07_live.yaml deleted file mode 100644 index 693eb68e6..000000000 --- a/overlay/files/system/oem/07_live.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Before you change this file manually, -# consider copying this file to /usr/local/cloud-config or -# -# This file is part of Kairos and will get reset during upgrades. -# Rootfs Kairos OEM configuration file -# copy the file with a prefix starting by 90, e.g. /oem/91_custom.yaml -name: LiveCD Detection -stages: - rootfs.before: - - if: | - cat /proc/cmdline | grep -q "CDLABEL" || cat /proc/cmdline | grep -q "rd.cos.disable" - name: Identify live mode - files: - - path: /run/cos/live_mode - content: '1' - permissions: 0600 - owner: 0 - group: 0 diff --git a/overlay/files/system/oem/09_services.yaml b/overlay/files/system/oem/09_services.yaml index ea2f19a20..a6a0507a4 100644 --- a/overlay/files/system/oem/09_services.yaml +++ b/overlay/files/system/oem/09_services.yaml @@ -4,13 +4,30 @@ name: "Default config" stages: + boot: + - name: "Default sysctl settings" + sysctl: + net.core.rmem_max: 2500000 + vm.max_map_count: 262144 initramfs: - - name: "Default systemd config" - if: '[ -e "/sbin/systemctl" ] || [ -e "/bin/systemctl" ] || [ -e "/usr/sbin/systemctl" ] || [ -e "/usr/bin/systemctl" ]' - systemctl: - enable: - - systemd-timesyncd - - name: "Generate host keys" - # Make sure the host always has available keys - commands: - - ssh-keygen -A + - name: "Default systemd config" + if: '[ -e "/sbin/systemctl" ] || [ -e "/usr/bin/systemctl" ] || [ -e "/usr/sbin/systemctl" ] || [ -e "/usr/bin/systemctl" ]' + systemctl: + enable: + - multi-user.target + - getty@tty1 + - iscsid + - systemd-timesyncd + - nohang + - nohang-desktop + - fail2ban + - logrotate.timer + - sshd + mask: + - purge-kernels + commands: + - systemctl set-default multi-user.target + - name: "Generate host keys" + # Make sure the host always has available keys + commands: + - ssh-keygen -A diff --git a/overlay/files/system/oem/10_accounting.yaml b/overlay/files/system/oem/10_accounting.yaml index b0dcd93ba..de2e58baf 100644 --- a/overlay/files/system/oem/10_accounting.yaml +++ b/overlay/files/system/oem/10_accounting.yaml @@ -1,4 +1,4 @@ -name: "Default user" +name: "Default user and permissions" stages: initramfs: - name: "Setup groups" @@ -38,3 +38,13 @@ stages: #includedir /etc/sudoers.d commands: - passwd -l root + - name: "Ensure runtime permission" + if: '[ -e "/oem" ]' + commands: + - chown -R root:admin /oem + - chmod 770 /oem + - name: "Ensure runtime permission" + if: '[ -e "/usr/local/cloud-config" ]' + commands: + - chown -R root:admin /usr/local/cloud-config + - chmod 770 /usr/local/cloud-config diff --git a/overlay/files/system/oem/11_persistency.yaml b/overlay/files/system/oem/11_persistency.yaml deleted file mode 100644 index a7b48f82d..000000000 --- a/overlay/files/system/oem/11_persistency.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "Configure persistent dirs bind-mounts" -stages: - rootfs.after: - - if: '[ -e "/run/cos/uki_mode" ]' - # omit the persistent partition on uki mode - # And mount all persistent mounts under the overlay - name: "Layout configuration for UKI" - environment_file: /run/cos/cos-layout.env - environment: - RW_PATHS: "/var /etc /srv /usr" - OVERLAY: "tmpfs:25%" - PERSISTENT_STATE_PATHS: >- - /var - /etc - /etc/systemd - /etc/modprobe.d - /etc/rancher - /etc/sysconfig - /etc/runlevels - /etc/ssh - /etc/ssl/certs - /etc/iscsi - /etc/cni - /etc/kubernetes - /home - /opt - /root - /var/snap - /usr/libexec - /var/log - /var/lib/rancher - /var/lib/kubelet - /var/lib/snapd - /var/lib/wicked - /var/lib/longhorn - /var/lib/cni - /usr/share/pki/trust - /usr/share/pki/trust/anchors - /var/lib/ca-certificates - - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -e "/run/cos/uki_mode" ]' - name: "Layout configuration for active/passive" - environment_file: /run/cos/cos-layout.env - environment: - VOLUMES: "LABEL=COS_OEM:/oem LABEL=COS_PERSISTENT:/usr/local" - OVERLAY: "tmpfs:25%" - RW_PATHS: "/var /etc /srv" - PERSISTENT_STATE_PATHS: >- - /etc/systemd - /etc/modprobe.d - /etc/rancher - /etc/sysconfig - /etc/runlevels - /etc/ssh - /etc/ssl/certs - /etc/iscsi - /etc/zfs - /etc/cni - /etc/kubernetes - /home - /opt - /root - /snap - /var/snap - /usr/libexec - /var/log - /var/lib/rancher - /var/lib/kubelet - /var/lib/snapd - /var/lib/wicked - /var/lib/longhorn - /var/lib/cni - /var/lib/dbus - /usr/share/pki/trust - /usr/share/pki/trust/anchors - /var/lib/ca-certificates - PERSISTENT_STATE_BIND: "true" - - if: '[ -r /run/cos/custom-layout.env ] && [ ! -f "/run/cos/recovery_mode" ] && [ ! -f /run/cos/live_mode ]' - name: "add custom bind and ephemeral mounts to /run/cos/cos-layout.env" - commands: - - | - source /run/cos/cos-layout.env - source /run/cos/custom-layout.env - PERSISTENT_STATE_PATHS="${PERSISTENT_STATE_PATHS} ${CUSTOM_BIND_MOUNTS} " - RW_PATHS="${RW_PATHS} ${CUSTOM_EPHEMERAL_MOUNTS}" - # Remove the existing lines that we are gonna rewrite - sed -i "/RW_PATHS/d" /run/cos/cos-layout.env - sed -i "/PERSISTENT_STATE_PATHS/d" /run/cos/cos-layout.env - # Add the new lines - echo "# rw paths with user bind mounts" >> /run/cos/cos-layout.env - echo RW_PATHS=\"${RW_PATHS}\" >> /run/cos/cos-layout.env - echo "# persistent state paths with user ephemeral mounts" >> /run/cos/cos-layout.env - echo PERSISTENT_STATE_PATHS=\"${PERSISTENT_STATE_PATHS}\" >> /run/cos/cos-layout.env - - if: | - cat /proc/cmdline | grep -q "kairos.boot_live_mode" - name: "Layout configuration for boot_live_mode" - environment_file: /run/cos/cos-layout.env - environment: - VOLUMES: "LABEL=COS_OEM:/oem LABEL=COS_PERSISTENT:/usr/local" - OVERLAY: "tmpfs:25%" - RW_PATHS: "/var /etc /srv" - PERSISTENT_STATE_PATHS: >- - /etc/systemd - /etc/modprobe.d - /etc/rancher - /etc/sysconfig - /etc/runlevels - /etc/ssh - /etc/ssl/certs - /etc/iscsi - /etc/cni - /etc/kubernetes - /home - /opt - /root - /snap - /var/snap - /usr/libexec - /var/log - /var/lib/rancher - /var/lib/kubelet - /var/lib/snapd - /var/lib/wicked - /var/lib/longhorn - /var/lib/cni - /usr/share/pki/trust - /usr/share/pki/trust/anchors - /var/lib/ca-certificates - PERSISTENT_STATE_BIND: "true" - - if: '[ ! -f /run/cos/recovery_mode ] && [ ! -f /run/cos/live_mode ]' - name: "Grow persistent" - layout: - device: - label: COS_PERSISTENT - expand_partition: - # Size 0 is required to specify all remaining space - size: 0 - fs.after: - - if: "[ ! -d /usr/local/cloud-config ]" - commands: - - mkdir /usr/local/cloud-config - - chmod 600 /usr/local/cloud-config - name: "Ensure /usr/local/cloud-config exists" diff --git a/overlay/files/system/oem/21_kcrypt.yaml b/overlay/files/system/oem/21_kcrypt.yaml index e65aad658..89ba01de5 100644 --- a/overlay/files/system/oem/21_kcrypt.yaml +++ b/overlay/files/system/oem/21_kcrypt.yaml @@ -1,10 +1,5 @@ name: "Kcrypt" stages: - rootfs: - - name: "Unlock encrypted volumes" - if: '[ ! -f "/run/cos/live_mode" ]' - commands: - - kcrypt unlock-all after-upgrade: - name: "Update plugins" if: "[ $(kairos-agent state get oem.found) == 'true' ]" diff --git a/overlay/files/system/oem/25_default_paths.yaml b/overlay/files/system/oem/25_default_paths.yaml deleted file mode 100644 index f3b8d70c2..000000000 --- a/overlay/files/system/oem/25_default_paths.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: "default-paths" -stages: - fs.before: - - name: "Default system dirs" - directories: - - path: /var/lib/longhorn - # TODO: create dir and set persistent path only when snapd is detected as installed - - path: /var/lib/snapd - - path: /snap - - path: /var/snap diff --git a/overlay/files/system/oem/27_harderning.yaml b/overlay/files/system/oem/27_harderning.yaml deleted file mode 100644 index 76bbede2c..000000000 --- a/overlay/files/system/oem/27_harderning.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: "Security configuration hardening" -stages: - initramfs: - - name: "Ensure runtime permission" - if: '[ -e "/oem" ]' - commands: - - chown -R root:admin /oem - - chmod 770 /oem - - name: "Ensure runtime permission" - if: '[ -e "/usr/local/cloud-config" ]' - commands: - - chown -R root:admin /usr/local/cloud-config - - chmod 770 /usr/local/cloud-config diff --git a/overlay/files/system/oem/32_journal_persistency.yaml b/overlay/files/system/oem/32_journal_persistency.yaml deleted file mode 100644 index e1b2c3aae..000000000 --- a/overlay/files/system/oem/32_journal_persistency.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: "Journalctl persistence" -stages: - initramfs: - - name: "Create journalctl /var/log/journal dir" - if: '[ -e "/sbin/systemctl" ] || [ -e "/bin/systemctl" ] || [ -e "/usr/sbin/systemctl" ] || [ -e "/usr/bin/systemctl" ]' - directories: - - path: /var/log/journal diff --git a/overlay/files/system/oem/20_recovery_mode.yaml b/overlay/files/system/oem/50_recovery.yaml similarity index 60% rename from overlay/files/system/oem/20_recovery_mode.yaml rename to overlay/files/system/oem/50_recovery.yaml index 7716e1cc8..d142483a7 100644 --- a/overlay/files/system/oem/20_recovery_mode.yaml +++ b/overlay/files/system/oem/50_recovery.yaml @@ -15,3 +15,14 @@ stages: commands: - sed -i -e 's/tty1.*//g' /etc/inittab - echo "tty1::respawn:/usr/bin/kairos-agent recovery" >> /etc/inittab + boot: + - name: "Recovery" + if: '[ -f "/run/cos/recovery_mode" ]' + hostname: "cos-recovery" + commands: + - | + source /etc/os-release + echo >> /etc/issue + echo "You are booting from recovery mode. Run 'kairos-agent reset' to reset the system to $VERSION" >> /etc/issue + echo " or 'kairos-agent upgrade' to upgrade the active partition" >> /etc/issue + echo >> /etc/issue diff --git a/overlay/files/system/oem/20_reset_mode.yaml b/overlay/files/system/oem/51_reset.yaml similarity index 100% rename from overlay/files/system/oem/20_reset_mode.yaml rename to overlay/files/system/oem/51_reset.yaml diff --git a/overlay/files/system/oem/04_installer.yaml b/overlay/files/system/oem/52_installer.yaml similarity index 100% rename from overlay/files/system/oem/04_installer.yaml rename to overlay/files/system/oem/52_installer.yaml From eb90fea7c4d44497b28d26f7ee9085b4b5947557 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 31 May 2023 14:42:56 +0200 Subject: [PATCH 23/62] Update quay.io/kairos/osbuilder-tools Docker tag to v0.7.0 (#1464) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index 232f7923e..6cf31399e 100644 --- a/Earthfile +++ b/Earthfile @@ -20,7 +20,7 @@ END ARG COSIGN_EXPERIMENTAL=0 ARG CGO_ENABLED=0 # renovate: datasource=docker depName=quay.io/kairos/osbuilder-tools versioning=semver-coerced -ARG OSBUILDER_VERSION=v0.6.7 +ARG OSBUILDER_VERSION=v0.7.0 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION ARG GOLINT_VERSION=1.52.2 # renovate: datasource=docker depName=golang From 5f0450f254413f78d70851448ffe6f5fef8358c3 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Wed, 31 May 2023 15:04:01 +0000 Subject: [PATCH 24/62] :robot: Improve CI PR building (#1466) - Only build iso+image - Only upload sarif and sboms on master merge - Do not upload anything but ISO on PRs - Only upload iso+image+sarif+sbom on master merge Signed-off-by: Itxaka --- .github/workflows/image.yaml | 31 ++++--------------------------- Earthfile | 7 ++++++- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 281da62ff..e3a28ca9d 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -87,12 +87,13 @@ jobs: FLAVOR: ${{ matrix.flavor }} IMAGE: quay.io/kairos/core-${{ matrix.flavor }}:latest run: | - earthly +all --IMAGE=$IMAGE --FLAVOR=$FLAVOR + earthly +ci --IMAGE=$IMAGE --FLAVOR=$FLAVOR sudo mv build/* . sudo rm -rf build mkdir sarif - mv *.sarif sarif/ + mv *.sarif sarif/ || true - name: Upload Trivy scan results to GitHub Security tab + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: github/codeql-action/upload-sarif@v2 with: sarif_file: 'sarif' @@ -105,37 +106,13 @@ jobs: *.sha256 if-no-files-found: error - uses: actions/upload-artifact@v3 + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} with: name: kairos-${{ matrix.flavor }}.sbom.zip path: | *.syft.json *.spdx.json if-no-files-found: error - - uses: actions/upload-artifact@v3 - with: - name: kairos-${{ matrix.flavor }}.initrd.zip - path: | - *-initrd - if-no-files-found: error - - uses: actions/upload-artifact@v3 - with: - name: kairos-${{ matrix.flavor }}.squashfs.zip - path: | - *.squashfs - if-no-files-found: error - - uses: actions/upload-artifact@v3 - with: - name: kairos-${{ matrix.flavor }}.kernel.zip - path: | - *-kernel - *-initrd - if-no-files-found: error - - uses: actions/upload-artifact@v3 - with: - name: kairos-${{ matrix.flavor }}.ipxe.zip - path: | - *.ipxe - if-no-files-found: error - name: Push to quay if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} env: diff --git a/Earthfile b/Earthfile index 6cf31399e..2c35b421a 100644 --- a/Earthfile +++ b/Earthfile @@ -47,6 +47,11 @@ all: BUILD +netboot BUILD +ipxe-iso +# For PR building, only image and iso are needed +ci: + BUILD +image + BUILD +iso + all-arm: ARG SECURITY_SCANS=true BUILD --platform=linux/arm64 +image --MODEL=rpi64 @@ -699,7 +704,7 @@ trivy-scan: RUN /trivy filesystem --skip-dirs /tmp --timeout 30m --format sarif -o report.sarif --no-progress / RUN /trivy filesystem --skip-dirs /tmp --timeout 30m --format template --template "@/contrib/html.tpl" -o report.html --no-progress / RUN /trivy filesystem --skip-dirs /tmp --timeout 30m -f json -o results.json --no-progress / - SAVE ARTIFACT /build/report.sarif report.sartif AS LOCAL build/${VARIANT}-${FLAVOR}-${VERSION}-trivy.sarif + SAVE ARTIFACT /build/report.sarif report.sarif AS LOCAL build/${VARIANT}-${FLAVOR}-${VERSION}-trivy.sarif SAVE ARTIFACT /build/report.html report.html AS LOCAL build/${VARIANT}-${FLAVOR}-${VERSION}-trivy.html SAVE ARTIFACT /build/results.json results.json AS LOCAL build/${VARIANT}-${FLAVOR}-${VERSION}-trivy.json From 51b395db8ebb60096d148ad2cfe71c828192c8fc Mon Sep 17 00:00:00 2001 From: "ci-robbot [bot]" <105103991+ci-robbot@users.noreply.github.com> Date: Thu, 1 Jun 2023 09:23:18 +0200 Subject: [PATCH 25/62] :arrow_up: Update repositories (#1457) Co-authored-by: mudler --- framework-profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework-profile.yaml b/framework-profile.yaml index c0f1b6d39..5d449d450 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -99,9 +99,9 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230526140327-repository.yaml + reference: 20230531144835-repository.yaml - !!merge <<: *kairos arch: arm64 urls: - "quay.io/kairos/packages-arm64" - reference: 20230524170431-repository.yaml + reference: 20230531144806-repository.yaml From 46ace193426adea0d4daa1cdd8fa33200d08e0fe Mon Sep 17 00:00:00 2001 From: Itxaka Date: Thu, 1 Jun 2023 12:01:42 +0000 Subject: [PATCH 26/62] :robot: Fix building security scans on master (#1469) --- .github/workflows/image.yaml | 14 ++++++++++++-- Earthfile | 6 ++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index e3a28ca9d..1f287e9eb 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -82,7 +82,17 @@ jobs: with: repository: quay.io/kairos/packages packages: utils/earthly - - name: Build 🔧 + - name: Build PR 🔧 + if: ${{ github.event_name == 'pull_request' }} + env: + FLAVOR: ${{ matrix.flavor }} + IMAGE: quay.io/kairos/core-${{ matrix.flavor }}:latest + run: | + earthly +ci --SECURITY_SCANS=false --IMAGE=$IMAGE --FLAVOR=$FLAVOR + sudo mv build/* . + sudo rm -rf build + - name: Build master 🔧 + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} env: FLAVOR: ${{ matrix.flavor }} IMAGE: quay.io/kairos/core-${{ matrix.flavor }}:latest @@ -91,7 +101,7 @@ jobs: sudo mv build/* . sudo rm -rf build mkdir sarif - mv *.sarif sarif/ || true + mv *.sarif sarif/ - name: Upload Trivy scan results to GitHub Security tab if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: github/codeql-action/upload-sarif@v2 diff --git a/Earthfile b/Earthfile index 2c35b421a..ad4da4408 100644 --- a/Earthfile +++ b/Earthfile @@ -49,7 +49,13 @@ all: # For PR building, only image and iso are needed ci: + ARG SECURITY_SCANS=true BUILD +image + IF [ "$SECURITY_SCANS" = "true" ] + BUILD +image-sbom + BUILD +trivy-scan + BUILD +grype-scan + END BUILD +iso all-arm: From 8b1b1e00640193f1cb5e71e9156c0cc605c46284 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Thu, 1 Jun 2023 16:14:13 +0000 Subject: [PATCH 27/62] Revert ":sparkles: Directly blacklist the vc4 module everywhere" (#1471) Revert ":sparkles: Directly blacklist the vc4 module everywhere (#1443)" This reverts commit ffd9f67353afdcd33b2cc165b934922f5708e01e. --- Earthfile | 5 +++++ overlay/files-rpi/etc/cos/bootargs.cfg | 18 ++++++++++++++++++ .../files/etc/modprobe.d/vc4-blacklist.conf | 9 --------- 3 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 overlay/files-rpi/etc/cos/bootargs.cfg delete mode 100644 overlay/files/etc/modprobe.d/vc4-blacklist.conf diff --git a/Earthfile b/Earthfile index ad4da4408..eb7e76313 100644 --- a/Earthfile +++ b/Earthfile @@ -223,6 +223,11 @@ framework: # TODO: Make this also a package? COPY overlay/files /framework + # Copy common overlay files for Raspberry Pi + IF [ "$MODEL" = "rpi64" ] + COPY overlay/files-rpi/ /framework + END + # Copy flavor-specific overlay files IF [[ "$FLAVOR" =~ ^alpine* ]] COPY overlay/files-alpine/ /framework diff --git a/overlay/files-rpi/etc/cos/bootargs.cfg b/overlay/files-rpi/etc/cos/bootargs.cfg new file mode 100644 index 000000000..8391be319 --- /dev/null +++ b/overlay/files-rpi/etc/cos/bootargs.cfg @@ -0,0 +1,18 @@ +set kernel=/boot/vmlinuz + +# Note on RPI bootargs +# We additionally set modprobe.blacklist=vc4 as certain Displays are not supported by vc4. +# As kairos main target is cloud and not graphics usage, we blacklist it to avoid +# that the HDMI output goes off due to drivers kicking during boot. vc4 is required where graphics +# or video playback is needed, which is not the case in this example here. +# A similar workaround could be applied at config.txt level, by diabling the vc4 overlay. +# See also: https://en.opensuse.org/HCL:Raspberry_Pi3#I_see_HDMI_output_in_U-Boot.2C_but_not_in_Linux , +# https://en.opensuse.org/HCL:Raspberry_Pi3#DSI_output_not_supported_by_VC4_driver, +# https://bugzilla.opensuse.org/show_bug.cgi?id=1181683 and https://github.com/raspberrypi/linux/issues/4020 +if [ -n "$recoverylabel" ]; then + set kernelcmd="console=tty1 console=ttyS0,115200 root=live:LABEL=$recoverylabel net.ifnames=1 rd.live.dir=/ rd.live.squashimg=$img panic=5 modprobe.blacklist=vc4 rd.cos.oemtimeout=10" +else + set kernelcmd="console=tty1 console=ttyS0,115200 root=LABEL=$label net.ifnames=1 cos-img/filename=$img panic=5 security=selinux selinux=1 modprobe.blacklist=vc4 rd.cos.oemtimeout=10 rd.cos.oemlabel=COS_OEM" +fi + +set initramfs=/boot/initrd \ No newline at end of file diff --git a/overlay/files/etc/modprobe.d/vc4-blacklist.conf b/overlay/files/etc/modprobe.d/vc4-blacklist.conf deleted file mode 100644 index 8b278baf8..000000000 --- a/overlay/files/etc/modprobe.d/vc4-blacklist.conf +++ /dev/null @@ -1,9 +0,0 @@ -# We additionally blacklist=vc4 as certain Displays are not supported by vc4. -# As kairos main target is cloud and not graphics usage, we blacklist it to avoid -# that the HDMI output goes off due to drivers kicking during boot. vc4 is required where graphics -# or video playback is needed, which is not the case in this example here. -# A similar workaround could be applied at config.txt level, by diabling the vc4 overlay. -# See also: https://en.opensuse.org/HCL:Raspberry_Pi3#I_see_HDMI_output_in_U-Boot.2C_but_not_in_Linux , -# https://en.opensuse.org/HCL:Raspberry_Pi3#DSI_output_not_supported_by_VC4_driver, -# https://bugzilla.opensuse.org/show_bug.cgi?id=1181683 and https://github.com/raspberrypi/linux/issues/4020 -blacklist vc4 \ No newline at end of file From 6cc3118c76762e7130ef0bbdf292ffc0e374c006 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Thu, 1 Jun 2023 17:34:12 +0000 Subject: [PATCH 28/62] :robot: Drop framework building from PR builds (#1470) --- .github/workflows/image.yaml | 39 +----------------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 1f287e9eb..f122e02ce 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -8,27 +8,11 @@ on: pull_request: paths: - '**' - workflow_dispatch: - inputs: - immucore_dev: - description: 'Build workflow with immucore from a given branch' - required: false - type: choice - options: - - true - - false - default: 'false' - immucore_dev_branch: - description: 'Branch to build immucore from' - required: false - type: string - default: 'master' concurrency: group: ci-image-${{ github.head_ref || github.ref }}-${{ github.repository }} cancel-in-progress: true env: - EARTHLY_BUILD_ARGS: 'IMMUCORE_DEV=${{ inputs.immucore_dev }},IMMUCORE_DEV_BRANCH=${{ inputs.immucore_dev_branch }}' FORCE_COLOR: 1 jobs: get-matrix: @@ -142,6 +126,7 @@ jobs: docker tag quay.io/kairos/core-${{ matrix.flavor }}:latest ttl.sh/kairos-${{ matrix.flavor }}-${{ github.sha }}:8h docker push ttl.sh/kairos-${{ matrix.flavor }}-${{ github.sha }}:8h build-framework: + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} needs: - get-matrix runs-on: self-hosted @@ -157,7 +142,6 @@ jobs: - name: Install Cosign uses: sigstore/cosign-installer@main - name: Login to Quay Registry - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} run: echo ${{ secrets.QUAY_PASSWORD }} | docker login -u ${{ secrets.QUAY_USERNAME }} --password-stdin quay.io - name: Install earthly uses: Luet-lab/luet-install-action@v1 @@ -165,7 +149,6 @@ jobs: repository: quay.io/kairos/packages packages: utils/earthly - name: Build framework image 🔧 - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} env: FLAVOR: ${{ matrix.flavor }} IMAGE: "quay.io/kairos/framework" @@ -187,26 +170,6 @@ jobs: earthly +build-framework-image --FLAVOR=${FLAVOR} --VERSION=master docker push "$IMAGE:$TAG" # Otherwise .RepoDigests will be empty for some reason cosign sign $(docker image inspect --format='{{index .RepoDigests 0}}' "$IMAGE:$TAG") - - name: Build framework image 🔧 - env: - FLAVOR: ${{ matrix.flavor }} - IMAGE: quay.io/kairos/core-${{ matrix.flavor }}:master - run: | - # Configure earthly to use the docker mirror in CI - # https://docs.earthly.dev/ci-integration/pull-through-cache#configuring-earthly-to-use-the-cache - mkdir -p ~/.earthly/ || true - cat << EOF > ~/.earthly/config.yml - global: - buildkit_additional_config: | - [registry."docker.io"] - mirrors = ["registry.docker-mirror.svc.cluster.local:5000"] - [registry."registry.docker-mirror.svc.cluster.local:5000"] - insecure = true - http = true - EOF - earthly +build-framework-image --FLAVOR=${FLAVOR} --VERSION=master - docker tag quay.io/kairos/framework:master_${{ matrix.flavor }} ttl.sh/kairos-framework-${{ matrix.flavor }}-${{ github.sha }}:8h - docker push ttl.sh/kairos-framework-${{ matrix.flavor }}-${{ github.sha }}:8h install-test: needs: - build From 9ae3cc67eb5d127472359f8f9d60066780f0f976 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 2 Jun 2023 09:06:14 +0200 Subject: [PATCH 29/62] Update quay.io/kairos/osbuilder-tools Docker tag to v0.7.1 (#1467) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index eb7e76313..9b7e67d57 100644 --- a/Earthfile +++ b/Earthfile @@ -20,7 +20,7 @@ END ARG COSIGN_EXPERIMENTAL=0 ARG CGO_ENABLED=0 # renovate: datasource=docker depName=quay.io/kairos/osbuilder-tools versioning=semver-coerced -ARG OSBUILDER_VERSION=v0.7.0 +ARG OSBUILDER_VERSION=v0.7.2 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION ARG GOLINT_VERSION=1.52.2 # renovate: datasource=docker depName=golang From 3dd781ca546ca9703232c4b1565217b022d068a6 Mon Sep 17 00:00:00 2001 From: "ci-robbot [bot]" <105103991+ci-robbot@users.noreply.github.com> Date: Fri, 2 Jun 2023 09:07:09 +0200 Subject: [PATCH 30/62] :arrow_up: Update repositories (#1472) Signed-off-by: GitHub Co-authored-by: mudler --- framework-profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework-profile.yaml b/framework-profile.yaml index 5d449d450..4a04b914e 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -99,9 +99,9 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230531144835-repository.yaml + reference: 20230601145033-repository.yaml - !!merge <<: *kairos arch: arm64 urls: - "quay.io/kairos/packages-arm64" - reference: 20230531144806-repository.yaml + reference: 20230601151801-repository.yaml From f8539e8a97a13af5e585b00573c1fb297b2aa962 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Fri, 2 Jun 2023 16:44:34 +0000 Subject: [PATCH 31/62] :seedling: Drop tmp.mount services (#1475) Should be not needed as immucore is mounting the tmp dir and systemd will make sure the mount is on once switched to the new root after initramfs Signed-off-by: Itxaka --- images/Dockerfile.debian | 4 ---- images/Dockerfile.rockylinux | 3 --- images/Dockerfile.ubuntu | 4 ---- images/Dockerfile.ubuntu-20-lts | 4 ---- images/Dockerfile.ubuntu-20-lts-arm-rpi | 4 ---- images/Dockerfile.ubuntu-22-lts | 3 --- images/Dockerfile.ubuntu-22-lts-arm-rpi | 4 ---- images/Dockerfile.ubuntu-arm-rpi | 4 ---- 8 files changed, 30 deletions(-) diff --git a/images/Dockerfile.debian b/images/Dockerfile.debian index adbb3e511..4fbb7e0da 100644 --- a/images/Dockerfile.debian +++ b/images/Dockerfile.debian @@ -113,10 +113,6 @@ RUN systemctl enable ssh COPY images/dracut-missing-overlay-dirs.patch / RUN cd /usr/lib/dracut/modules.d/90dmsquash-live && patch < /dracut-missing-overlay-dirs.patch && rm -rf /dracut-missing-overlay-dirs.patch -# Enable tmp -RUN cp -v /usr/share/systemd/tmp.mount /etc/systemd/system/ -RUN systemctl enable tmp.mount - # Fixup sudo perms RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo diff --git a/images/Dockerfile.rockylinux b/images/Dockerfile.rockylinux index ce9ea6351..633bbb57f 100644 --- a/images/Dockerfile.rockylinux +++ b/images/Dockerfile.rockylinux @@ -63,6 +63,3 @@ RUN systemctl enable systemd-resolved RUN systemctl disable dnf-makecache.service RUN systemctl disable NetworkManager RUN systemctl enable sshd - -# Enable tmp -RUN systemctl enable tmp.mount diff --git a/images/Dockerfile.ubuntu b/images/Dockerfile.ubuntu index 3c39d42eb..6032fd353 100644 --- a/images/Dockerfile.ubuntu +++ b/images/Dockerfile.ubuntu @@ -87,10 +87,6 @@ RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv RUN systemctl enable systemd-networkd RUN systemctl enable ssh -# Enable tmp -RUN cp -v /usr/share/systemd/tmp.mount /etc/systemd/system/ -RUN systemctl enable tmp.mount - # Fixup sudo perms RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo diff --git a/images/Dockerfile.ubuntu-20-lts b/images/Dockerfile.ubuntu-20-lts index c99ac0915..4817cc0e1 100644 --- a/images/Dockerfile.ubuntu-20-lts +++ b/images/Dockerfile.ubuntu-20-lts @@ -87,10 +87,6 @@ RUN systemctl enable ssh COPY images/dracut-broken-iscsi-ubuntu-20.patch / RUN cd /usr/lib/dracut/modules.d/95iscsi && patch < /dracut-broken-iscsi-ubuntu-20.patch && rm -rf /dracut-broken-iscsi-ubuntu-20.patch -# Enable tmp -RUN cp -v /usr/share/systemd/tmp.mount /etc/systemd/system/ -RUN systemctl enable tmp.mount - # Fixup sudo perms RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo diff --git a/images/Dockerfile.ubuntu-20-lts-arm-rpi b/images/Dockerfile.ubuntu-20-lts-arm-rpi index 396aaf04c..9b8d0b921 100644 --- a/images/Dockerfile.ubuntu-20-lts-arm-rpi +++ b/images/Dockerfile.ubuntu-20-lts-arm-rpi @@ -67,10 +67,6 @@ RUN systemctl enable systemd-networkd RUN systemctl enable ssh RUN systemctl disable rpi-eeprom-update -# Enable tmp -RUN cp -v /usr/share/systemd/tmp.mount /etc/systemd/system/ -RUN systemctl enable tmp.mount - # Fixup sudo perms RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo diff --git a/images/Dockerfile.ubuntu-22-lts b/images/Dockerfile.ubuntu-22-lts index a8c32c217..b745f42a8 100644 --- a/images/Dockerfile.ubuntu-22-lts +++ b/images/Dockerfile.ubuntu-22-lts @@ -83,9 +83,6 @@ RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv RUN systemctl enable systemd-networkd RUN systemctl enable ssh -# Enable tmp -RUN cp -v /usr/share/systemd/tmp.mount /etc/systemd/system/ -RUN systemctl enable tmp.mount # Fixup sudo perms RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo diff --git a/images/Dockerfile.ubuntu-22-lts-arm-rpi b/images/Dockerfile.ubuntu-22-lts-arm-rpi index 25afc2de2..c96df8b07 100644 --- a/images/Dockerfile.ubuntu-22-lts-arm-rpi +++ b/images/Dockerfile.ubuntu-22-lts-arm-rpi @@ -69,10 +69,6 @@ RUN systemctl enable systemd-networkd RUN systemctl enable ssh RUN systemctl disable rpi-eeprom-update -# Enable tmp -RUN cp -v /usr/share/systemd/tmp.mount /etc/systemd/system/ -RUN systemctl enable tmp.mount - # Fixup sudo perms RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo diff --git a/images/Dockerfile.ubuntu-arm-rpi b/images/Dockerfile.ubuntu-arm-rpi index 3be0b7d72..05b085fa1 100644 --- a/images/Dockerfile.ubuntu-arm-rpi +++ b/images/Dockerfile.ubuntu-arm-rpi @@ -62,10 +62,6 @@ RUN systemctl enable systemd-networkd RUN systemctl enable ssh RUN systemctl disable rpi-eeprom-update -# Enable tmp -RUN cp -v /usr/share/systemd/tmp.mount /etc/systemd/system/ -RUN systemctl enable tmp.mount - # Fixup sudo perms RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo From 0bae80ec86e51b1fee54d8213d3b933e75c7d694 Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Mon, 5 Jun 2023 12:16:50 +0200 Subject: [PATCH 32/62] :sparkles: Remove OverlayFS patch (#1476) * Remove overlayfs patch in tumbleweed Signed-off-by: Mauro Morales * Remove for Debian Signed-off-by: Mauro Morales --------- Signed-off-by: Mauro Morales --- images/Dockerfile.debian | 6 ------ images/Dockerfile.opensuse-tumbleweed | 8 +------- images/dracut-missing-overlay-dirs.patch | 15 --------------- overlay/files-iso/boot/grub2/grub.cfg | 12 ++++++------ 4 files changed, 7 insertions(+), 34 deletions(-) delete mode 100644 images/dracut-missing-overlay-dirs.patch diff --git a/images/Dockerfile.debian b/images/Dockerfile.debian index 4fbb7e0da..e007dae8d 100644 --- a/images/Dockerfile.debian +++ b/images/Dockerfile.debian @@ -107,12 +107,6 @@ RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv RUN systemctl enable systemd-networkd RUN systemctl enable ssh -# workaround https://github.com/systemd/systemd/issues/12231 -# see also: https://github.com/OSInside/kiwi/issues/1015 -# TODO: backport patch into packages -COPY images/dracut-missing-overlay-dirs.patch / -RUN cd /usr/lib/dracut/modules.d/90dmsquash-live && patch < /dracut-missing-overlay-dirs.patch && rm -rf /dracut-missing-overlay-dirs.patch - # Fixup sudo perms RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo diff --git a/images/Dockerfile.opensuse-tumbleweed b/images/Dockerfile.opensuse-tumbleweed index 6cf73a1e4..aeb62edef 100644 --- a/images/Dockerfile.opensuse-tumbleweed +++ b/images/Dockerfile.opensuse-tumbleweed @@ -64,10 +64,4 @@ RUN zypper in --force-resolution -y \ tmux \ vim \ e2fsprogs \ - which && zypper cc - -# workaround https://github.com/systemd/systemd/issues/12231 -# see also: https://github.com/OSInside/kiwi/issues/1015 -# TODO: backport patch into packages -COPY images/dracut-missing-overlay-dirs.patch / -RUN cd /usr/lib/dracut/modules.d/90dmsquash-live && patch < /dracut-missing-overlay-dirs.patch && rm -rf /dracut-missing-overlay-dirs.patch + which && zypper cc \ No newline at end of file diff --git a/images/dracut-missing-overlay-dirs.patch b/images/dracut-missing-overlay-dirs.patch deleted file mode 100644 index e9e2506e2..000000000 --- a/images/dracut-missing-overlay-dirs.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- dmsquash-generator.sh 2022-12-15 08:46:45.000000000 +0000 -+++ dmsquash-generator.sh.new 2022-12-15 10:48:06.771676518 +0000 -@@ -49,6 +49,12 @@ getargbool 0 rd.live.overlay.readonly -d - getargbool 0 rd.live.overlay.overlayfs && overlayfs="yes" - [ -e /xor_overlayfs ] && xor_overlayfs="yes" - [ -e /xor_readonly ] && xor_readonly="--readonly" -+ -+if [ "$overlayfs$xor_overlayfs" = "yes" ]; then -+ mkdir -p /run/overlayfs -+ mkdir -p /run/ovlwork -+fi -+ - ROOTFLAGS="$(getarg rootflags)" - { - echo "[Unit]" diff --git a/overlay/files-iso/boot/grub2/grub.cfg b/overlay/files-iso/boot/grub2/grub.cfg index cad2b204f..ab308bc37 100644 --- a/overlay/files-iso/boot/grub2/grub.cfg +++ b/overlay/files-iso/boot/grub2/grub.cfg @@ -22,42 +22,42 @@ if [ -f ${font} ];then fi menuentry "Kairos" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Kairos (manual)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "kairos (interactive install)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable interactive-install selinux=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable interactive-install selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Kairos (remote recovery mode)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset kairos.remote_recovery_mode selinux=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset kairos.remote_recovery_mode selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Kairos (boot local node from livecd)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 kairos.boot_live_mode selinux=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 kairos.boot_live_mode selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Kairos (debug)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty0 rd.debug rd.shell rd.cos.disable selinux=0 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty0 rd.debug rd.shell rd.cos.disable selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } From 56562d695b4d10707edfa9ba5f6a63aca66b2369 Mon Sep 17 00:00:00 2001 From: "ci-robbot [bot]" <105103991+ci-robbot@users.noreply.github.com> Date: Tue, 6 Jun 2023 10:46:31 +0200 Subject: [PATCH 33/62] :arrow_up: Update repositories (#1479) Co-authored-by: mudler --- framework-profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework-profile.yaml b/framework-profile.yaml index 4a04b914e..69fc9d3bd 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -99,9 +99,9 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230601145033-repository.yaml + reference: 20230605142655-repository.yaml - !!merge <<: *kairos arch: arm64 urls: - "quay.io/kairos/packages-arm64" - reference: 20230601151801-repository.yaml + reference: 20230605144452-repository.yaml From be38f0871daa597f93582f98a1d48b8dc8066c9e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 6 Jun 2023 12:01:36 +0200 Subject: [PATCH 34/62] Update quay.io/kairos/osbuilder-tools Docker tag to v0.7.3 (#1478) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Itxaka --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index 9b7e67d57..27b414401 100644 --- a/Earthfile +++ b/Earthfile @@ -20,7 +20,7 @@ END ARG COSIGN_EXPERIMENTAL=0 ARG CGO_ENABLED=0 # renovate: datasource=docker depName=quay.io/kairos/osbuilder-tools versioning=semver-coerced -ARG OSBUILDER_VERSION=v0.7.2 +ARG OSBUILDER_VERSION=v0.7.4 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION ARG GOLINT_VERSION=1.52.2 # renovate: datasource=docker depName=golang From a4fc8e83ab9aa590cc5f30764aba3b73fc74f0a0 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Wed, 7 Jun 2023 12:39:12 +0000 Subject: [PATCH 35/62] :robot: Run zfs test in self-hosted (#1484) --- .github/workflows/image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index f122e02ce..da5d6f2a6 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -229,7 +229,7 @@ jobs: zfs-tests: needs: - build - runs-on: ubuntu-latest + runs-on: self-hosted strategy: fail-fast: false matrix: From 8e305146bc801c203e7528a4e47f5b41caf315e8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:12:55 +0200 Subject: [PATCH 36/62] Update module github.com/kairos-io/kairos-sdk to v0.0.6 (#1428) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- profile-build/go.mod | 2 +- profile-build/go.sum | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/profile-build/go.mod b/profile-build/go.mod index bfc867f45..17609054b 100644 --- a/profile-build/go.mod +++ b/profile-build/go.mod @@ -3,7 +3,7 @@ module main go 1.20 require ( - github.com/kairos-io/kairos-sdk v0.0.3 + github.com/kairos-io/kairos-sdk v0.0.6 github.com/urfave/cli v1.22.13 ) diff --git a/profile-build/go.sum b/profile-build/go.sum index 227b5233f..8cfe74969 100644 --- a/profile-build/go.sum +++ b/profile-build/go.sum @@ -21,9 +21,11 @@ github.com/Microsoft/hcsshim v0.10.0-rc.8/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCzt github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= @@ -32,8 +34,11 @@ github.com/containerd/containerd v1.7.1 h1:k8DbDkSOwt5rgxQ3uCI4WMKIJxIndSCBUaGm5 github.com/containerd/containerd v1.7.1/go.mod h1:gA+nJUADRBm98QS5j5RPROnt0POQSMK+r7P7EGMC/Qc= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -51,12 +56,15 @@ github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryef github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -64,6 +72,7 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -90,12 +99,15 @@ github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQ github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.3 h1:twfIhZs4QLCtimkP7MOxlF3A0U/5cDPseRT9M/+2SCE= github.com/gookit/color v1.5.3/go.mod h1:NUzwzeehUfl7GIb36pqId+UGmRfQcU/WiiyTTeNjHtE= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kairos-io/kairos-sdk v0.0.1 h1:obJw0/5amn+/wWNuDTVq81HcPuX5TwHbGS4xxEy9Ju4= github.com/kairos-io/kairos-sdk v0.0.1/go.mod h1:E70cYgGQpu1MXI8ddhH4CHVIvNi3w7l6MQlxLTeBTXY= github.com/kairos-io/kairos-sdk v0.0.3 h1:V/GJHFZuR7QWvWdxUZ76RXJdrZO8dL1aBJM74oJ0rU4= github.com/kairos-io/kairos-sdk v0.0.3/go.mod h1:wAO6aJy/ek/Y/SE/iq5x7M9d8XYUR/jfDXA5DDG/IRU= +github.com/kairos-io/kairos-sdk v0.0.6 h1:+faSwcqxAGhQP/Saiufpr5sGgLhbdy2NT+0IjBX22nU= +github.com/kairos-io/kairos-sdk v0.0.6/go.mod h1:DW5fUA1rfH1lBak5eobLzfYLjzDoYggaFzxNjY/BO7U= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= @@ -106,6 +118,7 @@ github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuOb github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/lithammer/fuzzysearch v1.1.7 h1:q8rZNmBIUkqxsxb/IlwsXVbCoPIH/0juxjFHY0UIwhU= github.com/lithammer/fuzzysearch v1.1.7/go.mod h1:ZhIlfRGxnD8qa9car/yplC6GmnM14CS07BYAKJJBK2I= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= @@ -113,10 +126,14 @@ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -147,6 +164,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -158,6 +177,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/urfave/cli v1.22.13 h1:wsLILXG8qCJNse/qAgLNf23737Cx05GflHg/PJGe1Ok= github.com/urfave/cli v1.22.13/go.mod h1:VufqObjsMTF2BBwKawpx9R8eAneNEWhoO0yx8Vd+FkE= @@ -165,14 +185,17 @@ github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RV github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/wayneashleyberry/terminal-dimensions v1.1.0 h1:EB7cIzBdsOzAgmhTUtTTQXBByuPheP/Zv1zL2BRPY6g= github.com/wayneashleyberry/terminal-dimensions v1.1.0/go.mod h1:2lc/0eWCObmhRczn2SdGSQtgBooLUzIotkkEGXqghyg= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -291,6 +314,8 @@ google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -300,5 +325,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From c719e9df2f72f11544649d7780e50833f8128dec Mon Sep 17 00:00:00 2001 From: "ci-robbot [bot]" <105103991+ci-robbot@users.noreply.github.com> Date: Wed, 7 Jun 2023 20:22:14 +0200 Subject: [PATCH 37/62] :arrow_up: Update repositories (#1487) Signed-off-by: GitHub Co-authored-by: Itxaka --- framework-profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework-profile.yaml b/framework-profile.yaml index 69fc9d3bd..d32cbc486 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -99,9 +99,9 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230605142655-repository.yaml + reference: 20230607103430-repository.yaml - !!merge <<: *kairos arch: arm64 urls: - "quay.io/kairos/packages-arm64" - reference: 20230605144452-repository.yaml + reference: 20230607105220-repository.yaml From 1922fbba674aab9fbcd29625b0dc08ba75876a42 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:25:48 +0200 Subject: [PATCH 38/62] Update aquasec/trivy Docker tag to v0.42.0 (#1474) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index 27b414401..e704164c3 100644 --- a/Earthfile +++ b/Earthfile @@ -9,7 +9,7 @@ ARG ISO_NAME=kairos-${VARIANT}-${FLAVOR} ARG LUET_VERSION=0.34.0 ARG OS_ID=kairos # renovate: datasource=docker depName=aquasec/trivy -ARG TRIVY_VERSION=0.41.0 +ARG TRIVY_VERSION=0.42.0 ARG COSIGN_SKIP=".*quay.io/kairos/.*" IF [ "$FLAVOR" = "ubuntu" ] From 183e9160b666ba9a3b21166bc32b93d6b82519a5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:26:01 +0200 Subject: [PATCH 39/62] Update earthly/earthly Docker tag to v0.7.8 (#1473) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- earthly.ps1 | 2 +- earthly.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/earthly.ps1 b/earthly.ps1 index c250f6fc9..acddd319d 100644 --- a/earthly.ps1 +++ b/earthly.ps1 @@ -1 +1 @@ -docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.6 --allow-privileged @args +docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v ${pwd}:/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.8 --allow-privileged @args diff --git a/earthly.sh b/earthly.sh index 538eb5d3b..e623e2148 100755 --- a/earthly.sh +++ b/earthly.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.6 --allow-privileged "$@" +docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.8 --allow-privileged "$@" From 297d80aea33512f71eb876bb155484cd4ca2cb47 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Thu, 8 Jun 2023 12:26:15 +0000 Subject: [PATCH 40/62] :book: Update release template (#1490) --- .github/ISSUE_TEMPLATE/release.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md index 6c3b1a303..8c4c15081 100644 --- a/.github/ISSUE_TEMPLATE/release.md +++ b/.github/ISSUE_TEMPLATE/release.md @@ -17,13 +17,21 @@ assignees: mudler ## ✅ Release Checklist - [ ] **Stage 0 - Finishing Touches** - - [ ] Check kairos/packages, and for any needed update + - [ ] Check if Kairos-docs were updated and consider tagging them with the same version as Kairos + - [ ] Check if osbuilder is in the wanted version/latest + - [ ] Check if any kairos/packages was bumped and they were merged and repo updated (https://github.com/kairos-io/packages) + - [ ] Check latest repository update was merged, otherwise trigger its job (https://github.com/kairos-io/kairos/actions/workflows/bump_repos.yml) - [ ] Make sure CI tests are passing. - [ ] Consider cutting an `rc`, `alpha`, ... based on changes on the CI - [ ] **Stage 1 - Manual testing** - How: Using the assets from master, make sure that test scenarios not covered by automatic tests are passing, and that docs are still aligned - [ ] Fedora flavor install, and manual upgrade works + - [ ] Any flavor interactive install + - [ ] Any flavor recovery reset - [ ] ARM images (openSUSE, alpine) boots and manual upgrade works + - [ ] ARM images passive and recovery booting + - [ ] ARM images reset works + - [ ] ARM images /oem exists - [ ] **Stage 3 - Release** - [ ] Tag the release on master. - [ ] **Stage 4 - Update provider-kairos** @@ -34,4 +42,4 @@ assignees: mudler - [ ] Update the `CORE_VERSION` file of `kairos-io/provider` to match the release tag of `kairos-io/kairos` - [ ] Tag the release on `provider-kairos` - [ ] **Stage 5 - Announcement** - - [ ] Blog post announcement \ No newline at end of file + - [ ] Blog post announcement From 39ae8f46c817f5b0a60a2cff13c22cbb8b8d9976 Mon Sep 17 00:00:00 2001 From: "ci-robbot [bot]" <105103991+ci-robbot@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:31:36 +0200 Subject: [PATCH 41/62] :arrow_up: Update repositories (#1491) Signed-off-by: GitHub Co-authored-by: Itxaka Co-authored-by: Itxaka --- framework-profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework-profile.yaml b/framework-profile.yaml index d32cbc486..f4e3221ea 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -99,9 +99,9 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230607103430-repository.yaml + reference: 20230608120120-repository.yaml - !!merge <<: *kairos arch: arm64 urls: - "quay.io/kairos/packages-arm64" - reference: 20230607105220-repository.yaml + reference: 20230608111508-repository.yaml From 1eaf89b66e12f549bf1aafade5ea0b4e9463a077 Mon Sep 17 00:00:00 2001 From: "ci-robbot [bot]" <105103991+ci-robbot@users.noreply.github.com> Date: Fri, 9 Jun 2023 19:54:42 +0200 Subject: [PATCH 42/62] :arrow_up: Update repositories (#1495) Signed-off-by: GitHub Co-authored-by: jimmykarily --- framework-profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework-profile.yaml b/framework-profile.yaml index f4e3221ea..1522e9768 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -99,9 +99,9 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230608120120-repository.yaml + reference: 20230609142947-repository.yaml - !!merge <<: *kairos arch: arm64 urls: - "quay.io/kairos/packages-arm64" - reference: 20230608111508-repository.yaml + reference: 20230609142225-repository.yaml From 27570e0e3ede70a95ad783543f7eeff859056269 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Mon, 12 Jun 2023 17:16:32 +0200 Subject: [PATCH 43/62] :bug: Alpine: Mount bpf system once we boot (#1500) :bug: Mount bpf system once we boot If we do it initramfs we are trooubling the chroot mounts that the stage uses to run the config files. Nothing should be mounted in the binded paths during that stage to avoid issues as after the stage is finished, the chroot is removed fully, so those mounts would get lost. Signed-off-by: Itxaka --- overlay/files/system/oem/00_rootfs.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/overlay/files/system/oem/00_rootfs.yaml b/overlay/files/system/oem/00_rootfs.yaml index d3557c8d1..23e633450 100644 --- a/overlay/files/system/oem/00_rootfs.yaml +++ b/overlay/files/system/oem/00_rootfs.yaml @@ -13,6 +13,10 @@ stages: providers: ["aws", "gcp", "openstack", "cdrom"] path: "/oem" rootfs: + - name: "Mount BPF on Alpine systems" + if: '[ ! -e "/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ] && [ ! -e "/usr/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ]' + commands: + - mount bpffs -o rw,nosuid,nodev,noexec,relatime,mode=700 /sys/fs/bpf -t bpf - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -e "/run/cos/uki_mode" ]' name: "Layout configuration for active/passive mode" environment_file: /run/cos/cos-layout.env @@ -153,10 +157,6 @@ stages: # Size 0 is required to specify all remaining space size: 0 initramfs: - - name: "Mount BPF" - if: '[ ! -e "/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ] && [ ! -e "/usr/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ]' - commands: - - mount bpffs -o rw,nosuid,nodev,noexec,relatime,mode=700 /sys/fs/bpf -t bpf - name: "Create journalctl /var/log/journal dir" if: '[ -e "/sbin/systemctl" ] || [ -e "/bin/systemctl" ] || [ -e "/usr/sbin/systemctl" ] || [ -e "/usr/bin/systemctl" ]' directories: From b8c7f6dee46c1ec8ce3f4968c24c7c6ccbcf7925 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Mon, 12 Jun 2023 18:50:16 +0200 Subject: [PATCH 44/62] :arrow_up: Bump osbuilder (#1502) Signed-off-by: Itxaka --- .github/workflows/image-arm.yaml | 4 ++++ Earthfile | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/image-arm.yaml b/.github/workflows/image-arm.yaml index 00e954252..7db528f0c 100644 --- a/.github/workflows/image-arm.yaml +++ b/.github/workflows/image-arm.yaml @@ -125,6 +125,10 @@ jobs: http = true EOF docker run --privileged -v $HOME/.earthly/config.yml:/etc/.earthly/config.yml -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged +all-arm --IMAGE_NAME=kairos-$FLAVOR-latest.img --IMAGE=quay.io/kairos/core-$FLAVOR:latest --MODEL=$MODEL --FLAVOR=$FLAVOR + - name: Show img sizes + run: | + ls -ltra build + ls -ltrh build - name: Push 🔧 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} env: diff --git a/Earthfile b/Earthfile index e704164c3..8d617ac30 100644 --- a/Earthfile +++ b/Earthfile @@ -20,7 +20,7 @@ END ARG COSIGN_EXPERIMENTAL=0 ARG CGO_ENABLED=0 # renovate: datasource=docker depName=quay.io/kairos/osbuilder-tools versioning=semver-coerced -ARG OSBUILDER_VERSION=v0.7.4 +ARG OSBUILDER_VERSION=v0.7.5 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION ARG GOLINT_VERSION=1.52.2 # renovate: datasource=docker depName=golang From 5814eaa3894d21acec9345e2f75f157305a86fab Mon Sep 17 00:00:00 2001 From: Andreas Reitz <79543787+areitz86@users.noreply.github.com> Date: Wed, 14 Jun 2023 06:15:00 +0200 Subject: [PATCH 45/62] Removing unattended-upgrade from ubuntu images (#1513) * Update Dockerfile.ubuntu-20-lts removed unattended upgrades Signed-off-by: Andreas Reitz <79543787+areitz86@users.noreply.github.com> * Removed unattended upgrades Signed-off-by: Andreas Reitz <79543787+areitz86@users.noreply.github.com> * Removed unattended-upgrades Signed-off-by: Andreas Reitz <79543787+areitz86@users.noreply.github.com> * Removed unattended-upgrades Signed-off-by: Andreas Reitz <79543787+areitz86@users.noreply.github.com> * Removed unattended-upgrades Signed-off-by: Andreas Reitz <79543787+areitz86@users.noreply.github.com> * Removed unattended-upgrades Signed-off-by: Andreas Reitz <79543787+areitz86@users.noreply.github.com> * Removed unattended-upgrades Signed-off-by: Andreas Reitz <79543787+areitz86@users.noreply.github.com> --------- Signed-off-by: Andreas Reitz <79543787+areitz86@users.noreply.github.com> --- images/Dockerfile.ubuntu | 8 +++++++- images/Dockerfile.ubuntu-20-lts | 7 ++++++- .../Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin | 8 +++++++- images/Dockerfile.ubuntu-20-lts-arm-rpi | 7 ++++++- images/Dockerfile.ubuntu-22-lts | 7 ++++++- images/Dockerfile.ubuntu-22-lts-arm-rpi | 7 ++++++- images/Dockerfile.ubuntu-arm-rpi | 7 ++++++- 7 files changed, 44 insertions(+), 7 deletions(-) diff --git a/images/Dockerfile.ubuntu b/images/Dockerfile.ubuntu index 6032fd353..96346a0be 100644 --- a/images/Dockerfile.ubuntu +++ b/images/Dockerfile.ubuntu @@ -80,7 +80,13 @@ RUN apt-get update \ console-data \ zfsutils-linux \ zstd \ - linux-image-generic-hwe-22.04 && apt-get clean && rm -rf /var/lib/apt/lists/* + linux-image-generic-hwe-22.04 \ + && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* + +# not sure if this is really necessary, since the package is already removed +RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades +RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades + RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv diff --git a/images/Dockerfile.ubuntu-20-lts b/images/Dockerfile.ubuntu-20-lts index 4817cc0e1..eb64c310b 100644 --- a/images/Dockerfile.ubuntu-20-lts +++ b/images/Dockerfile.ubuntu-20-lts @@ -75,7 +75,12 @@ RUN apt-get update \ zerofree \ zstd \ console-data \ - zfsutils-linux && apt-get clean && rm -rf /var/lib/apt/lists/* + zfsutils-linux \ + && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* + +# not sure if this is really necessary, since the package is already removed +RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades +RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv diff --git a/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin b/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin index 0ed8452cd..5fd847481 100644 --- a/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin +++ b/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin @@ -89,6 +89,12 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ xxd \ xz-utils +RUN apt-get remove -y unattended-upgrades + +# not sure if this is really necessary, since the package is already removed +RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades +RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades + # https://docs.nvidia.com/jetson/l4t/index.html#page/Tegra%20Linux%20Driver%20Package%20Development%20Guide/updating_jetson_and_host.html RUN apt-get install -y -o Dpkg::Options::="--force-overwrite" \ nvidia-l4t-core \ @@ -112,4 +118,4 @@ RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install && \ # RUN rm -rf /var/lib/apt/lists/* # RUN useradd -ms /bin/bash jetson # RUN echo 'jetson:jetson' | chpasswd -# RUN usermod -a -G sudo jetson \ No newline at end of file +# RUN usermod -a -G sudo jetson diff --git a/images/Dockerfile.ubuntu-20-lts-arm-rpi b/images/Dockerfile.ubuntu-20-lts-arm-rpi index 9b8d0b921..ef778f030 100644 --- a/images/Dockerfile.ubuntu-20-lts-arm-rpi +++ b/images/Dockerfile.ubuntu-20-lts-arm-rpi @@ -59,7 +59,12 @@ RUN apt-get update && apt-get install -y \ systemd \ systemd-timesyncd \ tar \ - && apt-get clean && rm -rf /var/lib/apt/lists/* + && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* + +# not sure if this is really necessary, since the package is already removed +RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades +RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades + RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv diff --git a/images/Dockerfile.ubuntu-22-lts b/images/Dockerfile.ubuntu-22-lts index b745f42a8..5e80f5a40 100644 --- a/images/Dockerfile.ubuntu-22-lts +++ b/images/Dockerfile.ubuntu-22-lts @@ -76,7 +76,12 @@ RUN apt-get update \ zerofree \ zstd \ console-data \ - zfsutils-linux && apt-get clean && rm -rf /var/lib/apt/lists/* + zfsutils-linux \ + && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* + +# not sure if this is really necessary, since the package is already removed +RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades +RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv diff --git a/images/Dockerfile.ubuntu-22-lts-arm-rpi b/images/Dockerfile.ubuntu-22-lts-arm-rpi index c96df8b07..b5eaf9b52 100644 --- a/images/Dockerfile.ubuntu-22-lts-arm-rpi +++ b/images/Dockerfile.ubuntu-22-lts-arm-rpi @@ -61,7 +61,12 @@ RUN apt-get update && apt-get install -y \ systemd \ systemd-timesyncd \ tar \ - && apt-get clean && rm -rf /var/lib/apt/lists/* + && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* + +# not sure if this is really necessary, since the package is already removed +RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades +RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades + RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv diff --git a/images/Dockerfile.ubuntu-arm-rpi b/images/Dockerfile.ubuntu-arm-rpi index 05b085fa1..f57abd689 100644 --- a/images/Dockerfile.ubuntu-arm-rpi +++ b/images/Dockerfile.ubuntu-arm-rpi @@ -54,7 +54,12 @@ RUN apt-get update && apt-get install -y \ systemd \ systemd-timesyncd \ tar \ - && apt-get clean && rm -rf /var/lib/apt/lists/* + && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* + +# not sure if this is really necessary, since the package is already removed +RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades +RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades + RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv From 97fff814bb934ff7baada23964f23ff5254a1740 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 14 Jun 2023 10:18:56 +0200 Subject: [PATCH 46/62] :penguin: do not install unattended upgrades (#1514) We do remove it in case it is present in the base image, but there is no reason to install it in the first place if isn't there already. Signed-off-by: mudler --- images/Dockerfile.ubuntu | 6 ------ images/Dockerfile.ubuntu-20-lts | 4 ---- images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin | 4 ---- images/Dockerfile.ubuntu-20-lts-arm-rpi | 5 ----- images/Dockerfile.ubuntu-22-lts | 5 ----- images/Dockerfile.ubuntu-22-lts-arm-rpi | 5 ----- images/Dockerfile.ubuntu-arm-rpi | 5 ----- 7 files changed, 34 deletions(-) diff --git a/images/Dockerfile.ubuntu b/images/Dockerfile.ubuntu index 96346a0be..0023a5593 100644 --- a/images/Dockerfile.ubuntu +++ b/images/Dockerfile.ubuntu @@ -70,7 +70,6 @@ RUN apt-get update \ systemd-resolved \ systemd-timesyncd \ thermald \ - unattended-upgrades \ xdg-user-dirs \ xxd \ xz-utils \ @@ -83,11 +82,6 @@ RUN apt-get update \ linux-image-generic-hwe-22.04 \ && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* -# not sure if this is really necessary, since the package is already removed -RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades -RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades - - RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv RUN systemctl enable systemd-networkd diff --git a/images/Dockerfile.ubuntu-20-lts b/images/Dockerfile.ubuntu-20-lts index eb64c310b..15c341ca0 100644 --- a/images/Dockerfile.ubuntu-20-lts +++ b/images/Dockerfile.ubuntu-20-lts @@ -68,7 +68,6 @@ RUN apt-get update \ systemd-timesyncd \ thermald \ ubuntu-advantage-tools \ - unattended-upgrades \ xdg-user-dirs \ xxd \ xz-utils \ @@ -78,9 +77,6 @@ RUN apt-get update \ zfsutils-linux \ && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* -# not sure if this is really necessary, since the package is already removed -RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades -RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv diff --git a/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin b/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin index 5fd847481..a893587fc 100644 --- a/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin +++ b/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin @@ -91,10 +91,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ RUN apt-get remove -y unattended-upgrades -# not sure if this is really necessary, since the package is already removed -RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades -RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades - # https://docs.nvidia.com/jetson/l4t/index.html#page/Tegra%20Linux%20Driver%20Package%20Development%20Guide/updating_jetson_and_host.html RUN apt-get install -y -o Dpkg::Options::="--force-overwrite" \ nvidia-l4t-core \ diff --git a/images/Dockerfile.ubuntu-20-lts-arm-rpi b/images/Dockerfile.ubuntu-20-lts-arm-rpi index ef778f030..dce3e756e 100644 --- a/images/Dockerfile.ubuntu-20-lts-arm-rpi +++ b/images/Dockerfile.ubuntu-20-lts-arm-rpi @@ -61,11 +61,6 @@ RUN apt-get update && apt-get install -y \ tar \ && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* -# not sure if this is really necessary, since the package is already removed -RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades -RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades - - RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv RUN systemctl enable systemd-networkd diff --git a/images/Dockerfile.ubuntu-22-lts b/images/Dockerfile.ubuntu-22-lts index 5e80f5a40..14aa1d098 100644 --- a/images/Dockerfile.ubuntu-22-lts +++ b/images/Dockerfile.ubuntu-22-lts @@ -69,7 +69,6 @@ RUN apt-get update \ systemd-timesyncd \ thermald \ ubuntu-advantage-tools \ - unattended-upgrades \ xdg-user-dirs \ xxd \ xz-utils \ @@ -79,10 +78,6 @@ RUN apt-get update \ zfsutils-linux \ && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* -# not sure if this is really necessary, since the package is already removed -RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades -RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades - RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv RUN systemctl enable systemd-networkd diff --git a/images/Dockerfile.ubuntu-22-lts-arm-rpi b/images/Dockerfile.ubuntu-22-lts-arm-rpi index b5eaf9b52..e187fa398 100644 --- a/images/Dockerfile.ubuntu-22-lts-arm-rpi +++ b/images/Dockerfile.ubuntu-22-lts-arm-rpi @@ -63,11 +63,6 @@ RUN apt-get update && apt-get install -y \ tar \ && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* -# not sure if this is really necessary, since the package is already removed -RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades -RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades - - RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv RUN systemctl enable systemd-networkd diff --git a/images/Dockerfile.ubuntu-arm-rpi b/images/Dockerfile.ubuntu-arm-rpi index f57abd689..a44e68d30 100644 --- a/images/Dockerfile.ubuntu-arm-rpi +++ b/images/Dockerfile.ubuntu-arm-rpi @@ -56,11 +56,6 @@ RUN apt-get update && apt-get install -y \ tar \ && apt-get remove -y unattended-upgrades && apt-get clean && rm -rf /var/lib/apt/lists/* -# not sure if this is really necessary, since the package is already removed -RUN sed -i 's/APT::Periodic::Update-Package-Lists "1";/APT::Periodic::Update-Package-Lists "0";/g' /etc/apt/apt.conf.d/20auto-upgrades -RUN sed -i 's/APT::Periodic::Unattended-Upgrade "1";/APT::Periodic::Unattended-Upgrade "0";/g' /etc/apt/apt.conf.d/20auto-upgrades - - RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv RUN systemctl enable systemd-networkd From bb40b6d168dc9564517c7088f229f5b7254d3f9b Mon Sep 17 00:00:00 2001 From: Itxaka Date: Wed, 14 Jun 2023 10:49:28 +0200 Subject: [PATCH 47/62] :robot: Use zstd compression on PR building (#1507) --- .github/workflows/image-arm.yaml | 13 +++++++++++-- Earthfile | 8 +++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/image-arm.yaml b/.github/workflows/image-arm.yaml index 7db528f0c..5d3e4e041 100644 --- a/.github/workflows/image-arm.yaml +++ b/.github/workflows/image-arm.yaml @@ -99,13 +99,21 @@ jobs: with: repository: quay.io/kairos/packages packages: utils/earthly + - name: Set compression for PR + if: ${{ github.event_name == 'pull_request' }} + run: | + echo "IMG_COMPRESSION=zstd" >> $GITHUB_ENV + - name: Set compression for master + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: | + echo "IMG_COMPRESSION=xz" >> $GITHUB_ENV - name: Standard Build 🔧 if: ${{ matrix.worker != 'self-hosted' }} env: FLAVOR: ${{ matrix.flavor }} MODEL: ${{ matrix.model }} run: | - ./earthly.sh +all-arm --IMAGE_NAME=kairos-$FLAVOR-latest.img --IMAGE=quay.io/kairos/core-$FLAVOR:latest --MODEL=$MODEL --FLAVOR=$FLAVOR + ./earthly.sh +all-arm --IMAGE_NAME=kairos-$FLAVOR-latest.img --IMAGE=quay.io/kairos/core-$FLAVOR:latest --MODEL=$MODEL --FLAVOR=$FLAVOR --IMG_COMPRESSION=${{env.IMG_COMPRESSION}} - name: Selfhosted Build 🔧 if: ${{ matrix.worker == 'self-hosted' }} env: @@ -124,7 +132,7 @@ jobs: insecure = true http = true EOF - docker run --privileged -v $HOME/.earthly/config.yml:/etc/.earthly/config.yml -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged +all-arm --IMAGE_NAME=kairos-$FLAVOR-latest.img --IMAGE=quay.io/kairos/core-$FLAVOR:latest --MODEL=$MODEL --FLAVOR=$FLAVOR + docker run --privileged -v $HOME/.earthly/config.yml:/etc/.earthly/config.yml -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -v "$(pwd)":/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.7.5 --allow-privileged +all-arm --IMAGE_NAME=kairos-$FLAVOR-latest.img --IMAGE=quay.io/kairos/core-$FLAVOR:latest --MODEL=$MODEL --FLAVOR=$FLAVOR --IMG_COMPRESSION=${{env.IMG_COMPRESSION}} - name: Show img sizes run: | ls -ltra build @@ -168,6 +176,7 @@ jobs: # in-toto attestation cosign attest --type spdx --predicate $spdx $image_ref - name: Upload results + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: actions/upload-artifact@v3 with: name: ${{ matrix.flavor }}-image diff --git a/Earthfile b/Earthfile index 8d617ac30..a3a45b16a 100644 --- a/Earthfile +++ b/Earthfile @@ -20,7 +20,7 @@ END ARG COSIGN_EXPERIMENTAL=0 ARG CGO_ENABLED=0 # renovate: datasource=docker depName=quay.io/kairos/osbuilder-tools versioning=semver-coerced -ARG OSBUILDER_VERSION=v0.7.5 +ARG OSBUILDER_VERSION=v0.7.6 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION ARG GOLINT_VERSION=1.52.2 # renovate: datasource=docker depName=golang @@ -593,6 +593,7 @@ netboot: arm-image: ARG OSBUILDER_IMAGE ARG COMPRESS_IMG=true + ARG IMG_COMPRESSION=xz FROM $OSBUILDER_IMAGE ARG MODEL=rpi64 ARG IMAGE_NAME=${FLAVOR}.img @@ -614,8 +615,13 @@ arm-image: RUN /build-arm-image.sh --use-lvm --model $MODEL --directory "/build/image" /build/$IMAGE_NAME END IF [ "$COMPRESS_IMG" = "true" ] + IF [ "$IMG_COMPRESSION" = "zstd" ] + RUN zstd --rm /build/$IMAGE_NAME + SAVE ARTIFACT /build/$IMAGE_NAME.zst img AS LOCAL build/$IMAGE_NAME.zst + ELSE IF [ "$IMG_COMPRESSION" = "xz" ] RUN xz -v /build/$IMAGE_NAME SAVE ARTIFACT /build/$IMAGE_NAME.xz img AS LOCAL build/$IMAGE_NAME.xz + END ELSE SAVE ARTIFACT /build/$IMAGE_NAME img AS LOCAL build/$IMAGE_NAME END From b0c205b52b3932fafd311ef533c583f85593dff1 Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Wed, 14 Jun 2023 14:52:27 +0200 Subject: [PATCH 48/62] :penguin: bump Ubuntu core images to 23.04 (#1433) * bump ubuntu to 23.04 and use rolling tag Signed-off-by: Mauro Morales * apply overlayfs patch Signed-off-by: Mauro Morales * Enable selinux on Ubuntu Signed-off-by: Mauro Morales * add netboot cmdline stanzas Signed-off-by: Mauro Morales * remove selinux for netboot Signed-off-by: Mauro Morales --------- Signed-off-by: Mauro Morales --- images/Dockerfile.ubuntu | 3 ++- images/Dockerfile.ubuntu-arm-rpi | 4 ++-- overlay/files-ubuntu/etc/cos/bootargs.cfg | 5 +---- scripts/netboot.sh | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/images/Dockerfile.ubuntu b/images/Dockerfile.ubuntu index 0023a5593..9640c9050 100644 --- a/images/Dockerfile.ubuntu +++ b/images/Dockerfile.ubuntu @@ -1,4 +1,5 @@ -ARG BASE_IMAGE=ubuntu:22.10 +# rolling tag points to the latest release (regardless of LTS status) +ARG BASE_IMAGE=ubuntu:rolling FROM $BASE_IMAGE diff --git a/images/Dockerfile.ubuntu-arm-rpi b/images/Dockerfile.ubuntu-arm-rpi index a44e68d30..fa0051ae1 100644 --- a/images/Dockerfile.ubuntu-arm-rpi +++ b/images/Dockerfile.ubuntu-arm-rpi @@ -1,4 +1,5 @@ -ARG BASE_IMAGE=ubuntu:22.10 +# rolling tag points to the latest release (regardless of LTS status) +ARG BASE_IMAGE=ubuntu:rolling ARG MODEL=generic FROM $BASE_IMAGE as base @@ -60,7 +61,6 @@ RUN ln -s /usr/sbin/grub-install /usr/sbin/grub2-install RUN ln -s /usr/bin/grub-editenv /usr/bin/grub2-editenv RUN systemctl enable systemd-networkd RUN systemctl enable ssh -RUN systemctl disable rpi-eeprom-update # Fixup sudo perms RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo diff --git a/overlay/files-ubuntu/etc/cos/bootargs.cfg b/overlay/files-ubuntu/etc/cos/bootargs.cfg index d3256e933..31f82de6b 100644 --- a/overlay/files-ubuntu/etc/cos/bootargs.cfg +++ b/overlay/files-ubuntu/etc/cos/bootargs.cfg @@ -1,11 +1,8 @@ set kernel=/boot/vmlinuz if [ -n "$recoverylabel" ]; then - # Removed console=ttyS0 (don't use it), and also rd.neetnet=1 which results in dupliate ips - # Added fancy vga=795 nomodeset set kernelcmd="console=tty1 console=ttyS0 root=live:LABEL=$recoverylabel rd.live.dir=/ rd.live.squashimg=$img panic=5 rd.cos.oemlabel=COS_OEM" else - # set kernelcmd="console=tty1 root=LABEL=$label cos-img/filename=$img panic=5 security=selinux selinux=1 rd.cos.oemlabel=COS_OEM rd.neednet=0 vga=795 nomodeset" - set kernelcmd="console=tty1 console=ttyS0 root=LABEL=$label cos-img/filename=$img panic=5 security=selinux selinux=0 rd.cos.oemlabel=COS_OEM rd.neednet=0 vga=795" + set kernelcmd="console=tty1 console=ttyS0 root=LABEL=$label cos-img/filename=$img panic=5 security=selinux selinux=1 rd.cos.oemlabel=COS_OEM rd.neednet=0 vga=795" fi set initramfs=/boot/initrd diff --git a/scripts/netboot.sh b/scripts/netboot.sh index 1b61fee15..3c81290d0 100755 --- a/scripts/netboot.sh +++ b/scripts/netboot.sh @@ -24,7 +24,7 @@ set dns 8.8.8.8 ifconf # set config https://example.com/machine-config # set cmdline extra.values=1 -kernel \${url}/\${kernel} initrd=\${initrd} rd.neednet=1 ip=dhcp rd.cos.disable root=live:\${url}/\${rootfs} netboot nodepair.enable config_url=\${config} console=tty1 console=ttyS0 \${cmdline} +kernel \${url}/\${kernel} initrd=\${initrd} rd.neednet=1 ip=dhcp rd.cos.disable root=live:\${url}/\${rootfs} netboot nodepair.enable config_url=\${config} console=tty1 console=ttyS0 rd.live.overlay.overlayfs \${cmdline} initrd \${url}/\${initrd} boot EOF From 4bbb384a96199231dc7c8bc65b9c7f5015b8c7f8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 15:00:49 +0200 Subject: [PATCH 49/62] Update module github.com/kairos-io/kairos-sdk to v0.0.8 (#1488) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- profile-build/go.mod | 2 +- profile-build/go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/profile-build/go.mod b/profile-build/go.mod index 17609054b..0b4142dfe 100644 --- a/profile-build/go.mod +++ b/profile-build/go.mod @@ -3,7 +3,7 @@ module main go 1.20 require ( - github.com/kairos-io/kairos-sdk v0.0.6 + github.com/kairos-io/kairos-sdk v0.0.8 github.com/urfave/cli v1.22.13 ) diff --git a/profile-build/go.sum b/profile-build/go.sum index 8cfe74969..1b0e06e8e 100644 --- a/profile-build/go.sum +++ b/profile-build/go.sum @@ -108,6 +108,8 @@ github.com/kairos-io/kairos-sdk v0.0.3 h1:V/GJHFZuR7QWvWdxUZ76RXJdrZO8dL1aBJM74o github.com/kairos-io/kairos-sdk v0.0.3/go.mod h1:wAO6aJy/ek/Y/SE/iq5x7M9d8XYUR/jfDXA5DDG/IRU= github.com/kairos-io/kairos-sdk v0.0.6 h1:+faSwcqxAGhQP/Saiufpr5sGgLhbdy2NT+0IjBX22nU= github.com/kairos-io/kairos-sdk v0.0.6/go.mod h1:DW5fUA1rfH1lBak5eobLzfYLjzDoYggaFzxNjY/BO7U= +github.com/kairos-io/kairos-sdk v0.0.8 h1:3yfxdmUuJoN7ePg+ogpH1PJvuMsLmLcxEXuWoiGdIrg= +github.com/kairos-io/kairos-sdk v0.0.8/go.mod h1:Z+1CLqMZq97bzwX2XSIArr8EoniMth3mMYkOOb8L3QY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= @@ -118,6 +120,7 @@ github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuOb github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/lithammer/fuzzysearch v1.1.7 h1:q8rZNmBIUkqxsxb/IlwsXVbCoPIH/0juxjFHY0UIwhU= github.com/lithammer/fuzzysearch v1.1.7/go.mod h1:ZhIlfRGxnD8qa9car/yplC6GmnM14CS07BYAKJJBK2I= From 55ebef9c3afdb81e575a6be890f5ba4e735f1eda Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Thu, 15 Jun 2023 10:36:47 +0200 Subject: [PATCH 50/62] :book: Add Andreas Reitz to contributors (#1522) Update contributors Signed-off-by: Mauro Morales --- CONTRIBUTORS.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index f2de084f3..1f8c53042 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -4,11 +4,12 @@ This project would not be possible without the many amazing community contributo These are active contributors who have made multiple contributions to the project; by authoring PRs, commenting on issues and pull requests, or participating in community discussions on Matrix or our community channels. -| Contributor | GitHub ID | -| ------------------------ | ------------------------------------------------------ | -| Christian Prim | [@christianprim](https://github.com/christianprim) | -| Ludea | [@ludea](https://github.com/ludea) | -| Jason B. Alonso | [@jbalonso](https://github.com/jbalonso) | -| Shawn Wilsher | [@sdwilsh](https://github.com/sdwilsh) | -| scuzhanglei | [@scuzhanglei](https://github.com/scuzhanglei) | -| Martin Schuessler | [@c0ffee](https://github.com/c0ffee) | \ No newline at end of file +| Contributor | GitHub ID | +|-------------------| ------------------------------------------------------ | +| Christian Prim | [@christianprim](https://github.com/christianprim) | +| Ludea | [@ludea](https://github.com/ludea) | +| Jason B. Alonso | [@jbalonso](https://github.com/jbalonso) | +| Shawn Wilsher | [@sdwilsh](https://github.com/sdwilsh) | +| scuzhanglei | [@scuzhanglei](https://github.com/scuzhanglei) | +| Martin Schuessler | [@c0ffee](https://github.com/c0ffee) | +| Andreas Reitz | [@areitz86](https://github.com/areitz86) | From dbaa161fef4e465c8598a107949f64e56b558014 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Thu, 15 Jun 2023 17:47:24 +0200 Subject: [PATCH 51/62] :robot: Fix uki release artifacts (#1523) regex was missing the asterisk so it didnt find any artifacts for the release Signed-off-by: Itxaka --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 90d00d060..c6521986c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -177,7 +177,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') with: files: | - build/efi + build/*.efi # build-vm-images: # needs: build # runs-on: macos-12 From b7e030a7ab1787b7156c09de9585c4683af47a17 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Fri, 16 Jun 2023 11:56:00 +0200 Subject: [PATCH 52/62] :robot: Fix golang image not having qemu packages anymore (#1524) --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index a3a45b16a..fb4cd407a 100644 --- a/Earthfile +++ b/Earthfile @@ -795,7 +795,7 @@ run-qemu-netboot-test: ARG VERSION=$(cat VERSION) RUN apt update - RUN apt install -y qemu qemu-utils qemu-system git swtpm && apt clean + RUN apt install -y qemu-utils qemu-system git swtpm && apt clean # This is the IP at which qemu vm can see the host ARG IP="10.0.2.2" From 5451d0cd0f662b0f422bf26b242b4ee2b42d0aef Mon Sep 17 00:00:00 2001 From: Itxaka Date: Mon, 19 Jun 2023 12:29:59 +0200 Subject: [PATCH 53/62] :seedling: Sync all kairos entries in livecd (#1493) --- overlay/files-iso/boot/grub2/grub.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/overlay/files-iso/boot/grub2/grub.cfg b/overlay/files-iso/boot/grub2/grub.cfg index ab308bc37..49b8ba641 100644 --- a/overlay/files-iso/boot/grub2/grub.cfg +++ b/overlay/files-iso/boot/grub2/grub.cfg @@ -29,14 +29,14 @@ menuentry "Kairos" --class os --unrestricted { menuentry "Kairos (manual)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "kairos (interactive install)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable interactive-install selinux=0 rd.live.overlay.overlayfs + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset interactive-install selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } @@ -50,14 +50,14 @@ menuentry "Kairos (remote recovery mode)" --class os --unrestricted { menuentry "Kairos (boot local node from livecd)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 kairos.boot_live_mode selinux=0 rd.live.overlay.overlayfs + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 kairos.boot_live_mode vga=795 nomodeset selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Kairos (debug)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty0 rd.debug rd.shell rd.cos.disable selinux=0 rd.live.overlay.overlayfs + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty0 rd.debug rd.shell rd.cos.disable rd.immucore.debug vga=795 nomodeset selinux=0 rd.live.overlay.overlayfs echo Loading initrd... $initrd ($root)/boot/initrd } From aec22f6ade82ce0f2643aeca10f36e9555c5537f Mon Sep 17 00:00:00 2001 From: Itxaka Date: Mon, 19 Jun 2023 21:32:00 +0200 Subject: [PATCH 54/62] :sparkles: Allow serial login in openrc systems (#1505) * :sparkle: Allow serial login in openrc systems Currently is only possible to get the boot process via the serial output wiht no login available. This pathc enables login via the serial console under openrc systems Signed-off-by: Itxaka * Workaround for upgrade test issue Signed-off-by: Itxaka --------- Signed-off-by: Itxaka --- overlay/files/system/oem/10_accounting.yaml | 7 ++++++- tests/upgrade_latest_cli_test.go | 14 +++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/overlay/files/system/oem/10_accounting.yaml b/overlay/files/system/oem/10_accounting.yaml index de2e58baf..15c8dbe88 100644 --- a/overlay/files/system/oem/10_accounting.yaml +++ b/overlay/files/system/oem/10_accounting.yaml @@ -1,4 +1,4 @@ -name: "Default user and permissions" +name: "Default user, permissions and serial login" stages: initramfs: - name: "Setup groups" @@ -48,3 +48,8 @@ stages: commands: - chown -R root:admin /usr/local/cloud-config - chmod 770 /usr/local/cloud-config + - name: "Enable serial login for alpine" # https://wiki.alpinelinux.org/wiki/Enable_Serial_Console_on_Boot + if: '[ -e /sbin/rc-service ]' + commands: + - sed -i -e 's/ttyS0.*//g' /etc/inittab + - echo "ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100" >> /etc/inittab \ No newline at end of file diff --git a/tests/upgrade_latest_cli_test.go b/tests/upgrade_latest_cli_test.go index 72613cb65..3524d1412 100644 --- a/tests/upgrade_latest_cli_test.go +++ b/tests/upgrade_latest_cli_test.go @@ -14,6 +14,7 @@ import ( var _ = Describe("k3s upgrade manual test", Label("upgrade-latest-with-cli"), func() { var vm VM containerImage := os.Getenv("CONTAINER_IMAGE") + var installOutput string BeforeEach(func() { if containerImage == "" { @@ -21,15 +22,18 @@ var _ = Describe("k3s upgrade manual test", Label("upgrade-latest-with-cli"), fu } _, vm = startVM() vm.EventuallyConnects(1200) + // Workaround for v2.2.0 alpine flavors mounting /tmp and overwriting the install config + // TODO: drop in v2.2.1 or v2.3.0 whichever comes first + time.Sleep(5 * time.Minute) }) - AfterEach(func() { if CurrentSpecReport().Failed() { - gatherLogs(vm) + fmt.Print(installOutput) serial, _ := os.ReadFile(filepath.Join(vm.StateDir, "serial.log")) _ = os.MkdirAll("logs", os.ModePerm|os.ModeDir) _ = os.WriteFile(filepath.Join("logs", "serial.log"), serial, os.ModePerm) fmt.Println(string(serial)) + gatherLogs(vm) } Expect(vm.Destroy(nil)).ToNot(HaveOccurred()) }) @@ -41,10 +45,10 @@ var _ = Describe("k3s upgrade manual test", Label("upgrade-latest-with-cli"), fu err := vm.Scp("assets/config.yaml", "/tmp/config.yaml", "0770") Expect(err).ToNot(HaveOccurred()) By("Manually installing") - out, err := vm.Sudo("/bin/bash -c 'set -o pipefail && kairos-agent manual-install --device auto /tmp/config.yaml 2>&1 | tee manual-install.txt'") - Expect(err).ToNot(HaveOccurred(), out) + installOutput, err := vm.Sudo("kairos-agent --debug manual-install --device auto /tmp/config.yaml") + Expect(err).ToNot(HaveOccurred(), installOutput) - Expect(out).Should(ContainSubstring("Running after-install hook")) + Expect(installOutput).Should(ContainSubstring("Running after-install hook")) vm.Sudo("sync") err = vm.DetachCD() From bb25e05f7e7d31428c4fa3ce58a2597bd3dc24e8 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Tue, 20 Jun 2023 08:40:22 +0200 Subject: [PATCH 55/62] :bug: Do not remount /tmp under alpine (#1530) Immucore is already mounting /tmp on boot so we should not re-mount it, or at least if we do, we should check first if its mounted to not shadow existing /tmp files Signed-off-by: Itxaka --- overlay/files/system/oem/00_rootfs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/overlay/files/system/oem/00_rootfs.yaml b/overlay/files/system/oem/00_rootfs.yaml index 23e633450..6d2474d8c 100644 --- a/overlay/files/system/oem/00_rootfs.yaml +++ b/overlay/files/system/oem/00_rootfs.yaml @@ -192,5 +192,4 @@ stages: - name: "Mount tmp on alpine" if: "[[ $(kairos-agent state get kairos.flavor) =~ ^alpine ]]" commands: - - mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp - mount --make-rshared / From 6e902ae75a7b90c58ca2536896ba40ea1719d7f9 Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Wed, 21 Jun 2023 11:26:32 +0200 Subject: [PATCH 56/62] :robot: Lint (#1533) Lint Signed-off-by: Mauro Morales --- overlay/files/system/oem/10_accounting.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overlay/files/system/oem/10_accounting.yaml b/overlay/files/system/oem/10_accounting.yaml index 15c8dbe88..4ac0f7e2b 100644 --- a/overlay/files/system/oem/10_accounting.yaml +++ b/overlay/files/system/oem/10_accounting.yaml @@ -52,4 +52,4 @@ stages: if: '[ -e /sbin/rc-service ]' commands: - sed -i -e 's/ttyS0.*//g' /etc/inittab - - echo "ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100" >> /etc/inittab \ No newline at end of file + - echo "ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100" >> /etc/inittab From 2ab366b45c83a24db14633ae99d48cd9ab1d03fc Mon Sep 17 00:00:00 2001 From: Itxaka Date: Wed, 21 Jun 2023 13:11:00 +0200 Subject: [PATCH 57/62] :bug: Fix restoring machine-id on reboot (#1531) --- overlay/files/system/oem/00_rootfs.yaml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/overlay/files/system/oem/00_rootfs.yaml b/overlay/files/system/oem/00_rootfs.yaml index 6d2474d8c..e04e01261 100644 --- a/overlay/files/system/oem/00_rootfs.yaml +++ b/overlay/files/system/oem/00_rootfs.yaml @@ -7,11 +7,6 @@ # copy the file with a prefix starting by 90, e.g. /oem/91_custom.yaml name: "Rootfs Layout Settings" stages: - rootfs.before: - - name: "Pull data from provider" - datasource: - providers: ["aws", "gcp", "openstack", "cdrom"] - path: "/oem" rootfs: - name: "Mount BPF on Alpine systems" if: '[ ! -e "/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ] && [ ! -e "/usr/sbin/systemctl" ] && [ ! -e "/usr/bin/systemctl" ]' @@ -162,21 +157,21 @@ stages: directories: - path: /var/log/journal - if: '[ ! -f "/run/cos/recovery_mode" ] && [ -s /usr/local/etc/machine-id ]' - name: "Restore /etc/machine-id" + name: "Restore /etc/machine-id for systemd systems" commands: - cat /usr/local/etc/machine-id > /etc/machine-id - - if: '[ ! -f "/run/cos/recovery_mode" ] && [ -s /var/lib/dbus/machine-id ]' + - if: '[ ! -f "/run/cos/recovery_mode" ] && [ -s /var/lib/dbus/machine-id ] && [ -f "/sbin/openrc" ]' name: "Restore /etc/machine-id for openrc systems" commands: - cat /var/lib/dbus/machine-id > /etc/machine-id fs: - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -s /usr/local/etc/machine-id ] ' - name: "Save /etc/machine-id" + name: "Save /etc/machine-id for systemd systems" commands: - | mkdir -p /usr/local/etc cp /etc/machine-id /usr/local/etc - - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -s /var/lib/dbus/machine-id ] ' + - if: '[ ! -f "/run/cos/recovery_mode" ] && [ ! -s /var/lib/dbus/machine-id ] && [ -f "/sbin/openrc" ]' name: "Save /etc/machine-id for openrc systems" commands: - | From 0148879090bdcb26be3211e36041fb7497f00b95 Mon Sep 17 00:00:00 2001 From: "ci-robbot [bot]" <105103991+ci-robbot@users.noreply.github.com> Date: Wed, 21 Jun 2023 14:27:39 +0200 Subject: [PATCH 58/62] :arrow_up: Update repositories (#1511) Co-authored-by: Itxaka --- framework-profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework-profile.yaml b/framework-profile.yaml index 1522e9768..4fc8d7d75 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -99,9 +99,9 @@ repositories: priority: 2 urls: - "quay.io/kairos/packages" - reference: 20230609142947-repository.yaml + reference: 20230621111751-repository.yaml - !!merge <<: *kairos arch: arm64 urls: - "quay.io/kairos/packages-arm64" - reference: 20230609142225-repository.yaml + reference: 20230613201238-repository.yaml From dbacc56fbd7776767ebbd1c08e9bc1a045c8a9fc Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Wed, 21 Jun 2023 18:34:13 +0200 Subject: [PATCH 59/62] :penguin: Add AlmaLinux flavor (#1532) * Add alma linux flavor Signed-off-by: Mauro Morales * add dockerfile Signed-off-by: Mauro Morales * Cleanup packages Signed-off-by: Mauro Morales * Add alma to the list of CI flavors Signed-off-by: Mauro Morales --------- Signed-off-by: Mauro Morales --- .github/flavors.json | 3 ++ Earthfile | 4 +-- framework-profile.yaml | 3 ++ images/Dockerfile.almalinux | 61 ++++++++++++++++++++++++++++++++++++ images/Dockerfile.fedora | 3 -- images/Dockerfile.rockylinux | 4 --- 6 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 images/Dockerfile.almalinux diff --git a/.github/flavors.json b/.github/flavors.json index d71c47324..032dbebc4 100644 --- a/.github/flavors.json +++ b/.github/flavors.json @@ -28,5 +28,8 @@ }, { "flavor": "rockylinux" + }, + { + "flavor": "almalinux" } ] \ No newline at end of file diff --git a/Earthfile b/Earthfile index fb4cd407a..78e742ce8 100644 --- a/Earthfile +++ b/Earthfile @@ -231,7 +231,7 @@ framework: # Copy flavor-specific overlay files IF [[ "$FLAVOR" =~ ^alpine* ]] COPY overlay/files-alpine/ /framework - ELSE IF [ "$FLAVOR" = "fedora" ] || [ "$FLAVOR" = "rockylinux" ] + ELSE IF [ "$FLAVOR" = "fedora" ] || [ "$FLAVOR" = "rockylinux" ] || [ "$FLAVOR" = "almalinux" ] COPY overlay/files-fedora/ /framework ELSE IF [ "$FLAVOR" = "debian" ] || [ "$FLAVOR" = "ubuntu" ] || [ "$FLAVOR" = "ubuntu-20-lts" ] || [ "$FLAVOR" = "ubuntu-22-lts" ] || [[ "$FLAVOR" =~ ^ubuntu-.*-lts-arm-.*$ ]] COPY overlay/files-ubuntu/ /framework @@ -360,7 +360,7 @@ base-image: # Set /boot/vmlinuz pointing to our kernel so kairos-agent can use it # https://github.com/kairos-io/kairos-agent/blob/0288fb111bc568a1bfca59cb09f39302220475b6/pkg/elemental/elemental.go#L548 q - IF [ "$FLAVOR" = "fedora" ] || [ "$FLAVOR" = "rockylinux" ] + IF [ "$FLAVOR" = "fedora" ] || [ "$FLAVOR" = "rockylinux" ] || [ "$FLAVOR" = "almalinux" ] RUN rm -rf /boot/initramfs-* END diff --git a/framework-profile.yaml b/framework-profile.yaml index 4fc8d7d75..25318a0f2 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -38,6 +38,9 @@ flavors: rockylinux: - systemd-base - dracut-network-legacy-compat + almalinux: + - systemd-base + - dracut-network-legacy-compat opensuse-tumbleweed: - systemd-base - systemd-latest diff --git a/images/Dockerfile.almalinux b/images/Dockerfile.almalinux new file mode 100644 index 000000000..41666751c --- /dev/null +++ b/images/Dockerfile.almalinux @@ -0,0 +1,61 @@ +ARG BASE_IMAGE=almalinux:latest + +FROM $BASE_IMAGE + +RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf + +RUN dnf install -y epel-release && dnf clean all +RUN dnf update -y +RUN dnf makecache +RUN dnf install -y \ + dracut-live \ + livecd-tools \ + dracut-squash \ + dracut-network \ + efibootmgr \ + audit \ + device-mapper \ + dosfstools \ + dracut \ + dracut-live \ + dracut-network \ + dracut-squash \ + e2fsprogs \ + efibootmgr \ + gawk \ + grub2 \ + grub2-efi-x64 \ + grub2-efi-x64-modules \ + grub2-pc \ + kernel \ + kernel-modules \ + kernel-modules-extra \ + livecd-tools \ + lvm2 \ + nano \ + openssh-server \ + parted \ + polkit \ + rsync \ + shim-x64 \ + squashfs-tools \ + sudo \ + systemd \ + systemd-networkd \ + systemd-resolved \ + systemd-timesyncd \ + tar \ + which \ + epel-release \ + https://zfsonlinux.org/epel/zfs-release-2-2.el9.noarch.rpm \ + rsync && dnf clean all + +RUN mkdir -p /run/lock +RUN touch /usr/libexec/.keep +RUN systemctl enable getty@tty1.service +RUN systemctl enable getty@tty2.service +RUN systemctl enable getty@tty3.service +RUN systemctl enable systemd-networkd +RUN systemctl enable systemd-resolved +RUN systemctl disable dnf-makecache.service +RUN systemctl enable sshd diff --git a/images/Dockerfile.fedora b/images/Dockerfile.fedora index 303eb6689..58d2cd763 100644 --- a/images/Dockerfile.fedora +++ b/images/Dockerfile.fedora @@ -6,8 +6,6 @@ FROM $BASE_IMAGE RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf RUN dnf install -y "https://zfsonlinux.org/fedora/zfs-release-2-3$(rpm --eval "%{dist}").noarch.rpm" && dnf clean all RUN dnf install -y \ - NetworkManager \ - squashfs-tools \ dracut-live \ livecd-tools \ dracut-squash \ @@ -37,7 +35,6 @@ RUN dnf install -y \ livecd-tools \ lvm2 \ nano \ - NetworkManager \ openssh-server \ parted \ polkit \ diff --git a/images/Dockerfile.rockylinux b/images/Dockerfile.rockylinux index 633bbb57f..abb1d0335 100644 --- a/images/Dockerfile.rockylinux +++ b/images/Dockerfile.rockylinux @@ -8,8 +8,6 @@ RUN dnf install -y epel-release && dnf clean all RUN dnf update -y RUN dnf makecache RUN dnf install -y \ - NetworkManager \ - squashfs-tools \ dracut-live \ livecd-tools \ dracut-squash \ @@ -35,7 +33,6 @@ RUN dnf install -y \ livecd-tools \ lvm2 \ nano \ - NetworkManager \ openssh-server \ parted \ polkit \ @@ -61,5 +58,4 @@ RUN systemctl enable getty@tty3.service RUN systemctl enable systemd-networkd RUN systemctl enable systemd-resolved RUN systemctl disable dnf-makecache.service -RUN systemctl disable NetworkManager RUN systemctl enable sshd From f883e8ec7ed0d8021c0d3b7b0e862502e56cb5ad Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Thu, 22 Jun 2023 22:46:06 +0300 Subject: [PATCH 60/62] Build ubuntu fips framework image (#1544) * Build ubuntu fips framework image to allow users to build fips enabled ubuntu flavors from scratch https://kairos.io/docs/reference/build-from-scratch/ Signed-off-by: Dimitris Karakasilis * Build only framework images for fips since we don't need isos for them. Isos need to be built manually (from scratch) because to have a fips enabled flavor you need to have a fips enabled OS in general. Signed-off-by: Ettore Di Giacinto * Use a name that allows us to have other certifications in the future This was a PR suggestion Signed-off-by: Dimitris Karakasilis * Don't use deprecated flag for upgrade Signed-off-by: Dimitris Karakasilis --------- Signed-off-by: Dimitris Karakasilis Signed-off-by: Ettore Di Giacinto Signed-off-by: Dimitris Karakasilis Co-authored-by: Ettore Di Giacinto Co-authored-by: Dimitris Karakasilis --- .github/flavors.json | 10 +++-- .github/workflows/image.yaml | 28 +++++++++++-- .github/workflows/release.yaml | 27 ++++++++++-- framework-profile.yaml | 76 ++++++++++++++++++++++++++++++---- tests/upgrade_cli_test.go | 2 +- 5 files changed, 123 insertions(+), 20 deletions(-) diff --git a/.github/flavors.json b/.github/flavors.json index 032dbebc4..493a1b683 100644 --- a/.github/flavors.json +++ b/.github/flavors.json @@ -1,9 +1,9 @@ [ { - "flavor": "opensuse-leap" + "flavor": "opensuse-leap" }, { - "flavor": "opensuse-tumbleweed" + "flavor": "opensuse-tumbleweed" }, { "flavor": "ubuntu" @@ -11,6 +11,10 @@ { "flavor": "ubuntu-20-lts" }, + { + "flavor": "ubuntu-20-lts-fips", + "frameworkonly": "true" + }, { "flavor": "ubuntu-22-lts" }, @@ -32,4 +36,4 @@ { "flavor": "almalinux" } -] \ No newline at end of file +] diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index da5d6f2a6..fae43b40c 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -23,9 +23,10 @@ jobs: - uses: actions/checkout@v3 - run: | git fetch --prune --unshallow + sudo apt update && sudo apt install -y jq - id: set-matrix run: | - content=`cat ./.github/flavors.json` + content=`cat .github/flavors.json | jq 'map(select(.frameworkonly != "true"))'` # the following lines are only required for multi line json # the following lines are only required for multi line json content="${content//'%'/'%25'}" @@ -34,6 +35,27 @@ jobs: # end of optional handling for multi line json # end of optional handling for multi line json echo "::set-output name=matrix::{\"include\": $content }" + + get-framework-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + - run: | + git fetch --prune --unshallow + - id: set-matrix + run: | + content=`cat .github/flavors.json` + # the following lines are only required for multi line json + # the following lines are only required for multi line json + content="${content//'%'/'%25'}" + content="${content//$'\n'/'%0A'}" + content="${content//$'\r'/'%0D'}" + # end of optional handling for multi line json + # end of optional handling for multi line json + echo "::set-output name=matrix::{\"include\": $content }" + build: runs-on: ubuntu-latest needs: @@ -128,13 +150,13 @@ jobs: build-framework: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} needs: - - get-matrix + - get-framework-matrix runs-on: self-hosted permissions: id-token: write strategy: fail-fast: false - matrix: ${{fromJson(needs.get-matrix.outputs.matrix)}} + matrix: ${{fromJson(needs.get-framework-matrix.outputs.matrix)}} steps: - uses: actions/checkout@v3 - run: | diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c6521986c..3f9f6f35f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -13,9 +13,28 @@ jobs: - uses: actions/checkout@v3 - run: | git fetch --prune --unshallow + sudo apt update && sudo apt install -y jq - id: set-matrix run: | - content=`cat ./.github/flavors.json` + content=`cat .github/flavors.json | jq 'map(select(.frameworkonly != "true"))'` + # the following lines are only required for multi line json + content="${content//'%'/'%25'}" + content="${content//$'\n'/'%0A'}" + content="${content//$'\r'/'%0D'}" + # end of optional handling for multi line json + echo "::set-output name=matrix::{\"include\": $content }" + + get-framework-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + - run: | + git fetch --prune --unshallow + - id: set-matrix + run: | + content=`cat .github/flavors.json` # the following lines are only required for multi line json content="${content//'%'/'%25'}" content="${content//$'\n'/'%0A'}" @@ -26,13 +45,13 @@ jobs: build-framework: runs-on: self-hosted needs: - - get-matrix + - get-framework-matrix permissions: id-token: write # OIDC support contents: write strategy: fail-fast: false - matrix: ${{fromJson(needs.get-matrix.outputs.matrix)}} + matrix: ${{fromJson(needs.get-framework-matrix.outputs.matrix)}} steps: - uses: actions/checkout@v3 - run: | @@ -86,7 +105,7 @@ jobs: security-events: write strategy: fail-fast: false - matrix: ${{fromJson(needs.get-matrix.outputs.matrix)}} + matrix: ${{ fromJson(needs.get-matrix.outputs.matrix) }} steps: - uses: actions/checkout@v3 - run: | diff --git a/framework-profile.yaml b/framework-profile.yaml index 25318a0f2..4bdd2de0f 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -1,69 +1,104 @@ -common: - - dracut/immucore - - static/grub-config - - system/kcrypt - - system/kcrypt-challenger - - system/suc-upgrade - - system/grub2-efi - - system/immucore - - system/kairos-agent +# Additional common packages to be added to all the FLAVORS +## common: +## - category/name@version flavors: debian: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy ubuntu: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy ubuntu-arm-rpi: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy ubuntu-20-lts-arm-nvidia-jetson-agx-orin: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy-compat ubuntu-20-lts-arm-rpi: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy-compat ubuntu-22-lts-arm-rpi: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy-compat ubuntu-22-lts: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy-compat ubuntu-20-lts: + - common-packages + - kairos-toolchain + - systemd-base + - dracut-network-legacy-compat + ubuntu-20-lts-fips: + - common-packages + - kairos-toolchain-fips - systemd-base - dracut-network-legacy-compat fedora: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy-compat rockylinux: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy-compat almalinux: + - common-packages + - kairos-toolchain - systemd-base - dracut-network-legacy-compat opensuse-tumbleweed: + - common-packages + - kairos-toolchain - systemd-base - systemd-latest - dracut-network-legacy opensuse-tumbleweed-arm-rpi: + - common-packages + - kairos-toolchain - systemd-base - systemd-latest - dracut-network-legacy opensuse-leap: + - common-packages + - kairos-toolchain - systemd-base - systemd-latest - dracut-network-legacy opensuse-leap-arm-rpi: + - common-packages + - kairos-toolchain - systemd-base - systemd-latest - dracut-network-legacy alpine-arm-rpi: + - common-packages + - kairos-toolchain - opensuse-leap-kernel - openrc alpine-opensuse-leap: + - common-packages + - kairos-toolchain - openrc - opensuse-leap-kernel alpine-ubuntu: + - common-packages + - kairos-toolchain - ubuntu-kernel - openrc # See https://github.com/kairos-io/packages/pull/67 for rationale @@ -84,6 +119,29 @@ opensuse-leap-kernel: packages: - distro-kernels/opensuse-leap - distro-initrd/opensuse-leap + +## Packages (cat/name@version) that are added to ALL flavors +## Static files, no binary +common-packages: + packages: + - static/grub-config + - dracut/immucore + - system/suc-upgrade + - system/grub2-efi + +kairos-toolchain-fips: + packages: + - fips/kcrypt + - fips/kcrypt-challenger + - fips/immucore + - fips/kairos-agent + +kairos-toolchain: + packages: + - system/kcrypt + - system/kcrypt-challenger + - system/immucore + - system/kairos-agent ubuntu-kernel: packages: - distro-kernels/ubuntu diff --git a/tests/upgrade_cli_test.go b/tests/upgrade_cli_test.go index 04e63a2c2..79a3c1eec 100644 --- a/tests/upgrade_cli_test.go +++ b/tests/upgrade_cli_test.go @@ -60,7 +60,7 @@ var _ = Describe("k3s upgrade manual test", Label("upgrade-with-cli"), func() { Expect(currentVersion).To(ContainSubstring("v")) By(fmt.Sprintf("Upgrading to: %s", containerImage)) - out, err := vm.Sudo("kairos-agent --debug upgrade --force --image " + containerImage) + out, err := vm.Sudo("kairos-agent --debug upgrade --force --source " + containerImage) Expect(err).ToNot(HaveOccurred(), string(out)) Expect(out).To(ContainSubstring("Upgrade completed")) Expect(out).To(ContainSubstring(containerImage)) From 0e08fb85e0bd42635866567fb8219e42b92f5cfd Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Fri, 23 Jun 2023 09:15:41 +0300 Subject: [PATCH 61/62] Add 'oci://' prefix to the image to make it work in test Signed-off-by: Dimitris Karakasilis --- tests/upgrade_cli_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/upgrade_cli_test.go b/tests/upgrade_cli_test.go index 79a3c1eec..df177511d 100644 --- a/tests/upgrade_cli_test.go +++ b/tests/upgrade_cli_test.go @@ -60,7 +60,7 @@ var _ = Describe("k3s upgrade manual test", Label("upgrade-with-cli"), func() { Expect(currentVersion).To(ContainSubstring("v")) By(fmt.Sprintf("Upgrading to: %s", containerImage)) - out, err := vm.Sudo("kairos-agent --debug upgrade --force --source " + containerImage) + out, err := vm.Sudo("kairos-agent --debug upgrade --force --source oci://" + containerImage) Expect(err).ToNot(HaveOccurred(), string(out)) Expect(out).To(ContainSubstring("Upgrade completed")) Expect(out).To(ContainSubstring(containerImage)) From 6c5182faeb0c143e8d0010a9ed9d0fc9a2efc95e Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Fri, 23 Jun 2023 13:19:32 +0300 Subject: [PATCH 62/62] Bump the sdk so that we can override specific package versions (#1551) Signed-off-by: Dimitris Karakasilis --- framework-profile.yaml | 5 ++++ profile-build/go.mod | 2 +- profile-build/go.sum | 54 ++++++++++++++---------------------------- 3 files changed, 24 insertions(+), 37 deletions(-) diff --git a/framework-profile.yaml b/framework-profile.yaml index 4bdd2de0f..45254ecd8 100755 --- a/framework-profile.yaml +++ b/framework-profile.yaml @@ -1,6 +1,11 @@ # Additional common packages to be added to all the FLAVORS ## common: ## - category/name@version +# +# Apply arbitrary images (e.g. to stub a package to a specific version) +# These are applyied after the packages. +# images: +# - quay.io/kairos/packages:k9s-utils-0.27.4 flavors: debian: - common-packages diff --git a/profile-build/go.mod b/profile-build/go.mod index 0b4142dfe..6bf47cc62 100644 --- a/profile-build/go.mod +++ b/profile-build/go.mod @@ -3,7 +3,7 @@ module main go 1.20 require ( - github.com/kairos-io/kairos-sdk v0.0.8 + github.com/kairos-io/kairos-sdk v0.0.9 github.com/urfave/cli v1.22.13 ) diff --git a/profile-build/go.sum b/profile-build/go.sum index 1b0e06e8e..ce1e2fcf6 100644 --- a/profile-build/go.sum +++ b/profile-build/go.sum @@ -1,3 +1,4 @@ +atomicgo.dev/assert v0.0.2 h1:FiKeMiZSgRrZsPo9qn/7vmr7mCsh5SZyXY4YGYiYwrg= atomicgo.dev/cursor v0.1.1 h1:0t9sxQomCTRh5ug+hAMCs59x/UmC9QL6Ci5uosINKD4= atomicgo.dev/cursor v0.1.1/go.mod h1:Lr4ZJB3U7DfPPOkbH7/6TOtJ4vFGHlgj1nc+n900IpU= atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= @@ -5,6 +6,7 @@ atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtE atomicgo.dev/schedule v0.0.2 h1:2e/4KY6t3wokja01Cyty6qgkQM8MotJzjtqCH70oX2Q= atomicgo.dev/schedule v0.0.2/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= @@ -14,6 +16,7 @@ github.com/MarvinJWendt/testza v0.2.10/go.mod h1:pd+VWsoGUiFtq+hRKSU1Bktnn+DMCSr github.com/MarvinJWendt/testza v0.2.12/go.mod h1:JOIegYyV7rX+7VZ9r77L/eH6CfJHHzXjB69adAhzZkI= github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/2oUqKc6bF2c= github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE= +github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.10.0-rc.8 h1:YSZVvlIIDD1UxQpJp0h+dnpLUw+TrY0cx8obKsp3bek= @@ -21,11 +24,9 @@ github.com/Microsoft/hcsshim v0.10.0-rc.8/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCzt github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= @@ -34,11 +35,8 @@ github.com/containerd/containerd v1.7.1 h1:k8DbDkSOwt5rgxQ3uCI4WMKIJxIndSCBUaGm5 github.com/containerd/containerd v1.7.1/go.mod h1:gA+nJUADRBm98QS5j5RPROnt0POQSMK+r7P7EGMC/Qc= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -56,15 +54,12 @@ github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryef github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -72,7 +67,6 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -92,6 +86,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE= github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -99,17 +94,10 @@ github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQ github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.3 h1:twfIhZs4QLCtimkP7MOxlF3A0U/5cDPseRT9M/+2SCE= github.com/gookit/color v1.5.3/go.mod h1:NUzwzeehUfl7GIb36pqId+UGmRfQcU/WiiyTTeNjHtE= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/kairos-io/kairos-sdk v0.0.1 h1:obJw0/5amn+/wWNuDTVq81HcPuX5TwHbGS4xxEy9Ju4= -github.com/kairos-io/kairos-sdk v0.0.1/go.mod h1:E70cYgGQpu1MXI8ddhH4CHVIvNi3w7l6MQlxLTeBTXY= -github.com/kairos-io/kairos-sdk v0.0.3 h1:V/GJHFZuR7QWvWdxUZ76RXJdrZO8dL1aBJM74oJ0rU4= -github.com/kairos-io/kairos-sdk v0.0.3/go.mod h1:wAO6aJy/ek/Y/SE/iq5x7M9d8XYUR/jfDXA5DDG/IRU= -github.com/kairos-io/kairos-sdk v0.0.6 h1:+faSwcqxAGhQP/Saiufpr5sGgLhbdy2NT+0IjBX22nU= -github.com/kairos-io/kairos-sdk v0.0.6/go.mod h1:DW5fUA1rfH1lBak5eobLzfYLjzDoYggaFzxNjY/BO7U= -github.com/kairos-io/kairos-sdk v0.0.8 h1:3yfxdmUuJoN7ePg+ogpH1PJvuMsLmLcxEXuWoiGdIrg= -github.com/kairos-io/kairos-sdk v0.0.8/go.mod h1:Z+1CLqMZq97bzwX2XSIArr8EoniMth3mMYkOOb8L3QY= +github.com/kairos-io/kairos-sdk v0.0.9 h1:DnuvmnCTDBQg0nr3siaXb5pspi+GUsf1rPb0MFOTdTc= +github.com/kairos-io/kairos-sdk v0.0.9/go.mod h1:Z+1CLqMZq97bzwX2XSIArr8EoniMth3mMYkOOb8L3QY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= @@ -117,11 +105,12 @@ github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/lithammer/fuzzysearch v1.1.7 h1:q8rZNmBIUkqxsxb/IlwsXVbCoPIH/0juxjFHY0UIwhU= github.com/lithammer/fuzzysearch v1.1.7/go.mod h1:ZhIlfRGxnD8qa9car/yplC6GmnM14CS07BYAKJJBK2I= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= @@ -129,14 +118,13 @@ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -164,13 +152,13 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -180,7 +168,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/urfave/cli v1.22.13 h1:wsLILXG8qCJNse/qAgLNf23737Cx05GflHg/PJGe1Ok= github.com/urfave/cli v1.22.13/go.mod h1:VufqObjsMTF2BBwKawpx9R8eAneNEWhoO0yx8Vd+FkE= @@ -188,22 +175,20 @@ github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RV github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/wayneashleyberry/terminal-dimensions v1.1.0 h1:EB7cIzBdsOzAgmhTUtTTQXBByuPheP/Zv1zL2BRPY6g= github.com/wayneashleyberry/terminal-dimensions v1.1.0/go.mod h1:2lc/0eWCObmhRczn2SdGSQtgBooLUzIotkkEGXqghyg= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b h1:SCE/18RnFsLrjydh/R/s5EVvHoZprqEQUuoxK8q2Pc4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -253,8 +238,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -271,6 +254,7 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -314,11 +298,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -328,6 +310,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=