From 643c58dd7eec6ee13831f42a1312e52011f56d44 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Sun, 19 May 2024 01:02:42 +0200 Subject: [PATCH 1/3] github-actions: rename test-apt-packages job I think it looks better in the CI run page. Signed-off-by: Attila Szakacs --- .github/workflows/test-apt-packages.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-apt-packages.yml b/.github/workflows/test-apt-packages.yml index 8235ab02aa..a6155aaac9 100644 --- a/.github/workflows/test-apt-packages.yml +++ b/.github/workflows/test-apt-packages.yml @@ -11,7 +11,8 @@ on: type: string jobs: - TestPackagesWith: + test-apt-packages: + name: ${{ matrix.distro }} strategy: matrix: distro: From f4e8b2721d0b40f20b9685e8af0790c2b9592a02 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Sat, 18 May 2024 22:28:59 +0200 Subject: [PATCH 2/3] github-actions: upload packages to R2 Signed-off-by: Attila Szakacs --- .github/workflows/axosyslog-nightly.yml | 48 ++++++++++--------- .github/workflows/draft-release.yml | 16 ++++--- .github/workflows/index-packages.yml | 4 +- .github/workflows/test-apt-packages.yml | 24 +++++----- .github/workflows/upload-packages.yml | 38 +++++++-------- .../indexer/apt_conf/nightly.conf | 2 +- .../indexer/apt_conf/stable.conf | 2 +- 7 files changed, 67 insertions(+), 67 deletions(-) diff --git a/.github/workflows/axosyslog-nightly.yml b/.github/workflows/axosyslog-nightly.yml index a0eefe1ca0..4b5c355366 100644 --- a/.github/workflows/axosyslog-nightly.yml +++ b/.github/workflows/axosyslog-nightly.yml @@ -47,29 +47,31 @@ jobs: source-tarball-artifact-name: source-tarball dbld-image-mode: cache - # upload-packages: - # needs: create-packages - # uses: ./.github/workflows/upload-packages.yml - # with: - # pkg-type: nightly - # secrets: - # azure-sas-token: ${{ secrets.AZURE_SAS_TOKEN }} - - # index-packages: - # needs: upload-packages - # uses: ./.github/workflows/index-packages.yml - # with: - # pkg-type: nightly - # secrets: - # config-base64: ${{ secrets.PACKAGE_INDEXER_CONFIG_BASE64 }} - # gpg-key-base64: ${{ secrets.PACKAGE_INDEXER_GPG_KEY_BASE64 }} - # gpg-key-passphrase: ${{ secrets.PACKAGE_INDEXER_GPG_KEY_PASSPHRASE }} - - # test-packages: - # needs: index-packages - # uses: ./.github/workflows/test-apt-packages.yml - # with: - # pkg-type: nightly + upload-packages: + needs: create-packages + uses: ./.github/workflows/upload-packages.yml + with: + pkg-type: nightly + secrets: + r2-access-key: ${{ secrets.R2_ACCESS_KEY }} + r2-secret-key: ${{ secrets.R2_SECRET_KEY }} + r2-account-id: ${{ secrets.R2_ACCOUNT_ID }} + + index-packages: + needs: upload-packages + uses: ./.github/workflows/index-packages.yml + with: + pkg-type: nightly + secrets: + config-base64: ${{ secrets.PACKAGE_INDEXER_CONFIG_BASE64 }} + gpg-key-base64: ${{ secrets.PACKAGE_INDEXER_GPG_KEY_BASE64 }} + gpg-key-passphrase: ${{ secrets.PACKAGE_INDEXER_GPG_KEY_PASSPHRASE }} + + test-packages: + needs: index-packages + uses: ./.github/workflows/test-apt-packages.yml + with: + pkg-type: nightly publish-image: if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/draft-release.yml b/.github/workflows/draft-release.yml index 4d6b05accd..670d9f84cc 100644 --- a/.github/workflows/draft-release.yml +++ b/.github/workflows/draft-release.yml @@ -83,13 +83,15 @@ jobs: source-tarball-artifact-name: release-tarball dbld-image-mode: build - # upload-packages: - # needs: create-packages - # uses: ./.github/workflows/upload-packages.yml - # with: - # pkg-type: stable - # secrets: - # azure-sas-token: ${{ secrets.AZURE_SAS_TOKEN }} + upload-packages: + needs: create-packages + uses: ./.github/workflows/upload-packages.yml + with: + pkg-type: stable + secrets: + r2-access-key: ${{ secrets.R2_ACCESS_KEY }} + r2-secret-key: ${{ secrets.R2_SECRET_KEY }} + r2-account-id: ${{ secrets.R2_ACCOUNT_ID }} create-draft-release: runs-on: ubuntu-latest diff --git a/.github/workflows/index-packages.yml b/.github/workflows/index-packages.yml index 0a6fb564f2..ce7b17223b 100644 --- a/.github/workflows/index-packages.yml +++ b/.github/workflows/index-packages.yml @@ -1,4 +1,4 @@ -name: Index packages in Azure/incoming +name: Index packages in R2/incoming on: @@ -34,7 +34,7 @@ jobs: DOCKER_IMAGE: package-indexer DOCKER_CONTAINER_NAME: package_indexer_container VERBOSE_LOG_PATH: index-packages-verbose.log - GPG_KEY_PATH: syslog_ng_ose_signing_key.sub.priv.asc + GPG_KEY_PATH: axosyslog_signing_key.sub.priv.asc if: github.repository_owner == 'axoflow' diff --git a/.github/workflows/test-apt-packages.yml b/.github/workflows/test-apt-packages.yml index a6155aaac9..099a8f1918 100644 --- a/.github/workflows/test-apt-packages.yml +++ b/.github/workflows/test-apt-packages.yml @@ -1,6 +1,6 @@ -# Please note that the syslog-ng git repository is not available in this workflow. +# Please note that the axosyslog git repository is not available in this workflow. # It means that certain convenience/helper functions are not available (e.g. gh_output). -# This is intentional as syslog-ng will be acquired from the APT repository. +# This is intentional as axosyslog will be acquired from the APT repository. name: Run smoke tests on APT packages on: @@ -27,7 +27,7 @@ jobs: - "ubuntu:mantic" upgrade-from: - "debian-official" - - "syslog-ng-last" + - "axosyslog-last" exclude: # The official syslog-ng got removed from debian:testing because of pcre: # https://tracker.debian.org/news/1445295/syslog-ng-removed-from-testing/ @@ -49,18 +49,18 @@ jobs: - name: Add OSE repository run: | - wget -qO - https://ose-repo.syslog-ng.com/apt/syslog-ng-ose-pub.asc | gpg --dearmor > /usr/share/keyrings/ose-repo-archive-keyring.gpg - echo "deb [signed-by=/usr/share/keyrings/ose-repo-archive-keyring.gpg] https://ose-repo.syslog-ng.com/apt/ ${{ inputs.pkg-type }} $(echo ${{ matrix.distro }} | sed 's/:/-/g')" | tee --append /etc/apt/sources.list.d/syslog-ng-ose.list + wget -qO - https://pkg.axoflow.io/axoflow-code-signing-pub.asc | gpg --dearmor > /usr/share/keyrings/axoflow-code-signing-pub.gpg + echo "deb [signed-by=/usr/share/keyrings/axoflow-code-signing-pub.gpg] https://pkg.axoflow.io/apt ${{ inputs.pkg-type }} $(echo ${{ matrix.distro }} | sed 's/:/-/g')" | tee --append /etc/apt/sources.list.d/axosyslog.list apt-get update -qq - - name: Install the last but one syslog-ng OSE package - if: matrix.upgrade-from == 'syslog-ng-last' + - name: Install the last but one AxoSyslog package + if: matrix.upgrade-from == 'axosyslog-last' run: | - DEBIAN_FRONTEND=noninteractive apt-get install --yes syslog-ng=$(apt-cache madison syslog-ng | sed -n 2p | awk -F"|" '{print $2}' | sed 's/ //g') + DEBIAN_FRONTEND=noninteractive apt-get install --yes axosyslog=$(apt-cache madison axosyslog | sed -n 2p | awk -F"|" '{print $2}' | sed 's/ //g') - - name: Upgrade to the latest syslog-ng OSE package + - name: Upgrade to the latest AxoSyslog package run: | - DEBIAN_FRONTEND=noninteractive apt-get install --yes syslog-ng + DEBIAN_FRONTEND=noninteractive apt-get install --yes axosyslog - name: Get syslog-ng revision run: | @@ -79,8 +79,8 @@ jobs: - name: Check if installed package version matches with install revision run: | echo "Installed revision value: ${{ steps.syslog_ng_revision.outputs.REVISION }}" - dpkg-query --show syslog-ng\* - dpkg-query -f='${db:Status-Status}\t${package}\t${version}\t\n' --show syslog-ng\* | grep ^installed | while read installed_syslog_ng_package ; do echo $installed_syslog_ng_package | grep ${{ steps.syslog_ng_revision.outputs.REVISION }} ; done + dpkg-query --show axosyslog\* + dpkg-query -f='${db:Status-Status}\t${package}\t${version}\t\n' --show axosyslog\* | grep ^installed | while read installed_axosyslog_package ; do echo $installed_axosyslog_package | grep ${{ steps.syslog_ng_revision.outputs.REVISION }} ; done - name: Check if syslog-ng can start with default config run: | diff --git a/.github/workflows/upload-packages.yml b/.github/workflows/upload-packages.yml index 614b51a3ed..8937320ca3 100644 --- a/.github/workflows/upload-packages.yml +++ b/.github/workflows/upload-packages.yml @@ -1,4 +1,4 @@ -name: Upload package to Azure/incoming +name: Upload package to R2/incoming on: @@ -22,7 +22,11 @@ on: "ubuntu-mantic" ]' secrets: - azure-sas-token: + r2-access-key: + required: true + r2-secret-key: + required: true + r2-account-id: required: true @@ -46,23 +50,15 @@ jobs: name: package-${{ matrix.distro }} path: package - - name: Calculate Azure path + - name: Upload packages to R2 incoming + working-directory: package run: | - if [[ ! "${{ inputs.pkg-type }}" =~ ^(stable|nightly)$ ]]; then - echo Unexpected input: pkg-type=${{ inputs.pkg-type }} - false - fi - - AZURE_PATH="incoming/${{ inputs.pkg-type }}/${{ github.run_id }}" - - echo "AZURE_PATH=${AZURE_PATH}" >> ${GITHUB_ENV} - - - name: Upload packages to Azure incoming - uses: azure/CLI@v1 - with: - inlineScript: | - az storage blob upload-batch \ - --sas-token '${{ secrets.azure-sas-token }}' \ - --account-name 'syslogngose' \ - --source 'package' \ - --destination '${{ env.AZURE_PATH }}' + export AWS_ACCESS_KEY_ID='${{ secrets.r2-access-key }}' + export AWS_SECRET_ACCESS_KEY='${{ secrets.r2-secret-key }}' + export AWS_DEFAULT_REGION='auto' + find * -type f -exec \ + aws s3api put-object \ + --endpoint-url https://${{ secrets.r2-account-id }}.r2.cloudflarestorage.com \ + --bucket axosyslog-apt-incoming \ + --key ${{ inputs.pkg-type }}/${{ github.run_id }}/{} \ + --body {} \; diff --git a/packaging/package-indexer/indexer/apt_conf/nightly.conf b/packaging/package-indexer/indexer/apt_conf/nightly.conf index dd5e902cb9..b0dfc4f3bc 100644 --- a/packaging/package-indexer/indexer/apt_conf/nightly.conf +++ b/packaging/package-indexer/indexer/apt_conf/nightly.conf @@ -4,7 +4,7 @@ APT { FTPArchive { Release { Architectures "amd64"; - Origin "https://ose-repo.syslog-ng.com/apt"; + Origin "https://pkg.axoflow.io/apt"; Suite "nightly"; Codename "nightly"; }; diff --git a/packaging/package-indexer/indexer/apt_conf/stable.conf b/packaging/package-indexer/indexer/apt_conf/stable.conf index 4bac0f4f02..6cdae8ae2e 100644 --- a/packaging/package-indexer/indexer/apt_conf/stable.conf +++ b/packaging/package-indexer/indexer/apt_conf/stable.conf @@ -4,7 +4,7 @@ APT { FTPArchive { Release { Architectures "amd64"; - Origin "https://ose-repo.syslog-ng.com/apt"; + Origin "https://pkg.axoflow.io/apt"; Suite "stable"; Codename "stable"; }; From c9a34f977f9f81aad5095bdd1394e796e9bef95d Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 2 Jul 2024 10:27:07 +0200 Subject: [PATCH 3/3] package-indexer: expand absolute path for gpg-key-path This is not strictly necessary as we set an absolute path in the config, but it is a nice feature, I think. Signed-off-by: Attila Szakacs --- packaging/package-indexer/config/config.py | 2 +- packaging/package-indexer/index-packages.py | 2 +- packaging/package-indexer/indexer/deb_indexer.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/package-indexer/config/config.py b/packaging/package-indexer/config/config.py index bc659a6e16..06f5c07c0b 100644 --- a/packaging/package-indexer/config/config.py +++ b/packaging/package-indexer/config/config.py @@ -138,7 +138,7 @@ def create_cdn(self, suite: str) -> cdn.CDN: return cls.from_config(options) def get_gpg_key_path(self) -> Path: - return self.__cfg["gpg-key-path"] + return Path(self.__cfg["gpg-key-path"]).expanduser().absolute() @staticmethod def __get_storage_implementation(vendor: str) -> Type[remote_storage_synchronizer.RemoteStorageSynchronizer]: diff --git a/packaging/package-indexer/index-packages.py b/packaging/package-indexer/index-packages.py index 64012e1ebc..e653694707 100755 --- a/packaging/package-indexer/index-packages.py +++ b/packaging/package-indexer/index-packages.py @@ -117,7 +117,7 @@ def construct_indexers(cfg: Config, args: dict) -> List[Indexer]: indexed_remote_storage_synchronizer = cfg.create_indexed_remote_storage_synchronizer(suite) cdn = cfg.create_cdn(suite) - gpg_key_path = Path(cfg.get_gpg_key_path()) + gpg_key_path = cfg.get_gpg_key_path() gpg_key_passphrase = stdin.read() if args["gpg_key_passphrase_from_stdin"] else None indexers: List[Indexer] = [] diff --git a/packaging/package-indexer/indexer/deb_indexer.py b/packaging/package-indexer/indexer/deb_indexer.py index b0c46c08bd..5a563d5461 100644 --- a/packaging/package-indexer/indexer/deb_indexer.py +++ b/packaging/package-indexer/indexer/deb_indexer.py @@ -51,7 +51,7 @@ def __init__( gpg_key_passphrase: Optional[str], ) -> None: self.__apt_conf_file_path = apt_conf_file_path - self.__gpg_key_path = gpg_key_path.expanduser() + self.__gpg_key_path = gpg_key_path self.__gpg_key_passphrase = gpg_key_passphrase super().__init__( incoming_remote_storage_synchronizer=incoming_remote_storage_synchronizer,