Merge pull request #110 from alltilla/apt-repo-in-r2

axoflow · Jul 3, 2024 · f29a1f7 · f29a1f7
2 parents 3a04ea2 + c9a34f9
commit f29a1f7
Show file tree

Hide file tree

Showing 10 changed files with 72 additions and 71 deletions.
diff --git a/.github/workflows/axosyslog-nightly.yml b/.github/workflows/axosyslog-nightly.yml
@@ -47,29 +47,31 @@ jobs:
       source-tarball-artifact-name: source-tarball
       dbld-image-mode: cache
 
-  # upload-packages:
-  #   needs: create-packages
-  #   uses: ./.github/workflows/upload-packages.yml
-  #   with:
-  #     pkg-type: nightly
-  #   secrets:
-  #     azure-sas-token: ${{ secrets.AZURE_SAS_TOKEN }}
-
-  # index-packages:
-  #   needs: upload-packages
-  #   uses: ./.github/workflows/index-packages.yml
-  #   with:
-  #     pkg-type: nightly
-  #   secrets:
-  #     config-base64: ${{ secrets.PACKAGE_INDEXER_CONFIG_BASE64 }}
-  #     gpg-key-base64: ${{ secrets.PACKAGE_INDEXER_GPG_KEY_BASE64 }}
-  #     gpg-key-passphrase: ${{ secrets.PACKAGE_INDEXER_GPG_KEY_PASSPHRASE }}
-
-  # test-packages:
-  #   needs: index-packages
-  #   uses: ./.github/workflows/test-apt-packages.yml
-  #   with:
-  #     pkg-type: nightly
+  upload-packages:
+    needs: create-packages
+    uses: ./.github/workflows/upload-packages.yml
+    with:
+      pkg-type: nightly
+    secrets:
+      r2-access-key: ${{ secrets.R2_ACCESS_KEY }}
+      r2-secret-key: ${{ secrets.R2_SECRET_KEY }}
+      r2-account-id: ${{ secrets.R2_ACCOUNT_ID }}
+
+  index-packages:
+    needs: upload-packages
+    uses: ./.github/workflows/index-packages.yml
+    with:
+      pkg-type: nightly
+    secrets:
+      config-base64: ${{ secrets.PACKAGE_INDEXER_CONFIG_BASE64 }}
+      gpg-key-base64: ${{ secrets.PACKAGE_INDEXER_GPG_KEY_BASE64 }}
+      gpg-key-passphrase: ${{ secrets.PACKAGE_INDEXER_GPG_KEY_PASSPHRASE }}
+
+  test-packages:
+    needs: index-packages
+    uses: ./.github/workflows/test-apt-packages.yml
+    with:
+      pkg-type: nightly
 
   publish-image:
     if: github.ref == 'refs/heads/main'

diff --git a/.github/workflows/draft-release.yml b/.github/workflows/draft-release.yml
@@ -83,13 +83,15 @@ jobs:
       source-tarball-artifact-name: release-tarball
       dbld-image-mode: build
 
-  # upload-packages:
-  #   needs: create-packages
-  #   uses: ./.github/workflows/upload-packages.yml
-  #   with:
-  #     pkg-type: stable
-  #   secrets:
-  #     azure-sas-token: ${{ secrets.AZURE_SAS_TOKEN }}
+  upload-packages:
+    needs: create-packages
+    uses: ./.github/workflows/upload-packages.yml
+    with:
+      pkg-type: stable
+    secrets:
+      r2-access-key: ${{ secrets.R2_ACCESS_KEY }}
+      r2-secret-key: ${{ secrets.R2_SECRET_KEY }}
+      r2-account-id: ${{ secrets.R2_ACCOUNT_ID }}
 
   create-draft-release:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/index-packages.yml b/.github/workflows/index-packages.yml
@@ -1,4 +1,4 @@
-name: Index packages in Azure/incoming
+name: Index packages in R2/incoming
 
 
 on:
@@ -34,7 +34,7 @@ jobs:
       DOCKER_IMAGE: package-indexer
       DOCKER_CONTAINER_NAME: package_indexer_container
       VERBOSE_LOG_PATH: index-packages-verbose.log
-      GPG_KEY_PATH: syslog_ng_ose_signing_key.sub.priv.asc
+      GPG_KEY_PATH: axosyslog_signing_key.sub.priv.asc
 
     if: github.repository_owner == 'axoflow'
 

diff --git a/.github/workflows/test-apt-packages.yml b/.github/workflows/test-apt-packages.yml
@@ -1,6 +1,6 @@
-# Please note that the syslog-ng git repository is not available in this workflow.
+# Please note that the axosyslog git repository is not available in this workflow.
 # It means that certain convenience/helper functions are not available (e.g. gh_output).
-# This is intentional as syslog-ng will be acquired from the APT repository.
+# This is intentional as axosyslog will be acquired from the APT repository.
 
 name: Run smoke tests on APT packages
 on:
@@ -11,7 +11,8 @@ on:
         type: string
 
 jobs:
-  TestPackagesWith:
+  test-apt-packages:
+    name: ${{ matrix.distro }}
     strategy:
       matrix:
         distro:
@@ -26,7 +27,7 @@ jobs:
           - "ubuntu:mantic"
         upgrade-from:
           - "debian-official"
-          - "syslog-ng-last"
+          - "axosyslog-last"
         exclude:
           # The official syslog-ng got removed from debian:testing because of pcre:
           # https://tracker.debian.org/news/1445295/syslog-ng-removed-from-testing/
@@ -48,18 +49,18 @@ jobs:
 
       - name: Add OSE repository
         run: |
-          wget -qO - https://ose-repo.syslog-ng.com/apt/syslog-ng-ose-pub.asc | gpg --dearmor > /usr/share/keyrings/ose-repo-archive-keyring.gpg
-          echo "deb [signed-by=/usr/share/keyrings/ose-repo-archive-keyring.gpg] https://ose-repo.syslog-ng.com/apt/ ${{ inputs.pkg-type }} $(echo ${{ matrix.distro }} | sed 's/:/-/g')" | tee --append /etc/apt/sources.list.d/syslog-ng-ose.list
+          wget -qO - https://pkg.axoflow.io/axoflow-code-signing-pub.asc | gpg --dearmor > /usr/share/keyrings/axoflow-code-signing-pub.gpg
+          echo "deb [signed-by=/usr/share/keyrings/axoflow-code-signing-pub.gpg] https://pkg.axoflow.io/apt ${{ inputs.pkg-type }} $(echo ${{ matrix.distro }} | sed 's/:/-/g')" | tee --append /etc/apt/sources.list.d/axosyslog.list
           apt-get update -qq
 
-      - name: Install the last but one syslog-ng OSE package
-        if: matrix.upgrade-from == 'syslog-ng-last'
+      - name: Install the last but one AxoSyslog package
+        if: matrix.upgrade-from == 'axosyslog-last'
         run: |
-          DEBIAN_FRONTEND=noninteractive apt-get install --yes syslog-ng=$(apt-cache madison syslog-ng | sed -n 2p | awk -F"|" '{print $2}' | sed 's/ //g')
+          DEBIAN_FRONTEND=noninteractive apt-get install --yes axosyslog=$(apt-cache madison axosyslog | sed -n 2p | awk -F"|" '{print $2}' | sed 's/ //g')
 
-      - name: Upgrade to the latest syslog-ng OSE package
+      - name: Upgrade to the latest AxoSyslog package
         run: |
-          DEBIAN_FRONTEND=noninteractive apt-get install --yes syslog-ng
+          DEBIAN_FRONTEND=noninteractive apt-get install --yes axosyslog
 
       - name: Get syslog-ng revision
         run: |
@@ -78,8 +79,8 @@ jobs:
       - name: Check if installed package version matches with install revision
         run: |
           echo "Installed revision value: ${{ steps.syslog_ng_revision.outputs.REVISION }}"
-          dpkg-query --show syslog-ng\*
-          dpkg-query -f='${db:Status-Status}\t${package}\t${version}\t\n' --show syslog-ng\* | grep ^installed | while read installed_syslog_ng_package ; do echo $installed_syslog_ng_package | grep ${{ steps.syslog_ng_revision.outputs.REVISION }} ; done
+          dpkg-query --show axosyslog\*
+          dpkg-query -f='${db:Status-Status}\t${package}\t${version}\t\n' --show axosyslog\* | grep ^installed | while read installed_axosyslog_package ; do echo $installed_axosyslog_package | grep ${{ steps.syslog_ng_revision.outputs.REVISION }} ; done
 
       - name: Check if syslog-ng can start with default config
         run: |

diff --git a/.github/workflows/upload-packages.yml b/.github/workflows/upload-packages.yml
@@ -1,4 +1,4 @@
-name: Upload package to Azure/incoming
+name: Upload package to R2/incoming
 
 
 on:
@@ -22,7 +22,11 @@ on:
           "ubuntu-mantic"
         ]'
     secrets:
-      azure-sas-token:
+      r2-access-key:
+        required: true
+      r2-secret-key:
+        required: true
+      r2-account-id:
         required: true
 
 
@@ -46,23 +50,15 @@ jobs:
           name: package-${{ matrix.distro }}
           path: package
 
-      - name: Calculate Azure path
+      - name: Upload packages to R2 incoming
+        working-directory: package
         run: |
-          if [[ ! "${{ inputs.pkg-type }}" =~ ^(stable|nightly)$ ]]; then
-            echo Unexpected input: pkg-type=${{ inputs.pkg-type }}
-            false
-          fi
-
-          AZURE_PATH="incoming/${{ inputs.pkg-type }}/${{ github.run_id }}"
-
-          echo "AZURE_PATH=${AZURE_PATH}" >> ${GITHUB_ENV}
-
-      - name: Upload packages to Azure incoming
-        uses: azure/CLI@v1
-        with:
-          inlineScript: |
-            az storage blob upload-batch \
-              --sas-token '${{ secrets.azure-sas-token }}' \
-              --account-name 'syslogngose' \
-              --source 'package' \
-              --destination '${{ env.AZURE_PATH }}'
+          export AWS_ACCESS_KEY_ID='${{ secrets.r2-access-key }}'
+          export AWS_SECRET_ACCESS_KEY='${{ secrets.r2-secret-key }}'
+          export AWS_DEFAULT_REGION='auto'
+          find * -type f -exec \
+            aws s3api put-object \
+              --endpoint-url https://${{ secrets.r2-account-id }}.r2.cloudflarestorage.com \
+              --bucket axosyslog-apt-incoming \
+              --key ${{ inputs.pkg-type }}/${{ github.run_id }}/{} \
+              --body {} \;
diff --git a/packaging/package-indexer/config/config.py b/packaging/package-indexer/config/config.py
@@ -138,7 +138,7 @@ def create_cdn(self, suite: str) -> cdn.CDN:
         return cls.from_config(options)
 
     def get_gpg_key_path(self) -> Path:
-        return self.__cfg["gpg-key-path"]
+        return Path(self.__cfg["gpg-key-path"]).expanduser().absolute()
 
     @staticmethod
     def __get_storage_implementation(vendor: str) -> Type[remote_storage_synchronizer.RemoteStorageSynchronizer]:

diff --git a/packaging/package-indexer/index-packages.py b/packaging/package-indexer/index-packages.py
@@ -117,7 +117,7 @@ def construct_indexers(cfg: Config, args: dict) -> List[Indexer]:
     indexed_remote_storage_synchronizer = cfg.create_indexed_remote_storage_synchronizer(suite)
     cdn = cfg.create_cdn(suite)
 
-    gpg_key_path = Path(cfg.get_gpg_key_path())
+    gpg_key_path = cfg.get_gpg_key_path()
     gpg_key_passphrase = stdin.read() if args["gpg_key_passphrase_from_stdin"] else None
 
     indexers: List[Indexer] = []

diff --git a/packaging/package-indexer/indexer/apt_conf/nightly.conf b/packaging/package-indexer/indexer/apt_conf/nightly.conf
@@ -4,7 +4,7 @@ APT {
   FTPArchive {
     Release {
       Architectures "amd64";
-      Origin "https://ose-repo.syslog-ng.com/apt";
+      Origin "https://pkg.axoflow.io/apt";
       Suite "nightly";
       Codename "nightly";
     };

diff --git a/packaging/package-indexer/indexer/apt_conf/stable.conf b/packaging/package-indexer/indexer/apt_conf/stable.conf
@@ -4,7 +4,7 @@ APT {
   FTPArchive {
     Release {
       Architectures "amd64";
-      Origin "https://ose-repo.syslog-ng.com/apt";
+      Origin "https://pkg.axoflow.io/apt";
       Suite "stable";
       Codename "stable";
     };

diff --git a/packaging/package-indexer/indexer/deb_indexer.py b/packaging/package-indexer/indexer/deb_indexer.py
@@ -51,7 +51,7 @@ def __init__(
         gpg_key_passphrase: Optional[str],
     ) -> None:
         self.__apt_conf_file_path = apt_conf_file_path
-        self.__gpg_key_path = gpg_key_path.expanduser()
+        self.__gpg_key_path = gpg_key_path
         self.__gpg_key_passphrase = gpg_key_passphrase
         super().__init__(
             incoming_remote_storage_synchronizer=incoming_remote_storage_synchronizer,