Integrates alma image into build system

.github/workflows/build-citus-community-nightlies.yml

@@ -48,7 +48,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Clone tools branch
-        run: git clone -b v0.8.26 --depth=1  https://github.com/citusdata/tools.git tools
+        run: git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
 
       - name: Clone build branch
         run: git clone -b "${MAIN_BRANCH}" --depth=1  https://github.com/citusdata/packaging.git packaging

.github/workflows/build-package-test.yml

@@ -76,7 +76,7 @@ jobs:
           POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
 
       - name: Clone tools repo for test
-        run: git clone -b v0.8.26 --depth=1  https://github.com/citusdata/tools.git tools
+        run: git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
 
       - name: Execute packaging tests
         run: |

.github/workflows/build-package.yml

@@ -41,6 +41,7 @@ jobs:
           - centos,7
           - oraclelinux,8
           - oraclelinux,7
+          - almalinux,8
           - almalinux,9
         POSTGRES_VERSION:
           - 11
@@ -77,7 +78,7 @@ jobs:
           POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
 
       - name: Clone tools repo for test
-        run: git clone -b v0.8.26 --depth=1  https://github.com/citusdata/tools.git tools
+        run: git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
 
       - name: Execute packaging tests
         run: |

.github/workflows/build-pgazure-nightlies.yml

@@ -46,7 +46,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Clone tools branch
-        run: git clone -b v0.8.26 --depth=1  https://github.com/citusdata/tools.git tools
+        run: git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
 
       - name: Clone build branch
         run: git clone -b "${MAIN_BRANCH}" --depth=1  https://github.com/citusdata/packaging.git packaging

.github/workflows/image-health-check.yml

@@ -79,7 +79,7 @@ jobs:
           POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
 
       - name: Clone tools repo for test
-        run: git clone -b v0.8.26 --depth=1  https://github.com/citusdata/tools.git tools
+        run: git clone -b v0.8.27 --depth=1  https://github.com/citusdata/tools.git tools
 
       - name: Execute packaging tests
         run: |

.github/workflows/push-alma.yml

@@ -0,0 +1,50 @@
+name: Push Alma images
+
+env:
+  MAIN_BRANCH: "all-citus"
+  PACKAGE_CLOUD_REPO_NAME: "citusdata/community-nightlies"
+  PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }}
+  PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }}
+  PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }}
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+  DOCKERHUB_USER_NAME: ${{ secrets.DOCKERHUB_USER_NAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+on:
+  push:
+    branches:
+      - "**"
+  schedule:
+    - cron: "30 1 * * *"
+  workflow_dispatch:
+
+jobs:
+  build_package:
+    name: Build package
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - almalinux-8-pg12
+          - almalinux-8-pg13
+          - almalinux-8-pg14
+          - almalinux-8-pg15
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # This step is to fetch the images unanonymously to have higher bandwidth
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER_NAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and Publish Alma images
+        run: |
+          docker build -t citus/packaging:${{ matrix.image }} -f dockerfiles/${{ matrix.image }}/Dockerfile .
+          docker tag citus/packaging:${{ matrix.image }} citus/packaging-test:${{ matrix.image }}
+          docker push citus/packaging:${{ matrix.image }}
+          docker push citus/packaging-test:${{ matrix.image }}
+

.github/workflows/update-pgxn-version.yml

@@ -22,7 +22,7 @@ jobs:
         run: sudo apt install libcurl4-openssl-dev libssl-dev
 
       - name: Clone Tools branch
-        run: git clone --branch v0.8.26 https://github.com/citusdata/tools.git
+        run: git clone --branch v0.8.27 https://github.com/citusdata/tools.git
 
       - name: Install Python requirements
         run: python -m pip install -r tools/packaging_automation/requirements.txt

.github/workflows/update_package_properties.yml

@@ -23,7 +23,7 @@ jobs:
 
       # Runs a single command using the runners shell
       - name: Clone Tools branch
-        run: git clone --depth 1 --branch v0.8.26 https://github.com/citusdata/tools.git
+        run: git clone --depth 1 --branch v0.8.27 https://github.com/citusdata/tools.git
 
       # Runs a set of commands using the runners shell
       - name: Execute Package Properties Update

dockerfiles/almalinux-8-pg11/Dockerfile

@@ -0,0 +1,173 @@
+# vim:set ft=dockerfile:
+FROM almalinux:8
+RUN [[ almalinux != centos ]] || [[ 8 != 8 ]] || ( \
+    cd /etc/yum.repos.d/ && sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* \
+    && sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-* \
+    )
+
+RUN yum -y update
+
+RUN  [[ almalinux != centos ]] || [[ 8 != 8 ]] || ( \
+    dnf install epel-release -y && \
+    dnf install dnf-plugins-core -y && \
+    dnf install epel-release -y && \
+    dnf config-manager --set-enabled powertools && \
+    dnf install -y perl-IPC-Run \
+    )
+
+RUN  [[ almalinux != almalinux ]] || [[ 8 != 8 ]] || ( \
+    dnf install epel-release -y && \
+    dnf install dnf-plugins-core -y && \
+    dnf install epel-release -y && \
+    dnf config-manager --set-enabled powertools && \
+    dnf install -y perl-IPC-Run \
+    )
+
+RUN  [[ almalinux != oraclelinux ]] || [[ 8 != 8 ]] || ( \
+    curl -sO  https://vault.centos.org/centos/8/PowerTools/x86_64/os/Packages/perl-IO-Tty-1.12-11.el8.x86_64.rpm && \
+    curl -sO  https://vault.centos.org/centos/8/PowerTools/x86_64/os/Packages/perl-IPC-Run-0.99-1.el8.noarch.rpm && \
+    dnf install perl-IO-Tty-1.12-11.el8.x86_64.rpm -y  && \
+    dnf install perl-IPC-Run-0.99-1.el8.noarch.rpm -y && \
+    rm -f perl-IPC-Run-0.99-1.el8.noarch.rpm && \
+    rm -f perl-IO-Tty-1.12-11.el8.x86_64.rpm \
+    )
+
+RUN  [[ almalinux != almalinux ]] || [[ 8 != 9 ]] || ( \
+    dnf install epel-release -y && \
+    dnf -y install dnf-plugins-core && \
+    dnf config-manager --enable epel && \
+    dnf config-manager --set-enabled crb && \
+    dnf install -y perl-IPC-Run \
+    )
+
+# FIXME: Hack around docker/docker#10180
+RUN ( yum install -y yum-plugin-ovl || yum install -y yum-plugin-ovl || touch /var/lib/rpm/* ) \
+    && yum clean all
+
+# Enable some other repos for some dependencies in OL/7 
+# see https://yum.oracle.com/getting-started.html#installing-from-oracle-linux-yum-server
+RUN [[ almalinux != oraclelinux ]] || [[ 8 != 7 ]] || ( \
+       yum install -y oraclelinux-release-el7 oracle-softwarecollection-release-el7 oracle-epel-release-el7  oraclelinux-developer-release-el7 \
+       && yum-config-manager --enable \
+            ol7_software_collections \
+            ol7_developer \
+            ol7_developer_EPEL \
+            ol7_optional_latest \
+            ol7_optional_archive \
+            ol7_u9_base \
+            ol7_security_validation \
+            ol7_addons \
+         )
+
+# lz4 1.8 is preloaded in oracle 7 however, lz4-devel is not loaded and only 1.7.5 version exists
+# in oracle 7 repos. So package from centos repo was used
+# There is no package in oracle repos for lz4. Also it is not preloaded. So both lz4 and lz4-devel packages
+# were downloaded from centos el/6 repos
+RUN if [[ almalinux   == oraclelinux ]] && [[ 8 == 7 ]]; then yum install -y wget \
+        && wget http://mirror.centos.org/centos/7/os/x86_64/Packages/lz4-devel-1.8.3-1.el7.x86_64.rpm \
+        && rpm -ivh lz4-devel-1.8.3-1.el7.x86_64.rpm ; \
+        elif [[ almalinux   == oraclelinux ]] && [[ 8 == 6 ]]; then yum install -y wget \
+        && wget https://cbs.centos.org/kojifiles/packages/lz4/r131/1.el6/x86_64/lz4-r131-1.el6.x86_64.rpm \
+        && rpm -ivh lz4-r131-1.el6.x86_64.rpm \
+        && wget https://cbs.centos.org/kojifiles/packages/lz4/r131/1.el6/x86_64/lz4-devel-r131-1.el6.x86_64.rpm \
+        && rpm -ivh lz4-devel-r131-1.el6.x86_64.rpm;  \
+        else yum install -y lz4 lz4-devel; fi
+
+# install build tools and PostgreSQL development files
+RUN ( yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm  \
+    && [[ -z "epel-release" ]] || yum install -y epel-release) \
+    && yum groupinstall -y 'Development Tools' \
+    && yum install -y \
+        bzip2-devel \
+        libffi-devel \
+        xz-devel \
+        flex \
+        gcc-c++ \
+        hunspell-en \
+        libcurl-devel \
+        libicu-devel \
+        libstdc++-devel \
+        libxml2-devel \
+        libxslt-devel \
+        openssl-devel \
+        pam-devel \
+        readline-devel \
+        rpm-build \
+        rpmlint \
+        tar \
+        libzstd \
+        libzstd-devel \
+        llvm-toolset ccache spectool curl \
+    && ( [[ 8 != 8 ]] || dnf -qy module disable postgresql ) \
+    && yum install -y postgresql11-server postgresql11-devel \
+    && yum clean all
+
+# install jq to process JSON API responses
+RUN curl -sL https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 \
+         -o /usr/bin/jq \
+    && chmod +x /usr/bin/jq
+
+# install devtoolset-8-gcc on distros where it is available
+RUN { \
+        { yum search devtoolset-8-gcc 2>&1 | grep 'No matches found' ; } \
+        || yum install -y devtoolset-8-gcc devtoolset-8-libstdc++-devel; \
+    } \
+    && yum clean all
+
+# install sphinx on distros with python3
+RUN { \
+        { yum search python3-pip 2>&1 | grep 'No matches found' ; } \
+        || { \
+            yum install -y python3-pip && \
+            pip3 install sphinx==1.8 \
+            ; \
+        } \
+    } \
+    && yum clean all
+
+# install cmake, devtoolset and its dependencies to build azure sdk
+RUN yum -y install  perl-IPC-Cmd libuuid-devel cmake3
+
+# by default git 1.8.x is being installed in centos 7
+# Git 2.7.1 is minimum requirement for cmake so we remove and reinstall latests git from an up-to-date repo
+# Symbolic link is not being created for cmake from cmake3 by default. Therefore, we need to create the link as well.
+# devtoolset-8-gcc-c++ is required to compile azure sdk in pg azure storage project
+RUN [[ almalinux != centos ]] || [[ 8 != 7 ]] || ( \
+    yum -y install  perl-IPC-Cmd libuuid-devel cmake3 && \
+    ln -s /usr/bin/cmake3 /usr/bin/cmake  && \
+    yum -y remove git && \
+    yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm && \
+    yum -y install git && \
+    yum -y install  devtoolset-8-gcc-c++ && scl enable devtoolset-8 bash && \
+    yum -y install llvm-toolset-7 && \
+    yum clean all \
+    )
+
+# install python 3.8 to be able to execute tools scripts
+ARG PYTHON_VERSION=3.8.16
+RUN set -ex \
+    && curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \
+    && export PATH="$HOME/.pyenv/bin:$PATH" \
+    && pyenv update \
+    && pyenv install $PYTHON_VERSION \
+    && pyenv global $PYTHON_VERSION \
+    && pyenv rehash \
+    && echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc \
+    && echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc \
+    && echo 'eval "$(pyenv init -)"' >> ~/.bashrc
+
+# install llvm-toolset-7 to be able to install clang 5.0
+# clang 5.0 is required to build pg_azure_storage project
+ENV LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7/root/usr/lib64/:$LD_LIBRARY_PATH
+ENV PATH=/opt/rh/llvm-toolset-7/root/usr/bin:$PATH
+
+RUN touch /rpmlintrc \
+    && echo '%_build_pkgcheck %{_bindir}/rpmlint -f /rpmlintrc' >> /etc/rpm/macros
+
+# set PostgreSQL version, place scripts on path, and declare output volume
+ENV PGVERSION=11 \
+    PATH=/scripts:$PATH
+COPY scripts /scripts
+VOLUME /packages
+
+ENTRYPOINT ["/scripts/fetch_and_build_rpm"]