Add support for Java UDx (#248)

The Vertica server image is modified to install jre-headless (version 8) through
the package manager.  This is only done for the full image.  This allows us to
run Java UDx in that version of the image.  The minimal image will continue to
not support Java UDx.  This was done to keep the minimal image small as adding
jre-headless adds about 140mb to the size of the image.

New java udx tests were added but they haven't been added to any CI workflow
yet.  That is to follow.

This also changes the Kubernetes version used in the CI to 1.23.0. This change
was done locally so that tests could be run in a WSL2 environment.

changes/unreleased/Added-20220818-094450.yaml

@@ -0,0 +1,5 @@
+kind: Added
+body: Support for Java, C++ and Python UDx's
+time: 2022-08-18T09:44:50.678890201-03:00
+custom:
+  Issue: "248"

docker-vertica/Dockerfile

@@ -5,10 +5,11 @@
 
 ARG BASE_OS_VERSION="focal-20220801"
 ARG BUILDER_OS_VERSION="7.9.2009"
+ARG MINIMAL=""
 FROM centos:centos${BUILDER_OS_VERSION} as builder
 
 ARG VERTICA_RPM="vertica-x86_64.RHEL6.latest.rpm"
-ARG MINIMAL=""
+ARG MINIMAL
 ARG DBADMIN_GID=5000
 ARG DBADMIN_UID=5000
 
@@ -88,6 +89,11 @@ FROM ubuntu:${BASE_OS_VERSION}
 
 ARG DBADMIN_GID=5000
 ARG DBADMIN_UID=5000
+# Controls the version of jre to be installed by apt-get. The list of all
+# available jre packages can be queried through apt-cache. For instance,
+# "apt-cache search jre | grep jre"
+ARG JRE_PKG=openjdk-8-jre-headless
+ARG MINIMAL
 
 COPY --from=builder /opt/vertica /opt/vertica
 COPY --from=builder /home/dbadmin /home/dbadmin
@@ -129,6 +135,10 @@ RUN set -x \
   procps \
   sysstat \
   sudo \
+  # Install jre if not minimal
+  && if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") ]] ; then \
+    apt-get install -y --no-install-recommends $JRE_PKG; \
+  fi \
   && rm -rf /var/lib/apt/lists/* \
   # Make the "en_US.UTF-8" locale so vertica will be utf-8 enabled by default
   && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
@@ -138,9 +148,15 @@ RUN set -x \
   && /usr/sbin/useradd -r -m -s /bin/bash -g verticadba --uid ${DBADMIN_UID} dbadmin \
   # Allow passwordless sudo access from dbadmin
   && echo "dbadmin ALL=(ALL) NOPASSWD: ALL" | tee -a /etc/sudoers \
-  && echo "dbadmin -       nofile  65536" >> /etc/security/limits.conf 
-
-ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+  && echo "dbadmin -       nofile  65536" >> /etc/security/limits.conf \
+  # Set JAVA_HOME environment variable if not minimal, this will be loaded to all shells
+  && if [[ $MINIMAL != "YES" && $MINIMAL != "yes" ]]; then \
+    echo "JAVA_HOME=/usr" >> /etc/environment; \
+  fi \
+  # Create a symlink to python3 interpreter in vertica
+  && update-alternatives --install /usr/bin/python python /opt/vertica/oss/python3/bin/python3 1
+
+ENTRYPOINT [ "/usr/local/bin/docker-entrypoint.sh" ]
 
 # vertica port
 EXPOSE 5433

docker-vertica/docker-entrypoint.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
-set -e
+set -o errexit
+set -o xtrace
 
 start_cron(){
     # daemonizes, no need for &

docker-vertica/packages/cleanup.sh

@@ -24,15 +24,14 @@
 # wander around in the image looking for things you can remove
 rm -r -f \
    /opt/vertica/examples \
-   /opt/vertica/sdk \
    /opt/vertica/packages/*/examples \
    /opt/vertica/oss/python*/lib/python*/test \
    /opt/vertica/oss/python*/lib/python*/unittest/test \
    /opt/vertica/oss/python*/lib/python*/pip \
    /opt/vertica/oss/python*/lib/python*/site-packages/pip \
    /opt/vertica/oss/python*/lib/python*/config-[0-9]* \
    /opt/vertica/oss/python*/lib/python*/tkinter \
-   /opt/vertica/oss/python*/lib/python*/idlelib 
+   /opt/vertica/oss/python*/lib/python*/idlelib
 
 # cleanup many of the __pycache__ directories 
 find /opt/vertica/oss/ -type d -name "__pycache__" -exec rm -rf {} +
@@ -51,12 +50,13 @@ strip /opt/vertica/packages/*/lib/*.so* 2> /dev/null
 /opt/vertica/oss/python*/bin/python[0-9] \
     /tmp/package-checksum-patcher.py /opt/vertica/packages/*
 
-# (optional) minimal images remove packages that aren't auto installed
+# (optional) minimal images remove packages that aren't auto installed as well as the sdk folder
 if [ "$MINIMAL" = "YES" ] || [ "$MINIMAL" = "yes" ]
 then 
   cd /opt/vertica/packages
   for i in $(find . -name package.conf -exec grep Autoinstall=False {} + | cut -d"/" -f2)
   do
    rm -rf $i
   done
+  rm -r -f /opt/vertica/sdk
 fi

kuttl-test.yaml

@@ -19,6 +19,7 @@ skipDelete: true
 # These are overridden when running kuttl from the Makefile
 testDirs:
   - tests/e2e
+  - tests/e2e-udx
   - tests/e2e-extra
   - tests/e2e-server-upgrade
   - tests/e2e-operator-upgrade-overlays

scripts/kind.sh

@@ -19,7 +19,7 @@ set -e
 
 UPLOAD_IMAGES=
 TAG=latest
-KUBEVER=1.21.1
+KUBEVER=1.23.0
 IP_FAMILY=ipv4
 LISTEN_ALL_INTERFACES=N
 VSQL_PORT=5433

scripts/push-to-kind.sh

@@ -80,7 +80,8 @@ if [ $(( $# - $OPTIND )) -lt 0 ]
 then
     # Typical kind cluster name is something like: kind-matt1
     # We only want what is after 'kind-'
-    CLUSTER_NAME=$(kubectl config current-context | cut -d"-" -f2)
+    # Need to use "2-" after -f option to indidate everything after and including the 2nd field (in case we have cluster name like kind-cluster-test)
+    CLUSTER_NAME=$(kubectl config current-context | cut -d"-" -f2-)
 else
     CLUSTER_NAME=${@:$OPTIND:1}
 fi

scripts/setup-kustomize.sh

@@ -738,11 +738,11 @@ setup_creds_for_private_repo
 
 # Descend into each test and create the overlay kustomization.
 # The overlay is created in a directory like: overlay/<tc-name>
-for tdir in e2e/*/*/base e2e-extra/*/*/base e2e-server-upgrade/*/*/base e2e-operator-upgrade-overlays/*/*/base e2e-http-server/*/*/base
+for tdir in e2e/*/*/base e2e-extra/*/*/base e2e-server-upgrade/*/*/base e2e-operator-upgrade-overlays/*/*/base e2e-udx/*/*/base e2e-http-server/*/*/base
 do
     create_vdb_pod_kustomization $(dirname $tdir) $(basename $(realpath $tdir/../..))
 done
-for tdir in e2e/* e2e-extra/* e2e-disabled/* e2e-server-upgrade/* e2e-operator-upgrade-overlays/* e2e-http-server/*
+for tdir in e2e/* e2e-extra/* e2e-disabled/* e2e-server-upgrade/* e2e-operator-upgrade-overlays/* e2e-udx/* e2e-http-server/*
 do
     clean_communal_kustomization $tdir
 done

tests/e2e-udx/udx-java/00-create-creds.yaml

@@ -0,0 +1,18 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kustomize build ../../manifests/communal-creds/overlay | kubectl apply -f - --namespace $NAMESPACE
+  - script: kustomize build ../../manifests/priv-container-creds/overlay | kubectl apply -f - --namespace $NAMESPACE

tests/e2e-udx/udx-java/05-assert.yaml

@@ -0,0 +1,20 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    control-plane: controller-manager
+status:
+  phase: Running

tests/e2e-udx/udx-java/05-deploy-operator.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: sh -c "cd ../../.. && make deploy-operator NAMESPACE=$NAMESPACE"

tests/e2e-udx/udx-java/15-assert.yaml

@@ -0,0 +1,27 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-udx-java-sc1
+status:
+  replicas: 1
+---
+apiVersion: vertica.com/v1beta1
+kind: VerticaDB
+metadata:
+  name: v-udx-java
+status:
+  subclusters:
+    - installCount: 1

tests/e2e-udx/udx-java/15-setup-vdb.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "kustomize build setup-vdb/overlay | kubectl -n $NAMESPACE apply -f - "

tests/e2e-udx/udx-java/17-assert.yaml

@@ -0,0 +1,51 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-udx-java-sc1
+status:
+  replicas: 1
+  readyReplicas: 1
+---
+apiVersion: vertica.com/v1beta1
+kind: VerticaDB
+metadata:
+  name: v-udx-java
+status:
+  subclusters:
+    - installCount: 1
+      addedToDBCount: 1
+      upNodeCount: 1
+---
+# We enable spread encryption, so the cluster will restart as part of the create
+apiVersion: v1
+kind: Event
+reason: StopDBSucceeded
+source:
+  component: verticadb-operator
+involvedObject:
+  apiVersion: vertica.com/v1beta1
+  kind: VerticaDB
+  name: v-udx-java
+---
+apiVersion: v1
+kind: Event
+reason: ClusterRestartSucceeded
+source:
+  component: verticadb-operator
+involvedObject:
+  apiVersion: vertica.com/v1beta1
+  kind: VerticaDB
+  name: v-udx-java

tests/e2e-udx/udx-java/17-wait-for-createdb.yaml

@@ -0,0 +1 @@
+# Intentionally empty to give this step a name in kuttl

tests/e2e-udx/udx-java/20-assert.yaml

@@ -0,0 +1,22 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: integration-test-role
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: integration-test-rb

tests/e2e-udx/udx-java/20-rbac.yaml

@@ -0,0 +1,18 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl apply --namespace $NAMESPACE -f ../../manifests/rbac/base/rbac.yaml
+    ignoreFailure: true

tests/e2e-udx/udx-java/45-upload-files.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # upload the built library binaries
+  - command: kubectl -n $NAMESPACE cp ../../../bin/udx-libs/build v-udx-java-sc1-0:/opt/vertica/sdk/examples
+  # upload the expected outputs of executing sql files
+  - command: kubectl -n $NAMESPACE cp ./expected-outputs v-udx-java-sc1-0:/opt/vertica/sdk/examples

tests/e2e-udx/udx-java/50-assert.yaml

@@ -0,0 +1,23 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-verify-java-scalar-transform
+status:
+  containerStatuses:
+    - name: test
+      state:
+        terminated:
+          exitCode: 0