Add support for Java UDx

changes/unreleased/Added-20220818-094450.yaml

@@ -0,0 +1,5 @@
+kind: Added
+body: Support for Java, C++ and Python UDx's
+time: 2022-08-18T09:44:50.678890201-03:00
+custom:
+  Issue: "248"

docker-vertica/Dockerfile

@@ -5,10 +5,11 @@
 
 ARG BASE_OS_VERSION="focal-20220801"
 ARG BUILDER_OS_VERSION="7.9.2009"
+ARG MINIMAL=""
 FROM centos:centos${BUILDER_OS_VERSION} as builder
 
 ARG VERTICA_RPM="vertica-x86_64.RHEL6.latest.rpm"
-ARG MINIMAL=""
+ARG MINIMAL
 ARG DBADMIN_GID=5000
 ARG DBADMIN_UID=5000
 
@@ -88,6 +89,11 @@ FROM ubuntu:${BASE_OS_VERSION}
 
 ARG DBADMIN_GID=5000
 ARG DBADMIN_UID=5000
+# Controls the version of jre to be installed by apt-get. The list of all
+# available jre packages can be queried through apt-cache. For instance,
+# "apt-cache search jre | grep jre"
+ARG JRE_PKG=openjdk-8-jre-headless
+ARG MINIMAL
 
 COPY --from=builder /opt/vertica /opt/vertica
 COPY --from=builder /home/dbadmin /home/dbadmin
@@ -129,6 +135,10 @@ RUN set -x \
   procps \
   sysstat \
   sudo \
+  # Install jre if not minimal
+  && if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") ]] ; then \
+    apt-get install -y --no-install-recommends $JRE_PKG; \
+  fi \
   && rm -rf /var/lib/apt/lists/* \
   # Make the "en_US.UTF-8" locale so vertica will be utf-8 enabled by default
   && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
@@ -138,9 +148,15 @@ RUN set -x \
   && /usr/sbin/useradd -r -m -s /bin/bash -g verticadba --uid ${DBADMIN_UID} dbadmin \
   # Allow passwordless sudo access from dbadmin
   && echo "dbadmin ALL=(ALL) NOPASSWD: ALL" | tee -a /etc/sudoers \
-  && echo "dbadmin -       nofile  65536" >> /etc/security/limits.conf 
-
-ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+  && echo "dbadmin -       nofile  65536" >> /etc/security/limits.conf \
+  # Set JAVA_HOME environment variable if not minimal, this will be loaded to all shells
+  && if [[ $MINIMAL != "YES" && $MINIMAL != "yes" ]]; then \
+    echo "JAVA_HOME=/usr" >> /etc/environment; \
+  fi \
+  # Create a symlink to python3 interpreter in vertica
+  && update-alternatives --install /usr/bin/python python /opt/vertica/oss/python3/bin/python3 1
+
+ENTRYPOINT [ "/usr/local/bin/docker-entrypoint.sh" ]
 
 # vertica port
 EXPOSE 5433

docker-vertica/docker-entrypoint.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
-set -e
+set -o errexit
+set -o xtrace
 
 start_cron(){
     # daemonizes, no need for &

docker-vertica/packages/cleanup.sh

@@ -24,15 +24,14 @@
 # wander around in the image looking for things you can remove
 rm -r -f \
    /opt/vertica/examples \
-   /opt/vertica/sdk \
    /opt/vertica/packages/*/examples \
    /opt/vertica/oss/python*/lib/python*/test \
    /opt/vertica/oss/python*/lib/python*/unittest/test \
    /opt/vertica/oss/python*/lib/python*/pip \
    /opt/vertica/oss/python*/lib/python*/site-packages/pip \
    /opt/vertica/oss/python*/lib/python*/config-[0-9]* \
    /opt/vertica/oss/python*/lib/python*/tkinter \
-   /opt/vertica/oss/python*/lib/python*/idlelib 
+   /opt/vertica/oss/python*/lib/python*/idlelib
 
 # cleanup many of the __pycache__ directories 
 find /opt/vertica/oss/ -type d -name "__pycache__" -exec rm -rf {} +
@@ -51,12 +50,13 @@ strip /opt/vertica/packages/*/lib/*.so* 2> /dev/null
 /opt/vertica/oss/python*/bin/python[0-9] \
     /tmp/package-checksum-patcher.py /opt/vertica/packages/*
 
-# (optional) minimal images remove packages that aren't auto installed
+# (optional) minimal images remove packages that aren't auto installed as well as the sdk folder
 if [ "$MINIMAL" = "YES" ] || [ "$MINIMAL" = "yes" ]
 then 
   cd /opt/vertica/packages
   for i in $(find . -name package.conf -exec grep Autoinstall=False {} + | cut -d"/" -f2)
   do
    rm -rf $i
   done
+  rm -r -f /opt/vertica/sdk
 fi

kuttl-test.yaml

@@ -19,6 +19,7 @@ skipDelete: true
 # These are overridden when running kuttl from the Makefile
 testDirs:
   - tests/e2e
+  - tests/e2e-udx
   - tests/e2e-extra
   - tests/e2e-server-upgrade
   - tests/e2e-operator-upgrade-overlays

scripts/kind.sh

@@ -19,7 +19,7 @@ set -e
 
 UPLOAD_IMAGES=
 TAG=latest
-KUBEVER=1.21.1
+KUBEVER=1.23.0
 IP_FAMILY=ipv4
 LISTEN_ALL_INTERFACES=N
 VSQL_PORT=5433

scripts/push-to-kind.sh

@@ -80,7 +80,8 @@ if [ $(( $# - $OPTIND )) -lt 0 ]
 then
     # Typical kind cluster name is something like: kind-matt1
     # We only want what is after 'kind-'
-    CLUSTER_NAME=$(kubectl config current-context | cut -d"-" -f2)
+    # Need to use "2-" after -f option to indidate everything after and including the 2nd field (in case we have cluster name like kind-cluster-test)
+    CLUSTER_NAME=$(kubectl config current-context | cut -d"-" -f2-)
 else
     CLUSTER_NAME=${@:$OPTIND:1}
 fi

scripts/setup-kustomize.sh

@@ -738,11 +738,11 @@ setup_creds_for_private_repo
 
 # Descend into each test and create the overlay kustomization.
 # The overlay is created in a directory like: overlay/<tc-name>
-for tdir in e2e/*/*/base e2e-extra/*/*/base e2e-server-upgrade/*/*/base e2e-operator-upgrade-overlays/*/*/base
+for tdir in e2e/*/*/base e2e-extra/*/*/base e2e-server-upgrade/*/*/base e2e-operator-upgrade-overlays/*/*/base e2e-udx/*/*/base
 do
     create_vdb_pod_kustomization $(dirname $tdir) $(basename $(realpath $tdir/../..))
 done
-for tdir in e2e/* e2e-extra/* e2e-disabled/* e2e-server-upgrade/* e2e-operator-upgrade-overlays/*
+for tdir in e2e/* e2e-extra/* e2e-disabled/* e2e-server-upgrade/* e2e-operator-upgrade-overlays/* e2e-udx/*
 do
     clean_communal_kustomization $tdir
 done

tests/e2e-udx/udx-java/00-create-creds.yaml

@@ -0,0 +1,18 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kustomize build ../../manifests/communal-creds/overlay | kubectl apply -f - --namespace $NAMESPACE
+  - script: kustomize build ../../manifests/priv-container-creds/overlay | kubectl apply -f - --namespace $NAMESPACE

tests/e2e-udx/udx-java/05-assert.yaml

@@ -0,0 +1,20 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    control-plane: controller-manager
+status:
+  phase: Running

tests/e2e-udx/udx-java/05-deploy-operator.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: sh -c "cd ../../.. && make deploy-operator NAMESPACE=$NAMESPACE"

tests/e2e-udx/udx-java/15-assert.yaml

@@ -0,0 +1,27 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-udx-java-sc1
+status:
+  replicas: 1
+---
+apiVersion: vertica.com/v1beta1
+kind: VerticaDB
+metadata:
+  name: v-udx-java
+status:
+  subclusters:
+    - installCount: 1

tests/e2e-udx/udx-java/15-setup-vdb.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "kustomize build setup-vdb/overlay | kubectl -n $NAMESPACE apply -f - "

tests/e2e-udx/udx-java/17-assert.yaml

@@ -0,0 +1,51 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-udx-java-sc1
+status:
+  replicas: 1
+  readyReplicas: 1
+---
+apiVersion: vertica.com/v1beta1
+kind: VerticaDB
+metadata:
+  name: v-udx-java
+status:
+  subclusters:
+    - installCount: 1
+      addedToDBCount: 1
+      upNodeCount: 1
+---
+# We enable spread encryption, so the cluster will restart as part of the create
+apiVersion: v1
+kind: Event
+reason: StopDBSucceeded
+source:
+  component: verticadb-operator
+involvedObject:
+  apiVersion: vertica.com/v1beta1
+  kind: VerticaDB
+  name: v-udx-java
+---
+apiVersion: v1
+kind: Event
+reason: ClusterRestartSucceeded
+source:
+  component: verticadb-operator
+involvedObject:
+  apiVersion: vertica.com/v1beta1
+  kind: VerticaDB
+  name: v-udx-java

tests/e2e-udx/udx-java/17-wait-for-createdb.yaml

@@ -0,0 +1 @@
+# Intentionally empty to give this step a name in kuttl

tests/e2e-udx/udx-java/20-assert.yaml

@@ -0,0 +1,22 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: integration-test-role
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: integration-test-rb

tests/e2e-udx/udx-java/20-rbac.yaml

@@ -0,0 +1,18 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl apply --namespace $NAMESPACE -f ../../manifests/rbac/base/rbac.yaml
+    ignoreFailure: true

tests/e2e-udx/udx-java/45-upload-files.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # upload the built library binaries
+  - command: kubectl -n $NAMESPACE cp ../../../bin/udx-libs/build v-udx-java-sc1-0:/opt/vertica/sdk/examples
+  # upload the expected outputs of executing sql files
+  - command: kubectl -n $NAMESPACE cp ./expected-outputs v-udx-java-sc1-0:/opt/vertica/sdk/examples

tests/e2e-udx/udx-java/50-assert.yaml

@@ -0,0 +1,23 @@
+# (c) Copyright [2021-2022] Micro Focus or one of its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-verify-java-scalar-transform
+status:
+  containerStatuses:
+    - name: test
+      state:
+        terminated:
+          exitCode: 0