From ff9e1bbed10a08020d07f6525fd7f60240aa70fa Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Tue, 28 Jun 2022 11:20:02 -0400 Subject: [PATCH 01/30] Fix one command-line option in push-to-kind.sh Modify one command line option supplied to command cut in order to retrieve all fields after and including the second field (with - set as the field separator) --- scripts/push-to-kind.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/push-to-kind.sh b/scripts/push-to-kind.sh index a7f5d41de..6f78d38c9 100755 --- a/scripts/push-to-kind.sh +++ b/scripts/push-to-kind.sh @@ -80,7 +80,8 @@ if [ $(( $# - $OPTIND )) -lt 0 ] then # Typical kind cluster name is something like: kind-matt1 # We only want what is after 'kind-' - CLUSTER_NAME=$(kubectl config current-context | cut -d"-" -f2) + # Need to use "2-" after -f option to indidate everything after and including the 2nd field (in case we have cluster name like kind-cluster-test) + CLUSTER_NAME=$(kubectl config current-context | cut -d"-" -f2-) else CLUSTER_NAME=${@:$OPTIND:1} fi From e92a24677dfc2327176a493ad6b35eb60fa85ee2 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Fri, 1 Jul 2022 12:19:07 -0400 Subject: [PATCH 02/30] Change KUBEVER Change KUBEVER to 1.23.0 to ensure kind create cluster could succeed --- scripts/kind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/kind.sh b/scripts/kind.sh index 3bc39f28d..bcb9bc1e3 100755 --- a/scripts/kind.sh +++ b/scripts/kind.sh @@ -19,7 +19,7 @@ set -e UPLOAD_IMAGES= TAG=latest -KUBEVER=1.21.1 +KUBEVER=1.23.0 IP_FAMILY=ipv4 LISTEN_ALL_INTERFACES=N VSQL_PORT=5433 From fef8181d58ee126046b076df98f59100b1bbd594 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Sun, 3 Jul 2022 23:38:39 -0400 Subject: [PATCH 03/30] Vertica image with jre included Adding a new vertica image with jre pre-installed through apt; adding new make targets as well --- Makefile | 17 ++ docker-vertica-udx-java/.dockerignore | 1 + docker-vertica-udx-java/Dockerfile | 151 ++++++++++++++++ docker-vertica-udx-java/Makefile | 21 +++ docker-vertica-udx-java/README.md | 50 ++++++ docker-vertica-udx-java/dbadmin/.bash_profile | 14 ++ .../dbadmin/.ssh/authorized_keys | 1 + docker-vertica-udx-java/dbadmin/.ssh/id_rsa | 38 ++++ .../dbadmin/.ssh/id_rsa.pub | 1 + docker-vertica-udx-java/docker-entrypoint.sh | 37 ++++ docker-vertica-udx-java/packages/README.md | 9 + docker-vertica-udx-java/packages/cleanup.sh | 62 +++++++ .../packages/init.d.functions | 95 ++++++++++ .../packages/package-checksum-patcher.py | 165 ++++++++++++++++++ 14 files changed, 662 insertions(+) create mode 100644 docker-vertica-udx-java/.dockerignore create mode 100644 docker-vertica-udx-java/Dockerfile create mode 100644 docker-vertica-udx-java/Makefile create mode 100644 docker-vertica-udx-java/README.md create mode 100644 docker-vertica-udx-java/dbadmin/.bash_profile create mode 100644 docker-vertica-udx-java/dbadmin/.ssh/authorized_keys create mode 100644 docker-vertica-udx-java/dbadmin/.ssh/id_rsa create mode 100644 docker-vertica-udx-java/dbadmin/.ssh/id_rsa.pub create mode 100755 docker-vertica-udx-java/docker-entrypoint.sh create mode 100644 docker-vertica-udx-java/packages/README.md create mode 100644 docker-vertica-udx-java/packages/cleanup.sh create mode 100644 docker-vertica-udx-java/packages/init.d.functions create mode 100644 docker-vertica-udx-java/packages/package-checksum-patcher.py diff --git a/Makefile b/Makefile index 0bf970357..41cc630f4 100644 --- a/Makefile +++ b/Makefile @@ -84,6 +84,10 @@ export OPERATOR_IMG # Image URL to use for building/pushing of the vertica server VERTICA_IMG ?= $(IMG_REPO)vertica-k8s:$(TAG) export VERTICA_IMG + +VERTICA_UDX_JAVA_IMG ?= $(IMG_REPO)vertica-udx-java-k8s:$(TAG) +export VERTICA_UDX_JAVA_IMG + # This is the base image to use for some upgrade tests. We will always # upgrade to VERTICA_IMG, so BASE_VERTICA_IMG must be some image from a # version earlier than VERTICA_IMG. @@ -340,6 +344,19 @@ else scripts/push-to-kind.sh -i ${VERTICA_IMG} endif +.PHONY: docker-build-vertica-udx-java +docker-build-vertica-udx-java: docker-vertica-udx-java/Dockerfile ## Build vertica server docker image + cd docker-vertica-udx-java \ + && make VERTICA_IMG=${VERTICA_UDX_JAVA_IMG} MINIMAL_VERTICA_IMG=${MINIMAL_VERTICA_IMG} + +.PHONY: docker-push +docker-push-vertica-udx-java: ## Push vertica server docker image +ifeq ($(shell $(KIND_CHECK)), 0) + docker push ${VERTICA_UDX_JAVA_IMG} +else + scripts/push-to-kind.sh -i ${VERTICA_UDX_JAVA_IMG} +endif + .PHONY: bundle bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. ifneq ($(DEPLOY_WITH), $(filter $(DEPLOY_WITH), olm random)) diff --git a/docker-vertica-udx-java/.dockerignore b/docker-vertica-udx-java/.dockerignore new file mode 100644 index 000000000..b43bf86b5 --- /dev/null +++ b/docker-vertica-udx-java/.dockerignore @@ -0,0 +1 @@ +README.md diff --git a/docker-vertica-udx-java/Dockerfile b/docker-vertica-udx-java/Dockerfile new file mode 100644 index 000000000..2918c4602 --- /dev/null +++ b/docker-vertica-udx-java/Dockerfile @@ -0,0 +1,151 @@ +# Apache 2.0 license +# +# Copyright (c) 2021 Vertica +# + +ARG BASE_OS_VERSION="focal-20220316" +ARG BUILDER_OS_VERSION="7.9.2009" +FROM centos:centos${BUILDER_OS_VERSION} as builder + +ARG VERTICA_RPM="vertica-x86_64.RHEL6.latest.rpm" +ARG MINIMAL="" +ARG DBADMIN_GID=5000 +ARG DBADMIN_UID=5000 + +COPY ./packages/${VERTICA_RPM} /tmp/ +# this is a script which removes unnecessary stuff from the +# container image +COPY ./packages/cleanup.sh /tmp/ +COPY ./packages/package-checksum-patcher.py /tmp/ + +RUN set -x \ + # Update is needed to be confident that we're picking up + # fixed libraries. + && yum -q -y update \ + && yum install -y \ + cronie \ + dialog \ + iproute \ + mcelog \ + openssh-server \ + openssh-clients \ + openssl \ + which \ + zlib-devel \ + && /usr/sbin/groupadd -r verticadba --gid ${DBADMIN_GID} \ + && /usr/sbin/useradd -r -m -s /bin/bash -g verticadba --uid ${DBADMIN_UID} dbadmin \ + && yum localinstall -q -y /tmp/${VERTICA_RPM} \ + # Run install_vertica script to prepare environment + && /opt/vertica/sbin/install_vertica \ + --accept-eula \ + --debug \ + --dba-user-password-disabled \ + --failure-threshold NONE \ + --license CE \ + --hosts 127.0.0.1 \ + --no-system-configuration \ + -U \ + --data-dir /home/dbadmin \ + && mkdir -p /home/dbadmin/licensing/ce \ + && cp -r /opt/vertica/config/licensing/* /home/dbadmin/licensing/ce/ \ + && mkdir -p /home/dbadmin/logrotate \ + && cp -r /opt/vertica/config/logrotate /home/dbadmin/logrotate/ \ + && cp /opt/vertica/config/logrotate_base.conf /home/dbadmin/logrotate/ \ + && chown -R dbadmin:verticadba /opt/vertica \ + # reduce the size of the final image + && rm -rf /opt/vertica/lib64 \ + && yum clean all \ + && sh /tmp/cleanup.sh + +COPY dbadmin/.bash_profile /home/dbadmin/ +# Copy in a stable ssh key. This is done so that Vertica pods can communicate +# with pods running an older image. This is necessary when doing an online +# image change as the Vertica cluster will be running with two container +# versions at once. +COPY dbadmin/.ssh /home/dbadmin/.ssh +# prepare (put inside) entrypoint script and make right permissions +COPY ./docker-entrypoint.sh /usr/local/bin + +RUN set -x \ + && chmod a+x /usr/local/bin/docker-entrypoint.sh \ + && chown dbadmin:verticadba /home/dbadmin/.bash_profile \ + && chmod 600 /home/dbadmin/.bash_profile \ + && mkdir -p /root/.ssh \ + && cp -r /home/dbadmin/.ssh /root \ + && chmod 700 /root/.ssh \ + && chmod 600 /root/.ssh/* \ + && chmod 700 /home/dbadmin/.ssh \ + && chmod 600 /home/dbadmin/.ssh/* \ + && chown -R dbadmin:verticadba /home/dbadmin/ \ + # On older docker versions, the ownership of /opt/vertica is not preserved in + # the COPY. Working around this by changing it so that all files and + # directories are world writeable. + && chmod 777 -R /opt/vertica + +############################################################################################## +FROM ubuntu:${BASE_OS_VERSION} + +ARG DBADMIN_GID=5000 +ARG DBADMIN_UID=5000 + +COPY --from=builder /opt/vertica /opt/vertica +COPY --from=builder /home/dbadmin /home/dbadmin +COPY --from=builder /root/.ssh /root/.ssh +COPY --from=builder /var/spool/cron/ /var/spool/cron/crontabs +COPY --from=builder /usr/local/bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh + +ENV LANG en_US.UTF-8 +ENV TZ UTC +ENV PATH "$PATH:/opt/vertica/bin:/opt/vertica/sbin" +ENV DEBIAN_FRONTEND noninteractive + +COPY ./packages/init.d.functions /etc/rc.d/init.d/functions + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN set -x \ + # COPY may not preserve file permissions on older docker versions. This is a + # workaround for that. This will cause the image to increase a bit, but there + # isn't much we copy over for this directory. + && chown -R $DBADMIN_UID:$DBADMIN_GID /home/dbadmin \ + # Update is needed to be confident that we're picking up + # fixed libraries. + && apt-get -y update \ + && apt-get install -y --no-install-recommends \ + locales \ + cron \ + sysstat \ + ca-certificates \ + ntp \ + dialog \ + iproute2 \ + libkeyutils1\ + libz-dev \ + procps \ + krb5-user \ + logrotate \ + openssh-client \ + openssh-server \ + openssl \ + sudo \ + openjdk-8-jre \ + && rm -rf /var/lib/apt/lists/* \ + # Make the "en_US.UTF-8" locale so vertica will be utf-8 enabled by default + && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ + && mkdir -p /run/sshd \ + && ssh-keygen -q -A \ + && /usr/sbin/groupadd -r verticadba --gid ${DBADMIN_GID} \ + && /usr/sbin/useradd -r -m -s /bin/bash -g verticadba --uid ${DBADMIN_UID} dbadmin \ + # Allow passwordless sudo access from dbadmin + && echo "dbadmin ALL=(ALL) NOPASSWD: ALL" | tee -a /etc/sudoers \ + && echo "dbadmin - nofile 65536" >> /etc/security/limits.conf + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +# vertica port +EXPOSE 5433 +# agent port +EXPOSE 5444 +USER dbadmin +LABEL os-family="ubuntu" +LABEL image-name="vertica_k8s" +LABEL maintainer="K8s Team" diff --git a/docker-vertica-udx-java/Makefile b/docker-vertica-udx-java/Makefile new file mode 100644 index 000000000..b49f99747 --- /dev/null +++ b/docker-vertica-udx-java/Makefile @@ -0,0 +1,21 @@ +VERTICA_RPM?=vertica-x86_64.RHEL6.latest.rpm +BUILDER_OS_VERSION?=7.9.2009 +BASE_OS_VERSION?=focal-20220316 +VERTICA_IMG?=vertica-udx-java-k8s +MINIMAL_VERTICA_IMG?= +VERTICA_VERSION?=$(shell rpm --nosignature -qp --queryformat '%{VERSION}-%{RELEASE}' packages/$(VERTICA_RPM)) + +all: docker-build-vertica + +.PHONY: docker-build-vertica +docker-build-vertica: Dockerfile packages/package-checksum-patcher.py + docker build \ + -f Dockerfile \ + --label minimal=${MINIMAL_VERTICA_IMG} \ + --label os-version=${BASE_OS_VERSION} \ + --label vertica-version=${VERTICA_VERSION} \ + --build-arg MINIMAL=${MINIMAL_VERTICA_IMG} \ + --build-arg VERTICA_RPM=${VERTICA_RPM} \ + --build-arg BASE_OS_VERSION=${BASE_OS_VERSION} \ + --build-arg BUILDER_OS_VERSION=${BUILDER_OS_VERSION} \ + -t ${VERTICA_IMG} . diff --git a/docker-vertica-udx-java/README.md b/docker-vertica-udx-java/README.md new file mode 100644 index 000000000..badd28c94 --- /dev/null +++ b/docker-vertica-udx-java/README.md @@ -0,0 +1,50 @@ +# About + +* Maintained by: The [vertica-kubernetes community](https://github.com/vertica/vertica-kubernetes) +* Docker Community: [Docker Forums](https://forums.docker.com/), [Stack Overflow](https://stackoverflow.com/questions/tagged/docker) + +# Supported Tags +* [12.0.0-0-minimal, latest](https://github.com/vertica/vertica-kubernetes/blob/v1.6.0/docker-vertica/Dockerfile) +* [12.0.0-0](https://github.com/vertica/vertica-kubernetes/blob/v1.6.0/docker-vertica/Dockerfile) +* [11.1.1-2-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.5.0/docker-vertica/Dockerfile) +* [11.1.1-2](https://github.com/vertica/vertica-kubernetes/blob/v1.5.0/docker-vertica/Dockerfile) +* [11.1.1-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.4.0/docker-vertica/Dockerfile) +* [11.1.1-0](https://github.com/vertica/vertica-kubernetes/blob/v1.4.0/docker-vertica/Dockerfile) +* [11.1.0-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.3.0/docker-vertica/Dockerfile) +* [11.1.0-0](https://github.com/vertica/vertica-kubernetes/blob/v1.3.0/docker-vertica/Dockerfile) +* [11.0.2-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.2.0/docker-vertica/Dockerfile) +* [11.0.2-0](https://github.com/vertica/vertica-kubernetes/blob/v1.2.0/docker-vertica/Dockerfile) +* [11.0.1-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.1.0/docker-vertica/Dockerfile) +* [11.0.1-0](https://github.com/vertica/vertica-kubernetes/blob/v1.1.0/docker-vertica/Dockerfile) +* [11.0.0-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.0.0/docker-vertica/Dockerfile) +* [11.0.0-0](https://github.com/vertica/vertica-kubernetes/blob/v1.0.0/docker-vertica/Dockerfile) +* [10.1.1-0](https://github.com/vertica/vertica-kubernetes/blob/v0.1.0/docker-vertica/Dockerfile) + +# Quick Reference + +* [Vertica-Kubernetes GitHub repository](https://github.com/vertica/vertica-kubernetes) +* [Vertica Helm chart repository](https://github.com/vertica/charts) +* [Official Vertica Documentation](https://www.vertica.com/docs/latest/HTML/Content/Home.htm) +* Supported architectures: `amd64` + +# What is Vertica? + +Vertica is a unified analytics platform, based on a massively scalable architecture with the broadest set of analytical functions spanning event and time series, pattern matching, geospatial and end-to-end in-database machine learning. Vertica enables you to easily apply these powerful functions to the largest and most demanding analytical workloads, arming you and your customers with predictive business insights faster than any analytics data warehouse in the market. Vertica provides a unified analytics platform across major public clouds and on-premises data centers and integrates data in cloud object storage and HDFS without forcing you to move any of your data. + +https://www.vertica.com/ + +![](https://raw.githubusercontent.com/vertica/vertica-kubernetes/main/vertica-logo.png) + +# How to Use This Image + +This image runs the Vertica server that is optimized for use with the [Vertica operator](https://github.com/vertica/vertica-kubernetes/tree/main/docker-operator). The operator automates management and administrative tasks for an [Eon Mode](https://www.vertica.com/docs/latest/HTML/Content/Authoring/Eon/Architecture.htm) database in Kubernetes. + +Vertica provides two versions of this image, depending on whether you require the TensorFlow package: +- If you do not require the TensorFlow package, Vertica recommends downloading the image with the *-minimal* supported tag suffix. Because it does not include the TensorFlow package, it has a reduced size. The *-minimal* image is the default image included in the [Vertica Helm chart](https://github.com/vertica/charts). +- If you do require the TensorFlow package, download an image without the *-minimal* suffix. For details about TensorFlow and Vertica, see [Setting up TensorFlow Support in Vertica](https://www.vertica.com/docs/latest/HTML/Content/Authoring/AnalyzingData/MachineLearning/UsingExternalModels/UsingTensorFlow/TensorFlowExample.htm). + +For a brief overview on how to install and configure the operator, see the [Vertica GitHub repository](https://github.com/vertica/vertica-kubernetes). For an in-depth look at Vertica on Kubernetes, see the [Vertica documentation](https://www.vertica.com/docs/latest/HTML/Content/Authoring/Containers/ContainerizedVertica.htm). + +# License + +View the [license information](https://www.vertica.com/end-user-license-agreement-ce-version/) for this image. diff --git a/docker-vertica-udx-java/dbadmin/.bash_profile b/docker-vertica-udx-java/dbadmin/.bash_profile new file mode 100644 index 000000000..13f594aef --- /dev/null +++ b/docker-vertica-udx-java/dbadmin/.bash_profile @@ -0,0 +1,14 @@ +# .bash_profile + +# Get the aliases and functions +if [ -f ~/.bashrc ]; then + . ~/.bashrc +fi + +# User specific environment and startup programs + +PATH=$PATH:$HOME/.local/bin:$HOME/bin:/opt/vertica/bin +export PATH + +export LANG=en_US.UTF-8 +export TZ=UTC diff --git a/docker-vertica-udx-java/dbadmin/.ssh/authorized_keys b/docker-vertica-udx-java/dbadmin/.ssh/authorized_keys new file mode 100644 index 000000000..41346470c --- /dev/null +++ b/docker-vertica-udx-java/dbadmin/.ssh/authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2ExhX20Jeun+sp3aShRcUlHIaBdnUlCkqwf7h3vBVEV7KG5CUgabS7cYT/Qv2LQIfUsbLGq3EmVkihoy011vvQooFEV5qg5XOFESsCjp0xL7uYw8cB1YihqDsP31boep0tWs28QL2JVGn0snhgRwo/Nxa01n6t87+9CULRDwsKqnp1iDLPfbrEf3jHAU9niEDmoPMNfR+NhbYO2XsEGKMqr3Fx7bCtP0vVSiijLpvlp2QC5CfjK6oGpakqruk26LRiEe1drzz5fxn7ndh5kIwjUoOYfMTbt9/TGKTT8KTj7oVaH5oaJ4Reh9ymWzvQPCLci/XAbERy7h5bXWG05px4fxTT5ILX9dnT21XH+kSM9fdnqWbBj1EexCGraezMq9EaNCeMHzXUxt3iDhVlLrN2SkEliBDdEEilucjkPUqcJtzdJAzyKBOHQGWkCCercNiLCgrltXlhxXqQp0kIxx+pjyeHGxqoUSr0gnSsbbaM2K/kMZ8uDjyqqoHlFAsrgM= spilchen@sputnik diff --git a/docker-vertica-udx-java/dbadmin/.ssh/id_rsa b/docker-vertica-udx-java/dbadmin/.ssh/id_rsa new file mode 100644 index 000000000..13f47a2c9 --- /dev/null +++ b/docker-vertica-udx-java/dbadmin/.ssh/id_rsa @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAthMYV9tCXrp/rKd2koUXFJRyGgXZ1JQpKsH+4d7wVRFeyhuQlIGm +0u3GE/0L9i0CH1LGyxqtxJlZIoaMtNdb70KKBRFeaoOVzhRErAo6dMS+7mMPHAdWIoag7D +99W6HqdLVrNvEC9iVRp9LJ4YEcKPzcWtNZ+rfO/vQlC0Q8LCqp6dYgyz326xH94xwFPZ4h +A5qDzDX0fjYW2Dtl7BBijKq9xce2wrT9L1Uoooy6b5adkAuQn4yuqBqWpKq7pNui0YhHtX +a88+X8Z+53YeZCMI1KDmHzE27ff0xik0/Ck4+6FWh+aGieEXofcpls70Dwi3Iv1wGxEcu4 +eW11htOaceH8U0+SC1/XZ09tVx/pEjPX3Z6lmwY9RHsQhq2nszKvRGjQnjB811Mbd4g4VZ +S6zdkpBJYgQ3RBIpbnI5D1KnCbc3SQM8igTh0BlpAgnq3DYiwoK5bV5YcV6kKdJCMcfqY8 +nhxsaqFEq9IJ0rG22jNiv5DGfLg48qqqB5RQLK4DAAAFiKo3XLCqN1ywAAAAB3NzaC1yc2 +EAAAGBALYTGFfbQl66f6yndpKFFxSUchoF2dSUKSrB/uHe8FURXsobkJSBptLtxhP9C/Yt +Ah9SxssarcSZWSKGjLTXW+9CigURXmqDlc4URKwKOnTEvu5jDxwHViKGoOw/fVuh6nS1az +bxAvYlUafSyeGBHCj83FrTWfq3zv70JQtEPCwqqenWIMs99usR/eMcBT2eIQOag8w19H42 +Ftg7ZewQYoyqvcXHtsK0/S9VKKKMum+WnZALkJ+MrqgalqSqu6TbotGIR7V2vPPl/Gfud2 +HmQjCNSg5h8xNu339MYpNPwpOPuhVofmhonhF6H3KZbO9A8ItyL9cBsRHLuHltdYbTmnHh +/FNPkgtf12dPbVcf6RIz192epZsGPUR7EIatp7Myr0Ro0J4wfNdTG3eIOFWUus3ZKQSWIE +N0QSKW5yOQ9Spwm3N0kDPIoE4dAZaQIJ6tw2IsKCuW1eWHFepCnSQjHH6mPJ4cbGqhRKvS +CdKxttozYr+Qxny4OPKqqgeUUCyuAwAAAAMBAAEAAAGAMz20gCVIzHgYLnMCv8GM+VG9uu +0inN0FqO7n2JFxeb9lueOAEQtb88jPEhkJIMMR8dbN86MSfF3b5AeR2eMyEGfdiAeXVsqp +glHZ8cD/KpWkVlqyieTWrOjOqS5FDWDIJwhXoCK7/qIKFP0Rq9E2J6LiMP2COTk5nC8Bre +YKOTPYleKZYBmUQBvUI/Ji1zCFggoeRX55zHtUeo5c5M7HrWbv+ZV3LdzZ1FRXF0gbwAPX +XA6lKQHitZSmCLm0DXVyJ5awSrNTRcwrIy1vGVES16pUvJSXioWP9SOuAUvYuuzzrqxlZ6 +E8hImC8odVzxV452KoXqrxDmEsZGPqqVPZ9WE3LJLkhmVfLwCzM2penT6xJ8hig8SOhUHo +pWYuWaTP+d1XAPntAnDZLPkneobnEhRgWgBj4JxS2VtoJH2uI0hhH7q36zsSZ+e8KKYzdS +zAmyEWnBsKNYbP0a658/BsMkJNqstWh1wwVZ/U3rkuJtHYgSBXYrtmzZzcOXYzRdjhAAAA +wArbQfr2ItNim9Kw4Y1Cu5Eit6uGwNxj6X0OWtBAHSIaRV5CQq5mCMDkE3D+8tLzxVyCcw +/W9laTHc0ge5cKeYVR4UNaW1LK3vAXx4bXcrEmWyI/QvqSafwIvvkzQXazcCmjV3cnDLG0 +e5R6iqT5eNG6hCwqZ5kypmpOuxyi9xIUDfsSFiFX1VwPCrTCmA7zSOaf6pv17PlqfiKFGl +b0tGVV4ySvQNKa61YSL6qszaMkP3U2SgTRAVkk+m4Dm+SLQwAAAMEA2mHPBC/JVPzxX+ua +PdFBOKigisyoYaxdMbx6Do/ev8Tle5Y1nakBs1EQr2G7QdeWA9jV+Yw6HRsAyEt5x4mVPq +bZJBmbthLI/bxQVyQP4RauXgAVlVnURpusJP9joEfw8urL7iXCc5dYdVzHRrO6zWmEucAS +umWjxdAna6rYYidZzKCXp+rT2ztKaM1zEHB4CrS+Qb/F7AZvGIBWRO6CJG8Qxw1zQApC5Z +6IOKS5lhzgtC2iWIxNfUXdXxxNLigvAAAAwQDVcDPwr4XX4Yj1WUalKYszj28tnuUbLQQu +T0uppO9WPVSweOWiSy8/X9zLZxpaebtmLrOfHr9HfouIRCQXslnn2NxC5DjwSO0KP9vt+U +Qu4AB9p6LJHVxAqBGALVpzXaVokfAOzPaDE3zct58y/S6DagAk2b8gyfHW8XpgQ5IU3jWa +Oi/fRUObjDb6plyPvExJgfAyxQq/eHi7PPrULf6HGJTBq2icpQCHEsVMy3bafXNKY9sXMm +TrxAuVuaAdDm0AAAAQc3BpbGNoZW5Ac3B1dG5pawECAw== +-----END OPENSSH PRIVATE KEY----- diff --git a/docker-vertica-udx-java/dbadmin/.ssh/id_rsa.pub b/docker-vertica-udx-java/dbadmin/.ssh/id_rsa.pub new file mode 100644 index 000000000..41346470c --- /dev/null +++ b/docker-vertica-udx-java/dbadmin/.ssh/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2ExhX20Jeun+sp3aShRcUlHIaBdnUlCkqwf7h3vBVEV7KG5CUgabS7cYT/Qv2LQIfUsbLGq3EmVkihoy011vvQooFEV5qg5XOFESsCjp0xL7uYw8cB1YihqDsP31boep0tWs28QL2JVGn0snhgRwo/Nxa01n6t87+9CULRDwsKqnp1iDLPfbrEf3jHAU9niEDmoPMNfR+NhbYO2XsEGKMqr3Fx7bCtP0vVSiijLpvlp2QC5CfjK6oGpakqruk26LRiEe1drzz5fxn7ndh5kIwjUoOYfMTbt9/TGKTT8KTj7oVaH5oaJ4Reh9ymWzvQPCLci/XAbERy7h5bXWG05px4fxTT5ILX9dnT21XH+kSM9fdnqWbBj1EexCGraezMq9EaNCeMHzXUxt3iDhVlLrN2SkEliBDdEEilucjkPUqcJtzdJAzyKBOHQGWkCCercNiLCgrltXlhxXqQp0kIxx+pjyeHGxqoUSr0gnSsbbaM2K/kMZ8uDjyqqoHlFAsrgM= spilchen@sputnik diff --git a/docker-vertica-udx-java/docker-entrypoint.sh b/docker-vertica-udx-java/docker-entrypoint.sh new file mode 100755 index 000000000..71627e1e5 --- /dev/null +++ b/docker-vertica-udx-java/docker-entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -e + +start_cron(){ + # daemonizes, no need for & + sudo /usr/sbin/cron +} + +# We copy back the files normally stored in /opt/vertica/config/. We do this +# because we have a Persistent Volume that backs /opt/vertica/config, so +# it starts up empty and must be populated +copy_config_files() { + mkdir -p /opt/vertica/config/licensing + + mv /home/dbadmin/logrotate/* /opt/vertica/config/ 2>/dev/null || true + + cp -r /home/dbadmin/licensing/ce/* /opt/vertica/config/licensing 2>/dev/null || true + chmod -R ugo+r,u+rw /opt/vertica/config/licensing +} + +# Ensure all PV paths are owned by dbadmin. This is done for some PVs that +# start with restrictive ownership. +ensure_path_is_owned_by_dbadmin() { + # -z is to needed in case input arg is empty + [ -z "$1" ] || [ "$(stat -c "%U" "$1")" == "dbadmin" ] || sudo chown -R dbadmin:verticadba "$1" +} + +start_cron +ensure_path_is_owned_by_dbadmin /opt/vertica/config +ensure_path_is_owned_by_dbadmin /opt/vertica/log +ensure_path_is_owned_by_dbadmin $DATA_PATH +ensure_path_is_owned_by_dbadmin $DEPOT_PATH +copy_config_files + +echo "Vertica container is now running" + +sudo /usr/sbin/sshd -D diff --git a/docker-vertica-udx-java/packages/README.md b/docker-vertica-udx-java/packages/README.md new file mode 100644 index 000000000..6d129f57c --- /dev/null +++ b/docker-vertica-udx-java/packages/README.md @@ -0,0 +1,9 @@ +You have to download Vertica RPM package before building docker image (even if you are +running on Ubuntu or Debian, the contents of the container use a CentOS environment). + +Download either [Community Edition](https://www.vertica.com/try/)(registration required) +or you can download Enterprise edition, if you are Vertica customer. + +Store RPM packages into this folder, and remember to tell make the +name of the RPM when you build the image. + diff --git a/docker-vertica-udx-java/packages/cleanup.sh b/docker-vertica-udx-java/packages/cleanup.sh new file mode 100644 index 000000000..1bb57669a --- /dev/null +++ b/docker-vertica-udx-java/packages/cleanup.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Remove things not necessary for a non-interactive Kubernetes pod +# running Vertica + +# This script is hand-crafted: +# Build an image named bigimage +# +# docker run -it --entrypoint /bin/bash bigimage +# +# wander around in the image looking for things you can remove +rm -r -f \ + /opt/vertica/examples \ + /opt/vertica/sdk \ + /opt/vertica/packages/*/examples \ + /opt/vertica/oss/python*/lib/python*/test \ + /opt/vertica/oss/python*/lib/python*/unittest/test \ + /opt/vertica/oss/python*/lib/python*/pip \ + /opt/vertica/oss/python*/lib/python*/site-packages/pip \ + /opt/vertica/oss/python*/lib/python*/config-[0-9]* \ + /opt/vertica/oss/python*/lib/python*/tkinter \ + /opt/vertica/oss/python*/lib/python*/idlelib + +# cleanup many of the __pycache__ directories +find /opt/vertica/oss/ -type d -name "__pycache__" -exec rm -rf {} + + +# many of these directories contain things that aren't binaries +# thus divert error output to /dev/null +strip /opt/vertica/bin/* 2> /dev/null +strip /opt/vertica/lib/*.so* +strip /opt/vertica/oss/python*/bin/* 2> /dev/null +strip /opt/vertica/oss/python*/lib/libpython*.a +strip /opt/vertica/oss/python*/lib/python*/lib-dynload/*.so* + +# stripping the packages directory saves about 900MB, but... +strip /opt/vertica/packages/*/lib/*.so* 2> /dev/null +# it changes the checksums used to verify the libraries when loaded +/opt/vertica/oss/python*/bin/python[0-9] \ + /tmp/package-checksum-patcher.py /opt/vertica/packages/* + +# (optional) minimal images remove packages that aren't auto installed +if [ "$MINIMAL" = "YES" ] || [ "$MINIMAL" = "yes" ] +then + cd /opt/vertica/packages + for i in $(find . -name package.conf -exec grep Autoinstall=False {} + | cut -d"/" -f2) + do + rm -rf $i + done +fi diff --git a/docker-vertica-udx-java/packages/init.d.functions b/docker-vertica-udx-java/packages/init.d.functions new file mode 100644 index 000000000..b671a8cbc --- /dev/null +++ b/docker-vertica-udx-java/packages/init.d.functions @@ -0,0 +1,95 @@ +# -*-Shell-script-*- +# +# borrowed from /etc/rc/init.d/functions --- functions used +# by the vertica_agent script +# + +# Set up a default search path. +PATH="/sbin:/usr/sbin:/bin:/usr/bin" +export PATH + +# __proc_pids {program} [pidfile] +# Set $pid to pids from /var/run* for {program}. $pid should be declared +# local in the caller. +# Returns LSB exit code for the 'status' action. +__pids_var_run() { + local base=${1##*/} + local pid_file=${2:-/var/run/$base.pid} + + pid= + if [ -f "$pid_file" ] ; then + local line p + read line < "$pid_file" + for p in $line ; do + [ -z "${p//[0-9]/}" -a -d "/proc/$p" ] && pid="$pid $p" + done + if [ -n "$pid" ]; then + return 0 + fi + return 1 # "Program is dead and /var/run pid file exists" + fi + return 3 # "Program is not running" +} + +# Output PIDs of matching processes, found using pidof +__pids_pidof() { + pidof -c -o $$ -o $PPID -o %PPID -x "$1" || \ + pidof -c -o $$ -o $PPID -o %PPID -x "${1##*/}" +} + +status() { + local base pid pid_file= + + # Test syntax. + if [ "$#" = 0 ] ; then + echo $"Usage: status [-p pidfile] {program}" + return 1 + fi + if [ "$1" = "-p" ]; then + pid_file=$2 + shift 2 + fi + base=${1##*/} + + # First try "pidof" + __pids_var_run "$1" "$pid_file" + RC=$? + if [ -z "$pid_file" -a -z "$pid" ]; then + pid="$(__pids_pidof "$1")" + fi + if [ -n "$pid" ]; then + echo $"${base} (pid $pid) is running..." + return 0 + fi + + case "$RC" in + 0) + echo $"${base} (pid $pid) is running..." + return 0 + ;; + 1) + echo $"${base} dead but pid file exists" + return 1 + ;; + esac + # See if /var/lock/subsys/${base} exists + if [ -f /var/lock/subsys/${base} ]; then + echo $"${base} dead but subsys locked" + return 2 + fi + echo $"${base} is stopped" + return 3 +} + +echo_success() { + echo " OK " + return 0 +} + +echo_failure() { + echo -n "[" + echo -n $"FAILED" + echo -n "]" + echo -ne "\r" + return 1 +} diff --git a/docker-vertica-udx-java/packages/package-checksum-patcher.py b/docker-vertica-udx-java/packages/package-checksum-patcher.py new file mode 100644 index 000000000..4c28c1fba --- /dev/null +++ b/docker-vertica-udx-java/packages/package-checksum-patcher.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python + +# Copyright 2021 Vertica + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +To save about 900MB of space, we strip the package libraries in +/opt/vertica/packages. + +Unfortunately, each directory in /opt/vertica/packages has two files: + - package.conf + - ddl/isinstalled.sql + +package.conf looks like this: + +[Info] +Description=Amazon Web Services Package +Autoinstall=True +Version=11.0.0 +md5sum=b22b4486faa8df8d70fc399ac5a85521 + +and isinstalled.sql looks like this: + +SELECT (COUNT(0) = 5) FROM user_libraries +INNER JOIN user_library_manifest ON user_libraries.lib_name = user_library_manifest.lib_name +WHERE user_library_manifest.lib_name = 'awslib' +AND user_libraries.schema_name = 'public' +AND (user_libraries.md5_sum = 'b22b4486faa8df8d70fc399ac5a85521' + OR public.length('b22b4486faa8df8d70fc399ac5a85521') = 7); + +by stripping the libraries, we've changed their checksums, so these +files are no longer accurate. + +So, we patch the relevant files. + +This runs in a pretty stripped-down environment, so we try to keep ourselves +to core python. +""" +import os +import os.path +import re +import sys +import subprocess + +progname = sys.argv[0] + +def parse_conf(dir): + """ + Extract the Autoinstall and md5sum fields from the + package.conf file + + Args: + - dir: directory name + Returns: + (autoinstall value, checksum value), where + autoinstall value is "True" or "False" + """ + md5pat = re.compile('^md5sum=(.*)$') + autopat = re.compile('^Autoinstall=(.*)$') + checksum = None + autoinstall = None + + with open(dir + '/package.conf', 'r') as fp: + for line in fp: + m = md5pat.match(line) + if m: + checksum = m.group(1) + else: + m = autopat.match(line) + if m: + autoinstall = m.group(1) + if autoinstall and checksum: + return (autoinstall, checksum) + return (autoinstall, checksum) + +def patch_file(fname, old_checksum, new_checksum): + """ + Replace the old checksum with the new checksum in file + Args: + - fname: name of file to patch + - old_checksum: *string* containing the old checksum value to be replaced + - new_checksum: *string* containing the new checksum to insert + Returns: + None + + Backs fname up as fname~ + """ + print(f' file {fname} {old_checksum} -> {new_checksum}') + file_new = fname + '.new' + file_backup = fname + '~' + xsumpat = re.compile(old_checksum) + with open(fname, 'r') as ifp: + with open(file_new, 'w') as ofp: + for line in ifp: + edited = re.sub(xsumpat, new_checksum, line) + print(f'{edited}', file=ofp, end='') + + try: + os.remove(file_backup) + except FileNotFoundError: + # the first time through, the file won't be found + pass + os.remove(fname) + os.rename(file_new, fname) + +def patch_dir(dir, old_checksum): + """ + Patch the package.conf and ddl/isinstalled.sql files in directory dir + Args: + - dir: the package directory + - old_checksum: the old checksum to be replaced with the new checksum + Returns: + None + """ + libname = 'lib' + os.path.basename(dir) + '.so' + xsum_out = str(subprocess.check_output(['md5sum', dir + '/lib/' + libname])) + new_checksum = xsum_out.split(' ')[0] + # converting byte-like object to string prefixes the string with "b'" + if new_checksum.startswith("b'"): + new_checksum = new_checksum[2:] + patch_file(dir + '/package.conf', old_checksum, new_checksum) + patch_file(dir + '/ddl/isinstalled.sql', old_checksum, new_checksum) + +def process_dir(dir): + """ + Process a package directory: + - figure out if the package is auto-installed + - if so, patch its checksum + Skips packages that aren't automatically installed (maybe it + shouldn't --- how to install those, after all?) + """ + (autoinstall, checksum) = parse_conf(dir) + if checksum: + print(f'patching directory {dir}') + patch_dir(dir, checksum) + else: + # no package.conf file, or no checksum in it --> probably not set up + # with standard package mechanism. + print(f'skipping directory {dir} with no checksum in package.conf file') + +def main(argv): + """ + Iterate over the list of files passed as arguments + """ + if len(argv) < 2: + print(f'Usage: {progname} packagedir1 [packagedir2...]', file=sys.stderr) + sys.exit(1) + # argv[0] is the command name + for dir in argv[1:]: + process_dir(dir) + +if __name__ == '__main__': + main(sys.argv) + sys.exit(0) From 972523d9985a9e53ad5fec9a84701a1059714069 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Thu, 7 Jul 2022 19:02:21 -0400 Subject: [PATCH 04/30] Implementation --- .gitignore | 4 ++ api/v1beta1/verticadb_types.go | 14 ++++++ docker-udx-java/Dockerfile | 17 +++++++ pkg/builder/builder.go | 90 ++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100644 docker-udx-java/Dockerfile diff --git a/.gitignore b/.gitignore index 68dfb17f3..ec5dccecd 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,7 @@ config/release-manifests # bundle files are all auto generated, so omit them docker-bundle/Dockerfile bundle/ + + +docker-udx-java/java-se-8u41-ri +docker-udx-java/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz \ No newline at end of file diff --git a/api/v1beta1/verticadb_types.go b/api/v1beta1/verticadb_types.go index 81fb5786e..4f66c0cf0 100644 --- a/api/v1beta1/verticadb_types.go +++ b/api/v1beta1/verticadb_types.go @@ -302,6 +302,20 @@ type VerticaDBSpec struct { // creation of the CR. If set for initPolicy other than Create, then it has // no effect. EncryptSpreadComm string `json:"encryptSpreadComm,omitempty"` + + Init []InitDescriptor +} + +type InitDescriptor struct { + Name string + Image string + Command string + Paths []InitPaths +} + +type InitPaths struct { + InitPath string + ServerPath string } // LocalObjectReference is used instead of corev1.LocalObjectReference and behaves the same. diff --git a/docker-udx-java/Dockerfile b/docker-udx-java/Dockerfile new file mode 100644 index 000000000..c7fc82df7 --- /dev/null +++ b/docker-udx-java/Dockerfile @@ -0,0 +1,17 @@ +FROM phusion/baseimage:focal-1.2.0 as downloader + +CMD ["/sbin/my_init"] + +WORKDIR /tmp/ + +# ---- Option 1: download from web + +# RUN set -x \ +# && apt install wget \ +# && wget https://download.java.net/openjdk/jdk8u41/ri/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz + +# ---- Option 2: copy from local + +COPY ./java-se-8u41-ri /tmp/ + +RUN apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 3d2b68031..06ef84021 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -20,6 +20,7 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" vapi "github.com/vertica/vertica-kubernetes/api/v1beta1" @@ -117,6 +118,12 @@ func buildVolumeMounts(vdb *vapi.VerticaDB) []corev1.VolumeMount { volMnts = append(volMnts, buildCertSecretVolumeMounts(vdb)...) volMnts = append(volMnts, vdb.Spec.VolumeMounts...) + if len(vdb.Spec.Init) > 0 { + for _, ic := range vdb.Spec.Init { + volMnts = append(volMnts, buildMainContainerUDXVolumeMounts(vdb, &ic)...) + } + } + return volMnts } @@ -182,6 +189,11 @@ func buildVolumes(vdb *vapi.VerticaDB) []corev1.Volume { } vols = append(vols, buildCertSecretVolumes(vdb)...) vols = append(vols, vdb.Spec.Volumes...) + if len(vdb.Spec.Init) > 0 { + for _, ic := range vdb.Spec.Init { + vols = append(vols, buildUDXVolumes(vdb, &ic)...) + } + } return vols } @@ -375,12 +387,90 @@ func buildPodSpec(vdb *vapi.VerticaDB, sc *vapi.Subcluster, saName string) corev Tolerations: sc.Tolerations, ImagePullSecrets: GetK8sLocalObjectReferenceArray(vdb.Spec.ImagePullSecrets), Containers: makeContainers(vdb, sc), + InitContainers: makeInitContainers(vdb), Volumes: buildVolumes(vdb), TerminationGracePeriodSeconds: &termGracePeriod, ServiceAccountName: saName, } } +func makeInitContainers(vdb *vapi.VerticaDB) []corev1.Container { + var initContainers []corev1.Container + for _, ic := range vdb.Spec.Init { + initContainers = append(initContainers, corev1.Container{ + Image: ic.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + Name: ic.Name, + // Resources: sc.Resources, + Ports: []corev1.ContainerPort{ + // {ContainerPort: 5433, Name: "vertica"}, + // {ContainerPort: 5434, Name: "vertica-int"}, + {ContainerPort: 22, Name: "ssh"}, + }, + // ReadinessProbe: &corev1.Probe{ + // ProbeHandler: corev1.ProbeHandler{ + // Exec: &corev1.ExecAction{ + // Command: []string{"bash", "-c", buildReadinessProbeSQL(vdb)}, + // }, + // }, + // }, + // Is needed to run sshd on Openshift + // SecurityContext: &corev1.SecurityContext{ + // Capabilities: &corev1.Capabilities{ + // Add: []corev1.Capability{"SYS_CHROOT", "AUDIT_WRITE"}, + // }, + // }, + // Env: envVars, + VolumeMounts: buildInitContainerUDXVolumeMounts(vdb, &ic), + Command: []string{"/bin/bash"}, + Args: []string{"-c", ic.Command}, + }) + } + return initContainers +} + +func getInitContainerVolumeName(ic *vapi.InitDescriptor, idx int) string { + return ic.Name + "_volume_" + strconv.Itoa(idx) +} + +func buildUDXVolumeMounts(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor, where string) []corev1.VolumeMount { + var volumeMounts []corev1.VolumeMount + for idx, pathPair := range ic.Paths { + var mountPath string + if where == "initContainer" { + mountPath = pathPair.InitPath + } else if where == "mainContainer" { + mountPath = pathPair.ServerPath + } + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: getInitContainerVolumeName(ic, idx), + MountPath: mountPath, + }) + } + return volumeMounts +} + +func buildUDXVolumes(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor) []corev1.Volume { + var volumes []corev1.Volume + for idx := range ic.Paths { + volumes = append(volumes, corev1.Volume{ + Name: getInitContainerVolumeName(ic, idx), + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + } + return volumes +} + +func buildInitContainerUDXVolumeMounts(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor) []corev1.VolumeMount { + return buildUDXVolumeMounts(vdb, ic, "initContainer") +} + +func buildMainContainerUDXVolumeMounts(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor) []corev1.VolumeMount { + return buildUDXVolumeMounts(vdb, ic, "mainContainer") +} + // makeServerContainer builds the spec for the server container func makeServerContainer(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.Container { envVars := translateAnnotationsToEnvVars(vdb) From ac580d51cb8aa65e45c3f5b662d5c20ae1a915a4 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Thu, 7 Jul 2022 19:12:03 -0400 Subject: [PATCH 05/30] reverse the udx implementation --- pkg/builder/builder.go | 49 +++++++----------------------------------- 1 file changed, 8 insertions(+), 41 deletions(-) diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 032b0f190..3d2b68031 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -414,48 +414,15 @@ func makeServerContainer(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.Contai }, }, }, - SecurityContext: makeServerSecurityContext(vdb), - Env: envVars, - VolumeMounts: buildVolumeMounts(vdb), - } -} - -func makeServerSecurityContext(vdb *vapi.VerticaDB) *corev1.SecurityContext { - sc := &corev1.SecurityContext{} - if vdb.Spec.SecurityContext != nil { - sc = vdb.Spec.SecurityContext - } - if sc.Capabilities == nil { - sc.Capabilities = &corev1.Capabilities{} - } - capabilitiesNeeded := []corev1.Capability{ - // Needed to run sshd on OpenShift - "SYS_CHROOT", - // Needed to run sshd on OpenShift - "AUDIT_WRITE", - // Needed to be able to collect stacks via vstack - "SYS_PTRACE", - } - for i := range capabilitiesNeeded { - foundCap := false - for j := range sc.Capabilities.Add { - if capabilitiesNeeded[i] == sc.Capabilities.Add[j] { - foundCap = true - break - } - } - for j := range sc.Capabilities.Drop { - if capabilitiesNeeded[i] == sc.Capabilities.Drop[j] { - // If the capability we want to add is *dropped*, we won't bother adding it in - foundCap = false - break - } - } - if !foundCap { - sc.Capabilities.Add = append(sc.Capabilities.Add, capabilitiesNeeded[i]) - } + // Is needed to run sshd on Openshift + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_CHROOT", "AUDIT_WRITE"}, + }, + }, + Env: envVars, + VolumeMounts: buildVolumeMounts(vdb), } - return sc } // makeContainers creates the list of containers to include in the pod spec. From 6036bb0f9e11779660eb938c63ca82368ec2c991 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Thu, 7 Jul 2022 19:20:37 -0400 Subject: [PATCH 06/30] fix builder.go --- pkg/builder/builder.go | 51 +++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 3d2b68031..3babdd4f5 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -3,9 +3,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); You may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -414,15 +412,48 @@ func makeServerContainer(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.Contai }, }, }, - // Is needed to run sshd on Openshift - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"SYS_CHROOT", "AUDIT_WRITE"}, - }, - }, - Env: envVars, - VolumeMounts: buildVolumeMounts(vdb), + SecurityContext: makeServerSecurityContext(vdb), + Env: envVars, + VolumeMounts: buildVolumeMounts(vdb), + } +} + +func makeServerSecurityContext(vdb *vapi.VerticaDB) *corev1.SecurityContext { + sc := &corev1.SecurityContext{} + if vdb.Spec.SecurityContext != nil { + sc = vdb.Spec.SecurityContext + } + if sc.Capabilities == nil { + sc.Capabilities = &corev1.Capabilities{} + } + capabilitiesNeeded := []corev1.Capability{ + // Needed to run sshd on OpenShift + "SYS_CHROOT", + // Needed to run sshd on OpenShift + "AUDIT_WRITE", + // Needed to be able to collect stacks via vstack + "SYS_PTRACE", + } + for i := range capabilitiesNeeded { + foundCap := false + for j := range sc.Capabilities.Add { + if capabilitiesNeeded[i] == sc.Capabilities.Add[j] { + foundCap = true + break + } + } + for j := range sc.Capabilities.Drop { + if capabilitiesNeeded[i] == sc.Capabilities.Drop[j] { + // If the capability we want to add is *dropped*, we won't bother adding it in + foundCap = false + break + } + } + if !foundCap { + sc.Capabilities.Add = append(sc.Capabilities.Add, capabilitiesNeeded[i]) + } } + return sc } // makeContainers creates the list of containers to include in the pod spec. From 551a0b2c828285cc4d0cc05bcb367f3aff90ba5e Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Fri, 15 Jul 2022 12:57:37 -0400 Subject: [PATCH 07/30] implementation works and tested with single java udx init container --- Makefile | 26 +++++++--- api/v1beta1/verticadb_types.go | 47 ++++++++++++++--- .../samples/v1beta1_verticadb_new_design.yaml | 51 +++++++++++++++++++ docker-udx-java/Dockerfile | 17 +++++-- pkg/builder/builder.go | 46 +++++------------ 5 files changed, 135 insertions(+), 52 deletions(-) create mode 100644 config/samples/v1beta1_verticadb_new_design.yaml diff --git a/Makefile b/Makefile index 41cc630f4..275e72b00 100644 --- a/Makefile +++ b/Makefile @@ -84,9 +84,12 @@ export OPERATOR_IMG # Image URL to use for building/pushing of the vertica server VERTICA_IMG ?= $(IMG_REPO)vertica-k8s:$(TAG) export VERTICA_IMG - +# Image URL to use for building/pushing of the vertica server with java installed VERTICA_UDX_JAVA_IMG ?= $(IMG_REPO)vertica-udx-java-k8s:$(TAG) export VERTICA_UDX_JAVA_IMG +# Image URL to use for building/pushing of the java udx init container +UDX_JAVA_IMG ?= $(IMG_REPO)udx-java:$(TAG) +export UDX_JAVA_IMG # This is the base image to use for some upgrade tests. We will always # upgrade to VERTICA_IMG, so BASE_VERTICA_IMG must be some image from a @@ -336,7 +339,6 @@ docker-build-vertica: docker-vertica/Dockerfile ## Build vertica server docker i cd docker-vertica \ && make VERTICA_IMG=${VERTICA_IMG} MINIMAL_VERTICA_IMG=${MINIMAL_VERTICA_IMG} -.PHONY: docker-push docker-push-vertica: ## Push vertica server docker image ifeq ($(shell $(KIND_CHECK)), 0) docker push ${VERTICA_IMG} @@ -345,18 +347,28 @@ else endif .PHONY: docker-build-vertica-udx-java -docker-build-vertica-udx-java: docker-vertica-udx-java/Dockerfile ## Build vertica server docker image +docker-build-vertica-udx-java: docker-vertica-udx-java/Dockerfile ## Build vertica server docker image with jdk installed cd docker-vertica-udx-java \ && make VERTICA_IMG=${VERTICA_UDX_JAVA_IMG} MINIMAL_VERTICA_IMG=${MINIMAL_VERTICA_IMG} -.PHONY: docker-push -docker-push-vertica-udx-java: ## Push vertica server docker image +docker-push-vertica-udx-java: ## Push vertica server docker image with jdk installed ifeq ($(shell $(KIND_CHECK)), 0) docker push ${VERTICA_UDX_JAVA_IMG} else scripts/push-to-kind.sh -i ${VERTICA_UDX_JAVA_IMG} endif +.PHONY: docker-build-udx-java +docker-build-udx-java: docker-udx-java/Dockerfile ## Build docker image for java udx init container + docker build -t ${UDX_JAVA_IMG} -f docker-udx-java/Dockerfile docker-udx-java + +docker-push-udx-java: ## Push docker image for java udx init container +ifeq ($(shell $(KIND_CHECK)), 0) + docker push ${UDX_JAVA_IMG} +else + scripts/push-to-kind.sh -i ${UDX_JAVA_IMG} +endif + .PHONY: bundle bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. ifneq ($(DEPLOY_WITH), $(filter $(DEPLOY_WITH), olm random)) @@ -380,9 +392,9 @@ docker-build-olm-catalog: opm ## Build an OLM catalog that includes our bundle ( docker-push-olm-catalog: docker push $(OLM_CATALOG_IMG) -docker-build: docker-build-vertica docker-build-operator docker-build-vlogger docker-build-bundle ## Build all docker images except OLM catalog +docker-build: docker-build-vertica docker-build-operator docker-build-vlogger docker-build-udx-java docker-build-bundle ## Build all docker images except OLM catalog -docker-push: docker-push-vertica docker-push-operator docker-push-vlogger docker-push-bundle ## Push all docker images except OLM catalog +docker-push: docker-push-vertica docker-push-operator docker-push-vlogger docker-push-udx-java docker-push-bundle ## Push all docker images except OLM catalog echo-images: ## Print the names of all of the images used @echo "OPERATOR_IMG=$(OPERATOR_IMG)" diff --git a/api/v1beta1/verticadb_types.go b/api/v1beta1/verticadb_types.go index 6547be0e5..c456c2c2e 100644 --- a/api/v1beta1/verticadb_types.go +++ b/api/v1beta1/verticadb_types.go @@ -303,7 +303,16 @@ type VerticaDBSpec struct { // no effect. EncryptSpreadComm string `json:"encryptSpreadComm,omitempty"` - Init []InitDescriptor + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + // Descriptors of the init containers as well as corresponding volumes that + // will be used to prepare runtime environments for programs in the main + // container, e.g. UDx language runtimes. Multiple init containers may be + // specified and will be started in the same order as their descriptors. + // Multiple (emptyDir) volumes may be specified for each init container, + // and each volume may be mounted at different locations in the init + // container as well as the main container. + Init []InitDescriptor `json:"init,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:hidden" // Allows you to set any additional securityContext for the Vertica server @@ -314,16 +323,40 @@ type VerticaDBSpec struct { SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` } +// InitDescriptor is used to provide specifications for an init container +// which prepares runtime environments for programs inside the main container. type InitDescriptor struct { - Name string - Image string - Command string - Paths []InitPaths + // +kubebuilder:validation:Required + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + // The name of the init container, must be unique within the Pod. + Name string `json:"name,omitempty"` + // +kubebuilder:validation:Required + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + // The Docker image used for the init container. + Image string `json:"image,omitempty"` + // +kubebuilder:validation:Required + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + // The commands that will be executed by "/bin/sh" when the init + // container starts. + Command string `json:"command,omitempty"` + // +kubebuilder:validation:Required + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + // A list of pairs of paths that specify mount points of volumes attached + // to the init container. + Paths []InitPaths `json:"paths,omitempty"` } +// InitPaths defines a pair of paths which are mount points of a volume +// in an init container and the main container respectively. type InitPaths struct { - InitPath string - ServerPath string + // +kubebuilder:validation:Required + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + // The path inside the init container where a volume is mounted + InitPath string `json:"initPath,omitempty"` + // +kubebuilder:validation:Required + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + // The path inside the main container where a volume is mounted + ServerPath string `json:"serverPath,omitempty"` } // LocalObjectReference is used instead of corev1.LocalObjectReference and behaves the same. diff --git a/config/samples/v1beta1_verticadb_new_design.yaml b/config/samples/v1beta1_verticadb_new_design.yaml new file mode 100644 index 000000000..fdfd9f1e9 --- /dev/null +++ b/config/samples/v1beta1_verticadb_new_design.yaml @@ -0,0 +1,51 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1beta1 +kind: VerticaDB +metadata: + name: verticadb-sample +spec: + image: "vertica-k8s:1.6.0" + communal: + path: "s3://nimbusdb/db" + endpoint: "http://minio" + credentialSecret: s3-auth + includeUIDInPath: true + volumes: + - name: downloads + hostPath: + path: /host + volumeMounts: + - mountPath: /host + name: downloads + init: + - name: java + image: "udx-java:1.6.0" + # in the image: 2 part, 1: download jre; 2: FROM: alpine COPY from /usr/lib/jvm to /usr/lib/jvm + command: "time cp -r /tmp/java-se-8u41-ri/ /export/java-se-8u41-ri/" + paths: + - initPath: /export/java-se-8u41-ri/ + serverPath: /opt/jvm/ + # annotations: + # JAVA_TOOL_OPTIONS: "-Xms2G -Xmx2G" + subclusters: + - name: defaultsubcluster + size: 3 + # The CPU resource setting is here is a sample. We set it so that this + # will work with the sample VerticaAutoscaler resource. The actual amount + # should be sized according to: + # https://www.vertica.com/kb/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters/Content/Hardware/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters.htm + resources: + requests: + cpu: 500m diff --git a/docker-udx-java/Dockerfile b/docker-udx-java/Dockerfile index c7fc82df7..d68f33218 100644 --- a/docker-udx-java/Dockerfile +++ b/docker-udx-java/Dockerfile @@ -6,12 +6,19 @@ WORKDIR /tmp/ # ---- Option 1: download from web -# RUN set -x \ -# && apt install wget \ -# && wget https://download.java.net/openjdk/jdk8u41/ri/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz +RUN set -x \ +&& apt-get update \ +&& apt-get install wget \ +&& wget https://download.java.net/openjdk/jdk8u41/ri/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz \ +&& tar xzvf openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz # ---- Option 2: copy from local -COPY ./java-se-8u41-ri /tmp/ +# COPY ./java-se-8u41-ri /tmp/ -RUN apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file +############################################################################################## +FROM alpine:latest + +COPY --from=downloader /tmp/java-se-8u41-ri/ /tmp/java-se-8u41-ri/ + +CMD ["/bin/sh"] \ No newline at end of file diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index dcdbe60d5..cb293d94c 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -118,7 +118,7 @@ func buildVolumeMounts(vdb *vapi.VerticaDB) []corev1.VolumeMount { if len(vdb.Spec.Init) > 0 { for _, ic := range vdb.Spec.Init { - volMnts = append(volMnts, buildMainContainerUDXVolumeMounts(vdb, &ic)...) + volMnts = append(volMnts, buildMainContainerUDXVolumeMounts(ic)...) } } @@ -189,7 +189,7 @@ func buildVolumes(vdb *vapi.VerticaDB) []corev1.Volume { vols = append(vols, vdb.Spec.Volumes...) if len(vdb.Spec.Init) > 0 { for _, ic := range vdb.Spec.Init { - vols = append(vols, buildUDXVolumes(vdb, &ic)...) + vols = append(vols, buildUDXVolumes(ic)...) } } return vols @@ -399,39 +399,19 @@ func makeInitContainers(vdb *vapi.VerticaDB) []corev1.Container { Image: ic.Image, ImagePullPolicy: corev1.PullIfNotPresent, Name: ic.Name, - // Resources: sc.Resources, - Ports: []corev1.ContainerPort{ - // {ContainerPort: 5433, Name: "vertica"}, - // {ContainerPort: 5434, Name: "vertica-int"}, - {ContainerPort: 22, Name: "ssh"}, - }, - // ReadinessProbe: &corev1.Probe{ - // ProbeHandler: corev1.ProbeHandler{ - // Exec: &corev1.ExecAction{ - // Command: []string{"bash", "-c", buildReadinessProbeSQL(vdb)}, - // }, - // }, - // }, - // Is needed to run sshd on Openshift - // SecurityContext: &corev1.SecurityContext{ - // Capabilities: &corev1.Capabilities{ - // Add: []corev1.Capability{"SYS_CHROOT", "AUDIT_WRITE"}, - // }, - // }, - // Env: envVars, - VolumeMounts: buildInitContainerUDXVolumeMounts(vdb, &ic), - Command: []string{"/bin/bash"}, - Args: []string{"-c", ic.Command}, + VolumeMounts: buildInitContainerUDXVolumeMounts(ic), + Command: []string{"/bin/sh"}, + Args: []string{"-c", ic.Command}, }) } return initContainers } -func getInitContainerVolumeName(ic *vapi.InitDescriptor, idx int) string { - return ic.Name + "_volume_" + strconv.Itoa(idx) +func getInitContainerVolumeName(ic vapi.InitDescriptor, idx int) string { + return ic.Name + "-volume-" + strconv.Itoa(idx) } -func buildUDXVolumeMounts(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor, where string) []corev1.VolumeMount { +func buildUDXVolumeMounts(ic vapi.InitDescriptor, where string) []corev1.VolumeMount { var volumeMounts []corev1.VolumeMount for idx, pathPair := range ic.Paths { var mountPath string @@ -448,7 +428,7 @@ func buildUDXVolumeMounts(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor, where st return volumeMounts } -func buildUDXVolumes(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor) []corev1.Volume { +func buildUDXVolumes(ic vapi.InitDescriptor) []corev1.Volume { var volumes []corev1.Volume for idx := range ic.Paths { volumes = append(volumes, corev1.Volume{ @@ -461,12 +441,12 @@ func buildUDXVolumes(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor) []corev1.Volu return volumes } -func buildInitContainerUDXVolumeMounts(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor) []corev1.VolumeMount { - return buildUDXVolumeMounts(vdb, ic, "initContainer") +func buildInitContainerUDXVolumeMounts(ic vapi.InitDescriptor) []corev1.VolumeMount { + return buildUDXVolumeMounts(ic, "initContainer") } -func buildMainContainerUDXVolumeMounts(vdb *vapi.VerticaDB, ic *vapi.InitDescriptor) []corev1.VolumeMount { - return buildUDXVolumeMounts(vdb, ic, "mainContainer") +func buildMainContainerUDXVolumeMounts(ic vapi.InitDescriptor) []corev1.VolumeMount { + return buildUDXVolumeMounts(ic, "mainContainer") } // makeServerContainer builds the spec for the server container From 9062007d50016ec77094a2720a8e3a11a4a3f505 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Fri, 12 Aug 2022 14:57:37 -0400 Subject: [PATCH 08/30] for demo purpose, 2 flags (MINIMAL_VERTICA_IMG and USING_INIT) --- .gitignore | 7 +- Makefile | 15 -- .../samples/v1beta1_verticadb_host_mount.yaml | 42 +++++ .../v1beta1_verticadb_host_mount_java.yaml | 42 +++++ .../samples/v1beta1_verticadb_new_design.yaml | 5 +- docker-vertica-udx-java/.dockerignore | 1 - docker-vertica-udx-java/Dockerfile | 151 ---------------- docker-vertica-udx-java/Makefile | 21 --- docker-vertica-udx-java/README.md | 50 ------ docker-vertica-udx-java/dbadmin/.bash_profile | 14 -- .../dbadmin/.ssh/authorized_keys | 1 - docker-vertica-udx-java/dbadmin/.ssh/id_rsa | 38 ---- .../dbadmin/.ssh/id_rsa.pub | 1 - docker-vertica-udx-java/docker-entrypoint.sh | 37 ---- docker-vertica-udx-java/packages/README.md | 9 - docker-vertica-udx-java/packages/cleanup.sh | 62 ------- .../packages/init.d.functions | 95 ---------- .../packages/package-checksum-patcher.py | 165 ------------------ docker-vertica/Dockerfile | 33 +++- docker-vertica/Makefile | 2 + docker-vertica/dbadmin/.bash_profile | 1 + docker-vertica/docker-entrypoint.sh | 10 +- kuttl-test.yaml | 1 + scripts/setup-kustomize.sh | 4 +- tests/e2e-udx/udx-java/00-create-creds.yaml | 18 ++ tests/e2e-udx/udx-java/05-assert.yaml | 20 +++ .../e2e-udx/udx-java/05-deploy-operator.yaml | 17 ++ tests/e2e-udx/udx-java/15-assert.yaml | 27 +++ tests/e2e-udx/udx-java/15-setup-vdb.yaml | 17 ++ tests/e2e-udx/udx-java/17-assert.yaml | 51 ++++++ .../udx-java/17-wait-for-createdb.yaml | 1 + tests/e2e-udx/udx-java/20-assert.yaml | 22 +++ tests/e2e-udx/udx-java/20-rbac.yaml | 18 ++ tests/e2e-udx/udx-java/28-assert.yaml | 28 +++ .../e2e-udx/udx-java/28-verify-ip-family.yaml | 67 +++++++ tests/e2e-udx/udx-java/30-assert.yaml | 27 +++ .../udx-java/30-verify-vsql-access.yaml | 20 +++ tests/e2e-udx/udx-java/35-assert.yaml | 28 +++ .../udx-java/35-verify-memory-limit.yaml | 73 ++++++++ tests/e2e-udx/udx-java/40-assert.yaml | 23 +++ .../udx-java/40-verify-spread-encryption.yaml | 49 ++++++ tests/e2e-udx/udx-java/45-assert.yaml | 23 +++ tests/e2e-udx/udx-java/45-verify-vstack.yaml | 49 ++++++ tests/e2e-udx/udx-java/50-assert.yaml | 23 +++ .../50-verify-java-scalar-transform.yaml | 79 +++++++++ tests/e2e-udx/udx-java/51-assert.yaml | 23 +++ .../e2e-udx/udx-java/51-verify-java-uda.yaml | 59 +++++++ tests/e2e-udx/udx-java/52-assert.yaml | 23 +++ .../e2e-udx/udx-java/52-verify-java-udl.yaml | 60 +++++++ tests/e2e-udx/udx-java/60-errors.yaml | 18 ++ .../udx-java/60-uninstall-operator.yaml | 17 ++ tests/e2e-udx/udx-java/65-assert.yaml | 19 ++ tests/e2e-udx/udx-java/65-delete-crd.yaml | 22 +++ tests/e2e-udx/udx-java/65-errors.yaml | 24 +++ tests/e2e-udx/udx-java/99-delete-ns.yaml | 17 ++ tests/e2e-udx/udx-java/README.txt | 1 + .../setup-vdb/base/kustomization.yaml | 15 ++ .../udx-java/setup-vdb/base/setup-vdb.yaml | 38 ++++ .../base/kustomization.yaml | 15 ++ .../base/verify-vsql-access.yaml | 57 ++++++ 60 files changed, 1225 insertions(+), 670 deletions(-) create mode 100644 config/samples/v1beta1_verticadb_host_mount.yaml create mode 100644 config/samples/v1beta1_verticadb_host_mount_java.yaml delete mode 100644 docker-vertica-udx-java/.dockerignore delete mode 100644 docker-vertica-udx-java/Dockerfile delete mode 100644 docker-vertica-udx-java/Makefile delete mode 100644 docker-vertica-udx-java/README.md delete mode 100644 docker-vertica-udx-java/dbadmin/.bash_profile delete mode 100644 docker-vertica-udx-java/dbadmin/.ssh/authorized_keys delete mode 100644 docker-vertica-udx-java/dbadmin/.ssh/id_rsa delete mode 100644 docker-vertica-udx-java/dbadmin/.ssh/id_rsa.pub delete mode 100755 docker-vertica-udx-java/docker-entrypoint.sh delete mode 100644 docker-vertica-udx-java/packages/README.md delete mode 100644 docker-vertica-udx-java/packages/cleanup.sh delete mode 100644 docker-vertica-udx-java/packages/init.d.functions delete mode 100644 docker-vertica-udx-java/packages/package-checksum-patcher.py create mode 100644 tests/e2e-udx/udx-java/00-create-creds.yaml create mode 100644 tests/e2e-udx/udx-java/05-assert.yaml create mode 100644 tests/e2e-udx/udx-java/05-deploy-operator.yaml create mode 100644 tests/e2e-udx/udx-java/15-assert.yaml create mode 100644 tests/e2e-udx/udx-java/15-setup-vdb.yaml create mode 100644 tests/e2e-udx/udx-java/17-assert.yaml create mode 100644 tests/e2e-udx/udx-java/17-wait-for-createdb.yaml create mode 100644 tests/e2e-udx/udx-java/20-assert.yaml create mode 100644 tests/e2e-udx/udx-java/20-rbac.yaml create mode 100644 tests/e2e-udx/udx-java/28-assert.yaml create mode 100644 tests/e2e-udx/udx-java/28-verify-ip-family.yaml create mode 100644 tests/e2e-udx/udx-java/30-assert.yaml create mode 100644 tests/e2e-udx/udx-java/30-verify-vsql-access.yaml create mode 100644 tests/e2e-udx/udx-java/35-assert.yaml create mode 100644 tests/e2e-udx/udx-java/35-verify-memory-limit.yaml create mode 100644 tests/e2e-udx/udx-java/40-assert.yaml create mode 100644 tests/e2e-udx/udx-java/40-verify-spread-encryption.yaml create mode 100644 tests/e2e-udx/udx-java/45-assert.yaml create mode 100644 tests/e2e-udx/udx-java/45-verify-vstack.yaml create mode 100644 tests/e2e-udx/udx-java/50-assert.yaml create mode 100644 tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml create mode 100644 tests/e2e-udx/udx-java/51-assert.yaml create mode 100644 tests/e2e-udx/udx-java/51-verify-java-uda.yaml create mode 100644 tests/e2e-udx/udx-java/52-assert.yaml create mode 100644 tests/e2e-udx/udx-java/52-verify-java-udl.yaml create mode 100644 tests/e2e-udx/udx-java/60-errors.yaml create mode 100644 tests/e2e-udx/udx-java/60-uninstall-operator.yaml create mode 100644 tests/e2e-udx/udx-java/65-assert.yaml create mode 100644 tests/e2e-udx/udx-java/65-delete-crd.yaml create mode 100644 tests/e2e-udx/udx-java/65-errors.yaml create mode 100644 tests/e2e-udx/udx-java/99-delete-ns.yaml create mode 100644 tests/e2e-udx/udx-java/README.txt create mode 100644 tests/e2e-udx/udx-java/setup-vdb/base/kustomization.yaml create mode 100644 tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml create mode 100644 tests/e2e-udx/udx-java/verify-vsql-access/base/kustomization.yaml create mode 100644 tests/e2e-udx/udx-java/verify-vsql-access/base/verify-vsql-access.yaml diff --git a/.gitignore b/.gitignore index ec5dccecd..2fb791d12 100644 --- a/.gitignore +++ b/.gitignore @@ -71,4 +71,9 @@ bundle/ docker-udx-java/java-se-8u41-ri -docker-udx-java/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz \ No newline at end of file +docker-udx-java/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz + +docker-vertica-udx-java + +tests/e2e-udx/*/UDx-Examples +tests/e2e-udx/udx-java/examples \ No newline at end of file diff --git a/Makefile b/Makefile index 275e72b00..d850d4064 100644 --- a/Makefile +++ b/Makefile @@ -84,9 +84,6 @@ export OPERATOR_IMG # Image URL to use for building/pushing of the vertica server VERTICA_IMG ?= $(IMG_REPO)vertica-k8s:$(TAG) export VERTICA_IMG -# Image URL to use for building/pushing of the vertica server with java installed -VERTICA_UDX_JAVA_IMG ?= $(IMG_REPO)vertica-udx-java-k8s:$(TAG) -export VERTICA_UDX_JAVA_IMG # Image URL to use for building/pushing of the java udx init container UDX_JAVA_IMG ?= $(IMG_REPO)udx-java:$(TAG) export UDX_JAVA_IMG @@ -346,18 +343,6 @@ else scripts/push-to-kind.sh -i ${VERTICA_IMG} endif -.PHONY: docker-build-vertica-udx-java -docker-build-vertica-udx-java: docker-vertica-udx-java/Dockerfile ## Build vertica server docker image with jdk installed - cd docker-vertica-udx-java \ - && make VERTICA_IMG=${VERTICA_UDX_JAVA_IMG} MINIMAL_VERTICA_IMG=${MINIMAL_VERTICA_IMG} - -docker-push-vertica-udx-java: ## Push vertica server docker image with jdk installed -ifeq ($(shell $(KIND_CHECK)), 0) - docker push ${VERTICA_UDX_JAVA_IMG} -else - scripts/push-to-kind.sh -i ${VERTICA_UDX_JAVA_IMG} -endif - .PHONY: docker-build-udx-java docker-build-udx-java: docker-udx-java/Dockerfile ## Build docker image for java udx init container docker build -t ${UDX_JAVA_IMG} -f docker-udx-java/Dockerfile docker-udx-java diff --git a/config/samples/v1beta1_verticadb_host_mount.yaml b/config/samples/v1beta1_verticadb_host_mount.yaml new file mode 100644 index 000000000..94ec08ac4 --- /dev/null +++ b/config/samples/v1beta1_verticadb_host_mount.yaml @@ -0,0 +1,42 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1beta1 +kind: VerticaDB +metadata: + name: verticadb-sample +spec: + image: "vertica-k8s:1.6.0" + communal: + path: "s3://nimbusdb/db" + endpoint: "http://minio" + credentialSecret: s3-auth + includeUIDInPath: true + volumes: + - name: downloads + hostPath: + path: /host + volumeMounts: + - mountPath: /host + name: downloads + kSafety: "0" + subclusters: + - name: defaultsubcluster + size: 1 + # The CPU resource setting is here is a sample. We set it so that this + # will work with the sample VerticaAutoscaler resource. The actual amount + # should be sized according to: + # https://www.vertica.com/kb/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters/Content/Hardware/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters.htm + resources: + requests: + cpu: 500m diff --git a/config/samples/v1beta1_verticadb_host_mount_java.yaml b/config/samples/v1beta1_verticadb_host_mount_java.yaml new file mode 100644 index 000000000..514a4bae4 --- /dev/null +++ b/config/samples/v1beta1_verticadb_host_mount_java.yaml @@ -0,0 +1,42 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1beta1 +kind: VerticaDB +metadata: + name: verticadb-sample +spec: + image: "vertica-k8s-fat:1.6.0" + communal: + path: "s3://nimbusdb/db" + endpoint: "http://minio" + credentialSecret: s3-auth + includeUIDInPath: true + volumes: + - name: downloads + hostPath: + path: /host + volumeMounts: + - mountPath: /host + name: downloads + kSafety: "0" + subclusters: + - name: defaultsubcluster + size: 1 + # The CPU resource setting is here is a sample. We set it so that this + # will work with the sample VerticaAutoscaler resource. The actual amount + # should be sized according to: + # https://www.vertica.com/kb/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters/Content/Hardware/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters.htm + resources: + requests: + cpu: 500m diff --git a/config/samples/v1beta1_verticadb_new_design.yaml b/config/samples/v1beta1_verticadb_new_design.yaml index fdfd9f1e9..e5d3acee2 100644 --- a/config/samples/v1beta1_verticadb_new_design.yaml +++ b/config/samples/v1beta1_verticadb_new_design.yaml @@ -32,16 +32,17 @@ spec: init: - name: java image: "udx-java:1.6.0" - # in the image: 2 part, 1: download jre; 2: FROM: alpine COPY from /usr/lib/jvm to /usr/lib/jvm + # in the image: 2 part, 1: download jre; 2: FROM: copy jvm folder from first stage to second stage command: "time cp -r /tmp/java-se-8u41-ri/ /export/java-se-8u41-ri/" paths: - initPath: /export/java-se-8u41-ri/ serverPath: /opt/jvm/ # annotations: # JAVA_TOOL_OPTIONS: "-Xms2G -Xmx2G" + kSafety: "0" subclusters: - name: defaultsubcluster - size: 3 + size: 1 # The CPU resource setting is here is a sample. We set it so that this # will work with the sample VerticaAutoscaler resource. The actual amount # should be sized according to: diff --git a/docker-vertica-udx-java/.dockerignore b/docker-vertica-udx-java/.dockerignore deleted file mode 100644 index b43bf86b5..000000000 --- a/docker-vertica-udx-java/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -README.md diff --git a/docker-vertica-udx-java/Dockerfile b/docker-vertica-udx-java/Dockerfile deleted file mode 100644 index 2918c4602..000000000 --- a/docker-vertica-udx-java/Dockerfile +++ /dev/null @@ -1,151 +0,0 @@ -# Apache 2.0 license -# -# Copyright (c) 2021 Vertica -# - -ARG BASE_OS_VERSION="focal-20220316" -ARG BUILDER_OS_VERSION="7.9.2009" -FROM centos:centos${BUILDER_OS_VERSION} as builder - -ARG VERTICA_RPM="vertica-x86_64.RHEL6.latest.rpm" -ARG MINIMAL="" -ARG DBADMIN_GID=5000 -ARG DBADMIN_UID=5000 - -COPY ./packages/${VERTICA_RPM} /tmp/ -# this is a script which removes unnecessary stuff from the -# container image -COPY ./packages/cleanup.sh /tmp/ -COPY ./packages/package-checksum-patcher.py /tmp/ - -RUN set -x \ - # Update is needed to be confident that we're picking up - # fixed libraries. - && yum -q -y update \ - && yum install -y \ - cronie \ - dialog \ - iproute \ - mcelog \ - openssh-server \ - openssh-clients \ - openssl \ - which \ - zlib-devel \ - && /usr/sbin/groupadd -r verticadba --gid ${DBADMIN_GID} \ - && /usr/sbin/useradd -r -m -s /bin/bash -g verticadba --uid ${DBADMIN_UID} dbadmin \ - && yum localinstall -q -y /tmp/${VERTICA_RPM} \ - # Run install_vertica script to prepare environment - && /opt/vertica/sbin/install_vertica \ - --accept-eula \ - --debug \ - --dba-user-password-disabled \ - --failure-threshold NONE \ - --license CE \ - --hosts 127.0.0.1 \ - --no-system-configuration \ - -U \ - --data-dir /home/dbadmin \ - && mkdir -p /home/dbadmin/licensing/ce \ - && cp -r /opt/vertica/config/licensing/* /home/dbadmin/licensing/ce/ \ - && mkdir -p /home/dbadmin/logrotate \ - && cp -r /opt/vertica/config/logrotate /home/dbadmin/logrotate/ \ - && cp /opt/vertica/config/logrotate_base.conf /home/dbadmin/logrotate/ \ - && chown -R dbadmin:verticadba /opt/vertica \ - # reduce the size of the final image - && rm -rf /opt/vertica/lib64 \ - && yum clean all \ - && sh /tmp/cleanup.sh - -COPY dbadmin/.bash_profile /home/dbadmin/ -# Copy in a stable ssh key. This is done so that Vertica pods can communicate -# with pods running an older image. This is necessary when doing an online -# image change as the Vertica cluster will be running with two container -# versions at once. -COPY dbadmin/.ssh /home/dbadmin/.ssh -# prepare (put inside) entrypoint script and make right permissions -COPY ./docker-entrypoint.sh /usr/local/bin - -RUN set -x \ - && chmod a+x /usr/local/bin/docker-entrypoint.sh \ - && chown dbadmin:verticadba /home/dbadmin/.bash_profile \ - && chmod 600 /home/dbadmin/.bash_profile \ - && mkdir -p /root/.ssh \ - && cp -r /home/dbadmin/.ssh /root \ - && chmod 700 /root/.ssh \ - && chmod 600 /root/.ssh/* \ - && chmod 700 /home/dbadmin/.ssh \ - && chmod 600 /home/dbadmin/.ssh/* \ - && chown -R dbadmin:verticadba /home/dbadmin/ \ - # On older docker versions, the ownership of /opt/vertica is not preserved in - # the COPY. Working around this by changing it so that all files and - # directories are world writeable. - && chmod 777 -R /opt/vertica - -############################################################################################## -FROM ubuntu:${BASE_OS_VERSION} - -ARG DBADMIN_GID=5000 -ARG DBADMIN_UID=5000 - -COPY --from=builder /opt/vertica /opt/vertica -COPY --from=builder /home/dbadmin /home/dbadmin -COPY --from=builder /root/.ssh /root/.ssh -COPY --from=builder /var/spool/cron/ /var/spool/cron/crontabs -COPY --from=builder /usr/local/bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh - -ENV LANG en_US.UTF-8 -ENV TZ UTC -ENV PATH "$PATH:/opt/vertica/bin:/opt/vertica/sbin" -ENV DEBIAN_FRONTEND noninteractive - -COPY ./packages/init.d.functions /etc/rc.d/init.d/functions - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN set -x \ - # COPY may not preserve file permissions on older docker versions. This is a - # workaround for that. This will cause the image to increase a bit, but there - # isn't much we copy over for this directory. - && chown -R $DBADMIN_UID:$DBADMIN_GID /home/dbadmin \ - # Update is needed to be confident that we're picking up - # fixed libraries. - && apt-get -y update \ - && apt-get install -y --no-install-recommends \ - locales \ - cron \ - sysstat \ - ca-certificates \ - ntp \ - dialog \ - iproute2 \ - libkeyutils1\ - libz-dev \ - procps \ - krb5-user \ - logrotate \ - openssh-client \ - openssh-server \ - openssl \ - sudo \ - openjdk-8-jre \ - && rm -rf /var/lib/apt/lists/* \ - # Make the "en_US.UTF-8" locale so vertica will be utf-8 enabled by default - && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ - && mkdir -p /run/sshd \ - && ssh-keygen -q -A \ - && /usr/sbin/groupadd -r verticadba --gid ${DBADMIN_GID} \ - && /usr/sbin/useradd -r -m -s /bin/bash -g verticadba --uid ${DBADMIN_UID} dbadmin \ - # Allow passwordless sudo access from dbadmin - && echo "dbadmin ALL=(ALL) NOPASSWD: ALL" | tee -a /etc/sudoers \ - && echo "dbadmin - nofile 65536" >> /etc/security/limits.conf - -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -# vertica port -EXPOSE 5433 -# agent port -EXPOSE 5444 -USER dbadmin -LABEL os-family="ubuntu" -LABEL image-name="vertica_k8s" -LABEL maintainer="K8s Team" diff --git a/docker-vertica-udx-java/Makefile b/docker-vertica-udx-java/Makefile deleted file mode 100644 index b49f99747..000000000 --- a/docker-vertica-udx-java/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -VERTICA_RPM?=vertica-x86_64.RHEL6.latest.rpm -BUILDER_OS_VERSION?=7.9.2009 -BASE_OS_VERSION?=focal-20220316 -VERTICA_IMG?=vertica-udx-java-k8s -MINIMAL_VERTICA_IMG?= -VERTICA_VERSION?=$(shell rpm --nosignature -qp --queryformat '%{VERSION}-%{RELEASE}' packages/$(VERTICA_RPM)) - -all: docker-build-vertica - -.PHONY: docker-build-vertica -docker-build-vertica: Dockerfile packages/package-checksum-patcher.py - docker build \ - -f Dockerfile \ - --label minimal=${MINIMAL_VERTICA_IMG} \ - --label os-version=${BASE_OS_VERSION} \ - --label vertica-version=${VERTICA_VERSION} \ - --build-arg MINIMAL=${MINIMAL_VERTICA_IMG} \ - --build-arg VERTICA_RPM=${VERTICA_RPM} \ - --build-arg BASE_OS_VERSION=${BASE_OS_VERSION} \ - --build-arg BUILDER_OS_VERSION=${BUILDER_OS_VERSION} \ - -t ${VERTICA_IMG} . diff --git a/docker-vertica-udx-java/README.md b/docker-vertica-udx-java/README.md deleted file mode 100644 index badd28c94..000000000 --- a/docker-vertica-udx-java/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# About - -* Maintained by: The [vertica-kubernetes community](https://github.com/vertica/vertica-kubernetes) -* Docker Community: [Docker Forums](https://forums.docker.com/), [Stack Overflow](https://stackoverflow.com/questions/tagged/docker) - -# Supported Tags -* [12.0.0-0-minimal, latest](https://github.com/vertica/vertica-kubernetes/blob/v1.6.0/docker-vertica/Dockerfile) -* [12.0.0-0](https://github.com/vertica/vertica-kubernetes/blob/v1.6.0/docker-vertica/Dockerfile) -* [11.1.1-2-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.5.0/docker-vertica/Dockerfile) -* [11.1.1-2](https://github.com/vertica/vertica-kubernetes/blob/v1.5.0/docker-vertica/Dockerfile) -* [11.1.1-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.4.0/docker-vertica/Dockerfile) -* [11.1.1-0](https://github.com/vertica/vertica-kubernetes/blob/v1.4.0/docker-vertica/Dockerfile) -* [11.1.0-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.3.0/docker-vertica/Dockerfile) -* [11.1.0-0](https://github.com/vertica/vertica-kubernetes/blob/v1.3.0/docker-vertica/Dockerfile) -* [11.0.2-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.2.0/docker-vertica/Dockerfile) -* [11.0.2-0](https://github.com/vertica/vertica-kubernetes/blob/v1.2.0/docker-vertica/Dockerfile) -* [11.0.1-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.1.0/docker-vertica/Dockerfile) -* [11.0.1-0](https://github.com/vertica/vertica-kubernetes/blob/v1.1.0/docker-vertica/Dockerfile) -* [11.0.0-0-minimal](https://github.com/vertica/vertica-kubernetes/blob/v1.0.0/docker-vertica/Dockerfile) -* [11.0.0-0](https://github.com/vertica/vertica-kubernetes/blob/v1.0.0/docker-vertica/Dockerfile) -* [10.1.1-0](https://github.com/vertica/vertica-kubernetes/blob/v0.1.0/docker-vertica/Dockerfile) - -# Quick Reference - -* [Vertica-Kubernetes GitHub repository](https://github.com/vertica/vertica-kubernetes) -* [Vertica Helm chart repository](https://github.com/vertica/charts) -* [Official Vertica Documentation](https://www.vertica.com/docs/latest/HTML/Content/Home.htm) -* Supported architectures: `amd64` - -# What is Vertica? - -Vertica is a unified analytics platform, based on a massively scalable architecture with the broadest set of analytical functions spanning event and time series, pattern matching, geospatial and end-to-end in-database machine learning. Vertica enables you to easily apply these powerful functions to the largest and most demanding analytical workloads, arming you and your customers with predictive business insights faster than any analytics data warehouse in the market. Vertica provides a unified analytics platform across major public clouds and on-premises data centers and integrates data in cloud object storage and HDFS without forcing you to move any of your data. - -https://www.vertica.com/ - -![](https://raw.githubusercontent.com/vertica/vertica-kubernetes/main/vertica-logo.png) - -# How to Use This Image - -This image runs the Vertica server that is optimized for use with the [Vertica operator](https://github.com/vertica/vertica-kubernetes/tree/main/docker-operator). The operator automates management and administrative tasks for an [Eon Mode](https://www.vertica.com/docs/latest/HTML/Content/Authoring/Eon/Architecture.htm) database in Kubernetes. - -Vertica provides two versions of this image, depending on whether you require the TensorFlow package: -- If you do not require the TensorFlow package, Vertica recommends downloading the image with the *-minimal* supported tag suffix. Because it does not include the TensorFlow package, it has a reduced size. The *-minimal* image is the default image included in the [Vertica Helm chart](https://github.com/vertica/charts). -- If you do require the TensorFlow package, download an image without the *-minimal* suffix. For details about TensorFlow and Vertica, see [Setting up TensorFlow Support in Vertica](https://www.vertica.com/docs/latest/HTML/Content/Authoring/AnalyzingData/MachineLearning/UsingExternalModels/UsingTensorFlow/TensorFlowExample.htm). - -For a brief overview on how to install and configure the operator, see the [Vertica GitHub repository](https://github.com/vertica/vertica-kubernetes). For an in-depth look at Vertica on Kubernetes, see the [Vertica documentation](https://www.vertica.com/docs/latest/HTML/Content/Authoring/Containers/ContainerizedVertica.htm). - -# License - -View the [license information](https://www.vertica.com/end-user-license-agreement-ce-version/) for this image. diff --git a/docker-vertica-udx-java/dbadmin/.bash_profile b/docker-vertica-udx-java/dbadmin/.bash_profile deleted file mode 100644 index 13f594aef..000000000 --- a/docker-vertica-udx-java/dbadmin/.bash_profile +++ /dev/null @@ -1,14 +0,0 @@ -# .bash_profile - -# Get the aliases and functions -if [ -f ~/.bashrc ]; then - . ~/.bashrc -fi - -# User specific environment and startup programs - -PATH=$PATH:$HOME/.local/bin:$HOME/bin:/opt/vertica/bin -export PATH - -export LANG=en_US.UTF-8 -export TZ=UTC diff --git a/docker-vertica-udx-java/dbadmin/.ssh/authorized_keys b/docker-vertica-udx-java/dbadmin/.ssh/authorized_keys deleted file mode 100644 index 41346470c..000000000 --- a/docker-vertica-udx-java/dbadmin/.ssh/authorized_keys +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2ExhX20Jeun+sp3aShRcUlHIaBdnUlCkqwf7h3vBVEV7KG5CUgabS7cYT/Qv2LQIfUsbLGq3EmVkihoy011vvQooFEV5qg5XOFESsCjp0xL7uYw8cB1YihqDsP31boep0tWs28QL2JVGn0snhgRwo/Nxa01n6t87+9CULRDwsKqnp1iDLPfbrEf3jHAU9niEDmoPMNfR+NhbYO2XsEGKMqr3Fx7bCtP0vVSiijLpvlp2QC5CfjK6oGpakqruk26LRiEe1drzz5fxn7ndh5kIwjUoOYfMTbt9/TGKTT8KTj7oVaH5oaJ4Reh9ymWzvQPCLci/XAbERy7h5bXWG05px4fxTT5ILX9dnT21XH+kSM9fdnqWbBj1EexCGraezMq9EaNCeMHzXUxt3iDhVlLrN2SkEliBDdEEilucjkPUqcJtzdJAzyKBOHQGWkCCercNiLCgrltXlhxXqQp0kIxx+pjyeHGxqoUSr0gnSsbbaM2K/kMZ8uDjyqqoHlFAsrgM= spilchen@sputnik diff --git a/docker-vertica-udx-java/dbadmin/.ssh/id_rsa b/docker-vertica-udx-java/dbadmin/.ssh/id_rsa deleted file mode 100644 index 13f47a2c9..000000000 --- a/docker-vertica-udx-java/dbadmin/.ssh/id_rsa +++ /dev/null @@ -1,38 +0,0 @@ ------BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn -NhAAAAAwEAAQAAAYEAthMYV9tCXrp/rKd2koUXFJRyGgXZ1JQpKsH+4d7wVRFeyhuQlIGm -0u3GE/0L9i0CH1LGyxqtxJlZIoaMtNdb70KKBRFeaoOVzhRErAo6dMS+7mMPHAdWIoag7D -99W6HqdLVrNvEC9iVRp9LJ4YEcKPzcWtNZ+rfO/vQlC0Q8LCqp6dYgyz326xH94xwFPZ4h -A5qDzDX0fjYW2Dtl7BBijKq9xce2wrT9L1Uoooy6b5adkAuQn4yuqBqWpKq7pNui0YhHtX -a88+X8Z+53YeZCMI1KDmHzE27ff0xik0/Ck4+6FWh+aGieEXofcpls70Dwi3Iv1wGxEcu4 -eW11htOaceH8U0+SC1/XZ09tVx/pEjPX3Z6lmwY9RHsQhq2nszKvRGjQnjB811Mbd4g4VZ -S6zdkpBJYgQ3RBIpbnI5D1KnCbc3SQM8igTh0BlpAgnq3DYiwoK5bV5YcV6kKdJCMcfqY8 -nhxsaqFEq9IJ0rG22jNiv5DGfLg48qqqB5RQLK4DAAAFiKo3XLCqN1ywAAAAB3NzaC1yc2 -EAAAGBALYTGFfbQl66f6yndpKFFxSUchoF2dSUKSrB/uHe8FURXsobkJSBptLtxhP9C/Yt -Ah9SxssarcSZWSKGjLTXW+9CigURXmqDlc4URKwKOnTEvu5jDxwHViKGoOw/fVuh6nS1az -bxAvYlUafSyeGBHCj83FrTWfq3zv70JQtEPCwqqenWIMs99usR/eMcBT2eIQOag8w19H42 -Ftg7ZewQYoyqvcXHtsK0/S9VKKKMum+WnZALkJ+MrqgalqSqu6TbotGIR7V2vPPl/Gfud2 -HmQjCNSg5h8xNu339MYpNPwpOPuhVofmhonhF6H3KZbO9A8ItyL9cBsRHLuHltdYbTmnHh -/FNPkgtf12dPbVcf6RIz192epZsGPUR7EIatp7Myr0Ro0J4wfNdTG3eIOFWUus3ZKQSWIE -N0QSKW5yOQ9Spwm3N0kDPIoE4dAZaQIJ6tw2IsKCuW1eWHFepCnSQjHH6mPJ4cbGqhRKvS -CdKxttozYr+Qxny4OPKqqgeUUCyuAwAAAAMBAAEAAAGAMz20gCVIzHgYLnMCv8GM+VG9uu -0inN0FqO7n2JFxeb9lueOAEQtb88jPEhkJIMMR8dbN86MSfF3b5AeR2eMyEGfdiAeXVsqp -glHZ8cD/KpWkVlqyieTWrOjOqS5FDWDIJwhXoCK7/qIKFP0Rq9E2J6LiMP2COTk5nC8Bre -YKOTPYleKZYBmUQBvUI/Ji1zCFggoeRX55zHtUeo5c5M7HrWbv+ZV3LdzZ1FRXF0gbwAPX -XA6lKQHitZSmCLm0DXVyJ5awSrNTRcwrIy1vGVES16pUvJSXioWP9SOuAUvYuuzzrqxlZ6 -E8hImC8odVzxV452KoXqrxDmEsZGPqqVPZ9WE3LJLkhmVfLwCzM2penT6xJ8hig8SOhUHo -pWYuWaTP+d1XAPntAnDZLPkneobnEhRgWgBj4JxS2VtoJH2uI0hhH7q36zsSZ+e8KKYzdS -zAmyEWnBsKNYbP0a658/BsMkJNqstWh1wwVZ/U3rkuJtHYgSBXYrtmzZzcOXYzRdjhAAAA -wArbQfr2ItNim9Kw4Y1Cu5Eit6uGwNxj6X0OWtBAHSIaRV5CQq5mCMDkE3D+8tLzxVyCcw -/W9laTHc0ge5cKeYVR4UNaW1LK3vAXx4bXcrEmWyI/QvqSafwIvvkzQXazcCmjV3cnDLG0 -e5R6iqT5eNG6hCwqZ5kypmpOuxyi9xIUDfsSFiFX1VwPCrTCmA7zSOaf6pv17PlqfiKFGl -b0tGVV4ySvQNKa61YSL6qszaMkP3U2SgTRAVkk+m4Dm+SLQwAAAMEA2mHPBC/JVPzxX+ua -PdFBOKigisyoYaxdMbx6Do/ev8Tle5Y1nakBs1EQr2G7QdeWA9jV+Yw6HRsAyEt5x4mVPq -bZJBmbthLI/bxQVyQP4RauXgAVlVnURpusJP9joEfw8urL7iXCc5dYdVzHRrO6zWmEucAS -umWjxdAna6rYYidZzKCXp+rT2ztKaM1zEHB4CrS+Qb/F7AZvGIBWRO6CJG8Qxw1zQApC5Z -6IOKS5lhzgtC2iWIxNfUXdXxxNLigvAAAAwQDVcDPwr4XX4Yj1WUalKYszj28tnuUbLQQu -T0uppO9WPVSweOWiSy8/X9zLZxpaebtmLrOfHr9HfouIRCQXslnn2NxC5DjwSO0KP9vt+U -Qu4AB9p6LJHVxAqBGALVpzXaVokfAOzPaDE3zct58y/S6DagAk2b8gyfHW8XpgQ5IU3jWa -Oi/fRUObjDb6plyPvExJgfAyxQq/eHi7PPrULf6HGJTBq2icpQCHEsVMy3bafXNKY9sXMm -TrxAuVuaAdDm0AAAAQc3BpbGNoZW5Ac3B1dG5pawECAw== ------END OPENSSH PRIVATE KEY----- diff --git a/docker-vertica-udx-java/dbadmin/.ssh/id_rsa.pub b/docker-vertica-udx-java/dbadmin/.ssh/id_rsa.pub deleted file mode 100644 index 41346470c..000000000 --- a/docker-vertica-udx-java/dbadmin/.ssh/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2ExhX20Jeun+sp3aShRcUlHIaBdnUlCkqwf7h3vBVEV7KG5CUgabS7cYT/Qv2LQIfUsbLGq3EmVkihoy011vvQooFEV5qg5XOFESsCjp0xL7uYw8cB1YihqDsP31boep0tWs28QL2JVGn0snhgRwo/Nxa01n6t87+9CULRDwsKqnp1iDLPfbrEf3jHAU9niEDmoPMNfR+NhbYO2XsEGKMqr3Fx7bCtP0vVSiijLpvlp2QC5CfjK6oGpakqruk26LRiEe1drzz5fxn7ndh5kIwjUoOYfMTbt9/TGKTT8KTj7oVaH5oaJ4Reh9ymWzvQPCLci/XAbERy7h5bXWG05px4fxTT5ILX9dnT21XH+kSM9fdnqWbBj1EexCGraezMq9EaNCeMHzXUxt3iDhVlLrN2SkEliBDdEEilucjkPUqcJtzdJAzyKBOHQGWkCCercNiLCgrltXlhxXqQp0kIxx+pjyeHGxqoUSr0gnSsbbaM2K/kMZ8uDjyqqoHlFAsrgM= spilchen@sputnik diff --git a/docker-vertica-udx-java/docker-entrypoint.sh b/docker-vertica-udx-java/docker-entrypoint.sh deleted file mode 100755 index 71627e1e5..000000000 --- a/docker-vertica-udx-java/docker-entrypoint.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -set -e - -start_cron(){ - # daemonizes, no need for & - sudo /usr/sbin/cron -} - -# We copy back the files normally stored in /opt/vertica/config/. We do this -# because we have a Persistent Volume that backs /opt/vertica/config, so -# it starts up empty and must be populated -copy_config_files() { - mkdir -p /opt/vertica/config/licensing - - mv /home/dbadmin/logrotate/* /opt/vertica/config/ 2>/dev/null || true - - cp -r /home/dbadmin/licensing/ce/* /opt/vertica/config/licensing 2>/dev/null || true - chmod -R ugo+r,u+rw /opt/vertica/config/licensing -} - -# Ensure all PV paths are owned by dbadmin. This is done for some PVs that -# start with restrictive ownership. -ensure_path_is_owned_by_dbadmin() { - # -z is to needed in case input arg is empty - [ -z "$1" ] || [ "$(stat -c "%U" "$1")" == "dbadmin" ] || sudo chown -R dbadmin:verticadba "$1" -} - -start_cron -ensure_path_is_owned_by_dbadmin /opt/vertica/config -ensure_path_is_owned_by_dbadmin /opt/vertica/log -ensure_path_is_owned_by_dbadmin $DATA_PATH -ensure_path_is_owned_by_dbadmin $DEPOT_PATH -copy_config_files - -echo "Vertica container is now running" - -sudo /usr/sbin/sshd -D diff --git a/docker-vertica-udx-java/packages/README.md b/docker-vertica-udx-java/packages/README.md deleted file mode 100644 index 6d129f57c..000000000 --- a/docker-vertica-udx-java/packages/README.md +++ /dev/null @@ -1,9 +0,0 @@ -You have to download Vertica RPM package before building docker image (even if you are -running on Ubuntu or Debian, the contents of the container use a CentOS environment). - -Download either [Community Edition](https://www.vertica.com/try/)(registration required) -or you can download Enterprise edition, if you are Vertica customer. - -Store RPM packages into this folder, and remember to tell make the -name of the RPM when you build the image. - diff --git a/docker-vertica-udx-java/packages/cleanup.sh b/docker-vertica-udx-java/packages/cleanup.sh deleted file mode 100644 index 1bb57669a..000000000 --- a/docker-vertica-udx-java/packages/cleanup.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/sh - -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Remove things not necessary for a non-interactive Kubernetes pod -# running Vertica - -# This script is hand-crafted: -# Build an image named bigimage -# -# docker run -it --entrypoint /bin/bash bigimage -# -# wander around in the image looking for things you can remove -rm -r -f \ - /opt/vertica/examples \ - /opt/vertica/sdk \ - /opt/vertica/packages/*/examples \ - /opt/vertica/oss/python*/lib/python*/test \ - /opt/vertica/oss/python*/lib/python*/unittest/test \ - /opt/vertica/oss/python*/lib/python*/pip \ - /opt/vertica/oss/python*/lib/python*/site-packages/pip \ - /opt/vertica/oss/python*/lib/python*/config-[0-9]* \ - /opt/vertica/oss/python*/lib/python*/tkinter \ - /opt/vertica/oss/python*/lib/python*/idlelib - -# cleanup many of the __pycache__ directories -find /opt/vertica/oss/ -type d -name "__pycache__" -exec rm -rf {} + - -# many of these directories contain things that aren't binaries -# thus divert error output to /dev/null -strip /opt/vertica/bin/* 2> /dev/null -strip /opt/vertica/lib/*.so* -strip /opt/vertica/oss/python*/bin/* 2> /dev/null -strip /opt/vertica/oss/python*/lib/libpython*.a -strip /opt/vertica/oss/python*/lib/python*/lib-dynload/*.so* - -# stripping the packages directory saves about 900MB, but... -strip /opt/vertica/packages/*/lib/*.so* 2> /dev/null -# it changes the checksums used to verify the libraries when loaded -/opt/vertica/oss/python*/bin/python[0-9] \ - /tmp/package-checksum-patcher.py /opt/vertica/packages/* - -# (optional) minimal images remove packages that aren't auto installed -if [ "$MINIMAL" = "YES" ] || [ "$MINIMAL" = "yes" ] -then - cd /opt/vertica/packages - for i in $(find . -name package.conf -exec grep Autoinstall=False {} + | cut -d"/" -f2) - do - rm -rf $i - done -fi diff --git a/docker-vertica-udx-java/packages/init.d.functions b/docker-vertica-udx-java/packages/init.d.functions deleted file mode 100644 index b671a8cbc..000000000 --- a/docker-vertica-udx-java/packages/init.d.functions +++ /dev/null @@ -1,95 +0,0 @@ -# -*-Shell-script-*- -# -# borrowed from /etc/rc/init.d/functions --- functions used -# by the vertica_agent script -# - -# Set up a default search path. -PATH="/sbin:/usr/sbin:/bin:/usr/bin" -export PATH - -# __proc_pids {program} [pidfile] -# Set $pid to pids from /var/run* for {program}. $pid should be declared -# local in the caller. -# Returns LSB exit code for the 'status' action. -__pids_var_run() { - local base=${1##*/} - local pid_file=${2:-/var/run/$base.pid} - - pid= - if [ -f "$pid_file" ] ; then - local line p - read line < "$pid_file" - for p in $line ; do - [ -z "${p//[0-9]/}" -a -d "/proc/$p" ] && pid="$pid $p" - done - if [ -n "$pid" ]; then - return 0 - fi - return 1 # "Program is dead and /var/run pid file exists" - fi - return 3 # "Program is not running" -} - -# Output PIDs of matching processes, found using pidof -__pids_pidof() { - pidof -c -o $$ -o $PPID -o %PPID -x "$1" || \ - pidof -c -o $$ -o $PPID -o %PPID -x "${1##*/}" -} - -status() { - local base pid pid_file= - - # Test syntax. - if [ "$#" = 0 ] ; then - echo $"Usage: status [-p pidfile] {program}" - return 1 - fi - if [ "$1" = "-p" ]; then - pid_file=$2 - shift 2 - fi - base=${1##*/} - - # First try "pidof" - __pids_var_run "$1" "$pid_file" - RC=$? - if [ -z "$pid_file" -a -z "$pid" ]; then - pid="$(__pids_pidof "$1")" - fi - if [ -n "$pid" ]; then - echo $"${base} (pid $pid) is running..." - return 0 - fi - - case "$RC" in - 0) - echo $"${base} (pid $pid) is running..." - return 0 - ;; - 1) - echo $"${base} dead but pid file exists" - return 1 - ;; - esac - # See if /var/lock/subsys/${base} exists - if [ -f /var/lock/subsys/${base} ]; then - echo $"${base} dead but subsys locked" - return 2 - fi - echo $"${base} is stopped" - return 3 -} - -echo_success() { - echo " OK " - return 0 -} - -echo_failure() { - echo -n "[" - echo -n $"FAILED" - echo -n "]" - echo -ne "\r" - return 1 -} diff --git a/docker-vertica-udx-java/packages/package-checksum-patcher.py b/docker-vertica-udx-java/packages/package-checksum-patcher.py deleted file mode 100644 index 4c28c1fba..000000000 --- a/docker-vertica-udx-java/packages/package-checksum-patcher.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2021 Vertica - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -To save about 900MB of space, we strip the package libraries in -/opt/vertica/packages. - -Unfortunately, each directory in /opt/vertica/packages has two files: - - package.conf - - ddl/isinstalled.sql - -package.conf looks like this: - -[Info] -Description=Amazon Web Services Package -Autoinstall=True -Version=11.0.0 -md5sum=b22b4486faa8df8d70fc399ac5a85521 - -and isinstalled.sql looks like this: - -SELECT (COUNT(0) = 5) FROM user_libraries -INNER JOIN user_library_manifest ON user_libraries.lib_name = user_library_manifest.lib_name -WHERE user_library_manifest.lib_name = 'awslib' -AND user_libraries.schema_name = 'public' -AND (user_libraries.md5_sum = 'b22b4486faa8df8d70fc399ac5a85521' - OR public.length('b22b4486faa8df8d70fc399ac5a85521') = 7); - -by stripping the libraries, we've changed their checksums, so these -files are no longer accurate. - -So, we patch the relevant files. - -This runs in a pretty stripped-down environment, so we try to keep ourselves -to core python. -""" -import os -import os.path -import re -import sys -import subprocess - -progname = sys.argv[0] - -def parse_conf(dir): - """ - Extract the Autoinstall and md5sum fields from the - package.conf file - - Args: - - dir: directory name - Returns: - (autoinstall value, checksum value), where - autoinstall value is "True" or "False" - """ - md5pat = re.compile('^md5sum=(.*)$') - autopat = re.compile('^Autoinstall=(.*)$') - checksum = None - autoinstall = None - - with open(dir + '/package.conf', 'r') as fp: - for line in fp: - m = md5pat.match(line) - if m: - checksum = m.group(1) - else: - m = autopat.match(line) - if m: - autoinstall = m.group(1) - if autoinstall and checksum: - return (autoinstall, checksum) - return (autoinstall, checksum) - -def patch_file(fname, old_checksum, new_checksum): - """ - Replace the old checksum with the new checksum in file - Args: - - fname: name of file to patch - - old_checksum: *string* containing the old checksum value to be replaced - - new_checksum: *string* containing the new checksum to insert - Returns: - None - - Backs fname up as fname~ - """ - print(f' file {fname} {old_checksum} -> {new_checksum}') - file_new = fname + '.new' - file_backup = fname + '~' - xsumpat = re.compile(old_checksum) - with open(fname, 'r') as ifp: - with open(file_new, 'w') as ofp: - for line in ifp: - edited = re.sub(xsumpat, new_checksum, line) - print(f'{edited}', file=ofp, end='') - - try: - os.remove(file_backup) - except FileNotFoundError: - # the first time through, the file won't be found - pass - os.remove(fname) - os.rename(file_new, fname) - -def patch_dir(dir, old_checksum): - """ - Patch the package.conf and ddl/isinstalled.sql files in directory dir - Args: - - dir: the package directory - - old_checksum: the old checksum to be replaced with the new checksum - Returns: - None - """ - libname = 'lib' + os.path.basename(dir) + '.so' - xsum_out = str(subprocess.check_output(['md5sum', dir + '/lib/' + libname])) - new_checksum = xsum_out.split(' ')[0] - # converting byte-like object to string prefixes the string with "b'" - if new_checksum.startswith("b'"): - new_checksum = new_checksum[2:] - patch_file(dir + '/package.conf', old_checksum, new_checksum) - patch_file(dir + '/ddl/isinstalled.sql', old_checksum, new_checksum) - -def process_dir(dir): - """ - Process a package directory: - - figure out if the package is auto-installed - - if so, patch its checksum - Skips packages that aren't automatically installed (maybe it - shouldn't --- how to install those, after all?) - """ - (autoinstall, checksum) = parse_conf(dir) - if checksum: - print(f'patching directory {dir}') - patch_dir(dir, checksum) - else: - # no package.conf file, or no checksum in it --> probably not set up - # with standard package mechanism. - print(f'skipping directory {dir} with no checksum in package.conf file') - -def main(argv): - """ - Iterate over the list of files passed as arguments - """ - if len(argv) < 2: - print(f'Usage: {progname} packagedir1 [packagedir2...]', file=sys.stderr) - sys.exit(1) - # argv[0] is the command name - for dir in argv[1:]: - process_dir(dir) - -if __name__ == '__main__': - main(sys.argv) - sys.exit(0) diff --git a/docker-vertica/Dockerfile b/docker-vertica/Dockerfile index 16ea5038d..d1bb0e658 100644 --- a/docker-vertica/Dockerfile +++ b/docker-vertica/Dockerfile @@ -5,10 +5,11 @@ ARG BASE_OS_VERSION="focal-20220316" ARG BUILDER_OS_VERSION="7.9.2009" +ARG MINIMAL="" FROM centos:centos${BUILDER_OS_VERSION} as builder ARG VERTICA_RPM="vertica-x86_64.RHEL6.latest.rpm" -ARG MINIMAL="" +ARG MINIMAL ARG DBADMIN_GID=5000 ARG DBADMIN_UID=5000 @@ -87,6 +88,14 @@ FROM ubuntu:${BASE_OS_VERSION} ARG DBADMIN_GID=5000 ARG DBADMIN_UID=5000 +# Controls the version of jre to be installed by apt-get. The list of all +# available jre packages can be queried through apt-cache. For instance, +# "apt-cache search jre | grep jre" +ARG JRE_PKG=openjdk-8-jre-headless +ARG MINIMAL +ENV MINIMAL=${MINIMAL:-""} +ARG USING_INIT="" +ENV USING_INIT=${USING_INIT:-""} COPY --from=builder /opt/vertica /opt/vertica COPY --from=builder /home/dbadmin /home/dbadmin @@ -128,6 +137,16 @@ RUN set -x \ procps \ sysstat \ sudo \ + && rm -rf /var/lib/apt/lists/* +RUN set -x \ + # Install jre if not minimal and not using init container + && if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") \ + && ($USING_INIT != "YES" && $USING_INIT != "yes") ]]; then \ + apt-get -y update \ + && apt-get install -y --no-install-recommends $JRE_PKG \ + && rm -rf /var/lib/apt/lists/* ; \ + fi +RUN set -x \ && rm -rf /var/lib/apt/lists/* \ # Make the "en_US.UTF-8" locale so vertica will be utf-8 enabled by default && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ @@ -137,9 +156,15 @@ RUN set -x \ && /usr/sbin/useradd -r -m -s /bin/bash -g verticadba --uid ${DBADMIN_UID} dbadmin \ # Allow passwordless sudo access from dbadmin && echo "dbadmin ALL=(ALL) NOPASSWD: ALL" | tee -a /etc/sudoers \ - && echo "dbadmin - nofile 65536" >> /etc/security/limits.conf - -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + && echo "dbadmin - nofile 65536" >> /etc/security/limits.conf \ + # Set JAVA_HOME environment variable if not minimal + && if [[ $MINIMAL != "YES" && $MINIMAL != "yes" ]]; then \ + # echo "export JAVA_HOME=/usr" >> /home/dbadmin/.bashrc; \ + echo "JAVA_HOME=/usr" >> /etc/environment; \ + fi + +ENTRYPOINT [ "/usr/local/bin/docker-entrypoint.sh" ] +# ENTRYPOINT "MINIMAL=${MINIMAL} USING_INIT=${USING_INIT} /usr/local/bin/docker-entrypoint.sh" # vertica port EXPOSE 5433 diff --git a/docker-vertica/Makefile b/docker-vertica/Makefile index ebd60e7ff..32cb62b96 100644 --- a/docker-vertica/Makefile +++ b/docker-vertica/Makefile @@ -4,6 +4,7 @@ BASE_OS_VERSION?=focal-20220316 VERTICA_IMG?=vertica-k8s MINIMAL_VERTICA_IMG?= VERTICA_VERSION?=$(shell rpm --nosignature -qp --queryformat '%{VERSION}-%{RELEASE}' packages/$(VERTICA_RPM)) +USING_INIT?= all: docker-build-vertica @@ -18,4 +19,5 @@ docker-build-vertica: Dockerfile packages/package-checksum-patcher.py --build-arg VERTICA_RPM=${VERTICA_RPM} \ --build-arg BASE_OS_VERSION=${BASE_OS_VERSION} \ --build-arg BUILDER_OS_VERSION=${BUILDER_OS_VERSION} \ + --build-arg USING_INIT=${USING_INIT} \ -t ${VERTICA_IMG} . diff --git a/docker-vertica/dbadmin/.bash_profile b/docker-vertica/dbadmin/.bash_profile index 13f594aef..a18d4a051 100644 --- a/docker-vertica/dbadmin/.bash_profile +++ b/docker-vertica/dbadmin/.bash_profile @@ -12,3 +12,4 @@ export PATH export LANG=en_US.UTF-8 export TZ=UTC +# export JAVA_HOME=/usr diff --git a/docker-vertica/docker-entrypoint.sh b/docker-vertica/docker-entrypoint.sh index 71627e1e5..ce812669a 100755 --- a/docker-vertica/docker-entrypoint.sh +++ b/docker-vertica/docker-entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -ex start_cron(){ # daemonizes, no need for & @@ -34,4 +34,12 @@ copy_config_files echo "Vertica container is now running" +if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") \ + && ($USING_INIT == "YES" || $USING_INIT == "yes") ]]; then + sudo mkdir -p /usr/lib/jvm + sudo cp -r /opt/jvm /usr/lib/ + # echo $(ls /usr/lib) + sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-se-8u41-ri/bin/java 1 +fi + sudo /usr/sbin/sshd -D diff --git a/kuttl-test.yaml b/kuttl-test.yaml index 884c64a43..21c226d66 100644 --- a/kuttl-test.yaml +++ b/kuttl-test.yaml @@ -19,6 +19,7 @@ skipDelete: true # These are overridden when running kuttl from the Makefile testDirs: - tests/e2e + - tests/e2e-udx - tests/e2e-extra - tests/e2e-server-upgrade - tests/e2e-operator-upgrade-overlays diff --git a/scripts/setup-kustomize.sh b/scripts/setup-kustomize.sh index 41ec9d19a..39610ae39 100755 --- a/scripts/setup-kustomize.sh +++ b/scripts/setup-kustomize.sh @@ -721,11 +721,11 @@ setup_creds_for_private_repo # Descend into each test and create the overlay kustomization. # The overlay is created in a directory like: overlay/ -for tdir in e2e/*/*/base e2e-extra/*/*/base e2e-server-upgrade/*/*/base e2e-operator-upgrade-overlays/*/*/base +for tdir in e2e/*/*/base e2e-extra/*/*/base e2e-server-upgrade/*/*/base e2e-operator-upgrade-overlays/*/*/base e2e-udx/*/*/base do create_vdb_pod_kustomization $(dirname $tdir) $(basename $(realpath $tdir/../..)) done -for tdir in e2e/* e2e-extra/* e2e-disabled/* e2e-server-upgrade/* e2e-operator-upgrade-overlays/* +for tdir in e2e/* e2e-extra/* e2e-disabled/* e2e-server-upgrade/* e2e-operator-upgrade-overlays/* e2e-udx/* e2e-udx/* do clean_communal_kustomization $tdir done diff --git a/tests/e2e-udx/udx-java/00-create-creds.yaml b/tests/e2e-udx/udx-java/00-create-creds.yaml new file mode 100644 index 000000000..42005d8c6 --- /dev/null +++ b/tests/e2e-udx/udx-java/00-create-creds.yaml @@ -0,0 +1,18 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kustomize build ../../manifests/communal-creds/overlay | kubectl apply -f - --namespace $NAMESPACE + - script: kustomize build ../../manifests/priv-container-creds/overlay | kubectl apply -f - --namespace $NAMESPACE diff --git a/tests/e2e-udx/udx-java/05-assert.yaml b/tests/e2e-udx/udx-java/05-assert.yaml new file mode 100644 index 000000000..fc6dfbf1c --- /dev/null +++ b/tests/e2e-udx/udx-java/05-assert.yaml @@ -0,0 +1,20 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + labels: + control-plane: controller-manager +status: + phase: Running diff --git a/tests/e2e-udx/udx-java/05-deploy-operator.yaml b/tests/e2e-udx/udx-java/05-deploy-operator.yaml new file mode 100644 index 000000000..298db6cf6 --- /dev/null +++ b/tests/e2e-udx/udx-java/05-deploy-operator.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: sh -c "cd ../../.. && make deploy-operator NAMESPACE=$NAMESPACE" diff --git a/tests/e2e-udx/udx-java/15-assert.yaml b/tests/e2e-udx/udx-java/15-assert.yaml new file mode 100644 index 000000000..23cc51a4c --- /dev/null +++ b/tests/e2e-udx/udx-java/15-assert.yaml @@ -0,0 +1,27 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-udx-java-sc1 +status: + replicas: 2 +--- +apiVersion: vertica.com/v1beta1 +kind: VerticaDB +metadata: + name: v-udx-java +status: + subclusters: + - installCount: 2 diff --git a/tests/e2e-udx/udx-java/15-setup-vdb.yaml b/tests/e2e-udx/udx-java/15-setup-vdb.yaml new file mode 100644 index 000000000..3e10786a3 --- /dev/null +++ b/tests/e2e-udx/udx-java/15-setup-vdb.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "kustomize build setup-vdb/overlay | kubectl -n $NAMESPACE apply -f - " diff --git a/tests/e2e-udx/udx-java/17-assert.yaml b/tests/e2e-udx/udx-java/17-assert.yaml new file mode 100644 index 000000000..af18170ed --- /dev/null +++ b/tests/e2e-udx/udx-java/17-assert.yaml @@ -0,0 +1,51 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-udx-java-sc1 +status: + replicas: 2 + readyReplicas: 2 +--- +apiVersion: vertica.com/v1beta1 +kind: VerticaDB +metadata: + name: v-udx-java +status: + subclusters: + - installCount: 2 + addedToDBCount: 2 + upNodeCount: 2 +--- +# We enable spread encryption, so the cluster will restart as part of the create +apiVersion: v1 +kind: Event +reason: StopDBSucceeded +source: + component: verticadb-operator +involvedObject: + apiVersion: vertica.com/v1beta1 + kind: VerticaDB + name: v-udx-java +--- +apiVersion: v1 +kind: Event +reason: ClusterRestartSucceeded +source: + component: verticadb-operator +involvedObject: + apiVersion: vertica.com/v1beta1 + kind: VerticaDB + name: v-udx-java diff --git a/tests/e2e-udx/udx-java/17-wait-for-createdb.yaml b/tests/e2e-udx/udx-java/17-wait-for-createdb.yaml new file mode 100644 index 000000000..bf3726035 --- /dev/null +++ b/tests/e2e-udx/udx-java/17-wait-for-createdb.yaml @@ -0,0 +1 @@ +# Intentionally empty to give this step a name in kuttl \ No newline at end of file diff --git a/tests/e2e-udx/udx-java/20-assert.yaml b/tests/e2e-udx/udx-java/20-assert.yaml new file mode 100644 index 000000000..6ff84e84a --- /dev/null +++ b/tests/e2e-udx/udx-java/20-assert.yaml @@ -0,0 +1,22 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: integration-test-role +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: integration-test-rb diff --git a/tests/e2e-udx/udx-java/20-rbac.yaml b/tests/e2e-udx/udx-java/20-rbac.yaml new file mode 100644 index 000000000..7881fc2b0 --- /dev/null +++ b/tests/e2e-udx/udx-java/20-rbac.yaml @@ -0,0 +1,18 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl apply --namespace $NAMESPACE -f ../../manifests/rbac/base/rbac.yaml + ignoreFailure: true diff --git a/tests/e2e-udx/udx-java/28-assert.yaml b/tests/e2e-udx/udx-java/28-assert.yaml new file mode 100644 index 000000000..122e1b8bc --- /dev/null +++ b/tests/e2e-udx/udx-java/28-assert.yaml @@ -0,0 +1,28 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# 3 minute timeout mainly to wait for the docker image pull +timeout: 180 +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-ip-family +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-udx/udx-java/28-verify-ip-family.yaml b/tests/e2e-udx/udx-java/28-verify-ip-family.yaml new file mode 100644 index 000000000..a2df6ad15 --- /dev/null +++ b/tests/e2e-udx/udx-java/28-verify-ip-family.yaml @@ -0,0 +1,67 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Verifies external access through the service to the agent port 5444. It +# does this by invoking the REST API and doing basic sanity on what it +# received. + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-verify-ip-family +data: + entrypoint.sh: |- + #!/bin/bash + set -o errexit + set -o xtrace + + VDB_NAME=v-udx-java + SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME + POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") + + IP_FAMILY=$(kubectl exec $POD_NAME -i -- cat /opt/vertica/config/admintools.conf | grep "ipv6 = ") + POD_IP=$(kubectl get po $POD_NAME --no-headers -o custom-columns=":status.podIP") + + if [[ "$POD_IP" == *":"* ]]; then + if [[ "$IP_FAMILY" != "ipv6 = True" ]]; then + exit 5 + fi + else + if [[ "$IP_FAMILY" == "ipv6 = True" ]]; then + exit 5 + fi + fi +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-ip-family + labels: + stern: include +spec: + restartPolicy: OnFailure + containers: + - name: test + image: bitnami/kubectl:1.20.4 + command: ["/bin/entrypoint.sh"] + volumeMounts: + - name: entrypoint-volume + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + volumes: + - name: entrypoint-volume + configMap: + defaultMode: 0777 + name: script-verify-ip-family diff --git a/tests/e2e-udx/udx-java/30-assert.yaml b/tests/e2e-udx/udx-java/30-assert.yaml new file mode 100644 index 000000000..b1446b7c1 --- /dev/null +++ b/tests/e2e-udx/udx-java/30-assert.yaml @@ -0,0 +1,27 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 180 +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-udx-java +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-udx/udx-java/30-verify-vsql-access.yaml b/tests/e2e-udx/udx-java/30-verify-vsql-access.yaml new file mode 100644 index 000000000..b2c5e44ef --- /dev/null +++ b/tests/e2e-udx/udx-java/30-verify-vsql-access.yaml @@ -0,0 +1,20 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Verifies access to the external service through port 5433. It does this +# by connecting using vsql + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "kustomize build verify-vsql-access/overlay | kubectl -n $NAMESPACE apply -f - " diff --git a/tests/e2e-udx/udx-java/35-assert.yaml b/tests/e2e-udx/udx-java/35-assert.yaml new file mode 100644 index 000000000..2bbddc671 --- /dev/null +++ b/tests/e2e-udx/udx-java/35-assert.yaml @@ -0,0 +1,28 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# 3 minute timeout mainly to wait for the docker image pull +timeout: 180 +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-memory-limit +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-udx/udx-java/35-verify-memory-limit.yaml b/tests/e2e-udx/udx-java/35-verify-memory-limit.yaml new file mode 100644 index 000000000..e6d7bf161 --- /dev/null +++ b/tests/e2e-udx/udx-java/35-verify-memory-limit.yaml @@ -0,0 +1,73 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Verify that Vertica node correctly reads information from /etc/podinfo/memory-limit. The pre-req for +# this test to run is to have an existing Vertica database running + +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-verify-memory-limit +data: + entrypoint.sh: |- + #!/bin/bash + set -o xtrace + set -o errexit + + VDB_NAME=v-udx-java + SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME + POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") + MEM_LIMIT_BYTES=$(kubectl exec $POD_NAME -i -- cat /etc/podinfo/memory-limit) + echo "Memory limit is ${MEM_LIMIT_BYTES} bytes" + + cat << EOF > /tmp/vsql_mem_limit_cmds.in + select total_memory_bytes from host_resources order by host_name limit 1; + select case when round((rs.max_memory_size_kb * 1024)/hr.total_memory_bytes, 2) < 0.95 then 'passed' else 'failed' end from resource_pool_status as rs, host_resources as hr where rs.pool_name = 'general' limit 1; + EOF + + kubectl cp /tmp/vsql_mem_limit_cmds.in $POD_NAME:/home/dbadmin/vsql_mem_limit_cmds.in + kubectl exec $POD_NAME -i -- sh -c "cd /home/dbadmin && vsql --echo-queries --file vsql_mem_limit_cmds.in 2>&1 | tee vsql_mem_limit_cmds.out" + + cat << EOF > /tmp/vsql_mem_limit_cmds_verify.sh + set -e + cd /home/dbadmin + cat vsql_mem_limit_cmds.out + grep $MEM_LIMIT_BYTES vsql_mem_limit_cmds.out + grep "passed" vsql_mem_limit_cmds.out + EOF + + kubectl cp /tmp/vsql_mem_limit_cmds_verify.sh $POD_NAME:/home/dbadmin/vsql_mem_limit_cmds_verify.sh + kubectl exec $POD_NAME -i -- sh /home/dbadmin/vsql_mem_limit_cmds_verify.sh +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-memory-limit + labels: + stern: include +spec: + restartPolicy: Never + containers: + - name: test + image: quay.io/helmpack/chart-testing:v3.3.1 + command: ["/bin/entrypoint.sh"] + volumeMounts: + - name: entrypoint-volume + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + volumes: + - name: entrypoint-volume + configMap: + defaultMode: 0700 + name: script-verify-memory-limit diff --git a/tests/e2e-udx/udx-java/40-assert.yaml b/tests/e2e-udx/udx-java/40-assert.yaml new file mode 100644 index 000000000..53e3ed40f --- /dev/null +++ b/tests/e2e-udx/udx-java/40-assert.yaml @@ -0,0 +1,23 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-spread-encryption +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-udx/udx-java/40-verify-spread-encryption.yaml b/tests/e2e-udx/udx-java/40-verify-spread-encryption.yaml new file mode 100644 index 000000000..0cdce22d2 --- /dev/null +++ b/tests/e2e-udx/udx-java/40-verify-spread-encryption.yaml @@ -0,0 +1,49 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-verify-spread-encryption +data: + entrypoint.sh: |- + #!/bin/bash + set -o xtrace + set -o errexit + + SEC_CHK=$(kubectl exec svc/v-udx-java-sc1 -- vsql -c "SELECT SECURITY_CONFIG_CHECK('NETWORK');") + echo $SEC_CHK + echo $SEC_CHK | grep -cq 'Spread encryption is enabled' +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-spread-encryption + labels: + stern: include +spec: + restartPolicy: Never + containers: + - name: test + image: quay.io/helmpack/chart-testing:v3.3.1 + command: ["/bin/entrypoint.sh"] + volumeMounts: + - name: entrypoint-volume + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + volumes: + - name: entrypoint-volume + configMap: + defaultMode: 0700 + name: script-verify-spread-encryption diff --git a/tests/e2e-udx/udx-java/45-assert.yaml b/tests/e2e-udx/udx-java/45-assert.yaml new file mode 100644 index 000000000..8ebd5474e --- /dev/null +++ b/tests/e2e-udx/udx-java/45-assert.yaml @@ -0,0 +1,23 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-vstack +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-udx/udx-java/45-verify-vstack.yaml b/tests/e2e-udx/udx-java/45-verify-vstack.yaml new file mode 100644 index 000000000..cc1a08d68 --- /dev/null +++ b/tests/e2e-udx/udx-java/45-verify-vstack.yaml @@ -0,0 +1,49 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-verify-vstack +data: + entrypoint.sh: |- + #!/bin/bash + set -o xtrace + set -o errexit + + VSTACK_OP=$(kubectl exec svc/v-udx-java-sc1 -- sudo /opt/vertica/bin/vstack) + echo $VSTACK_OP + echo $VSTACK_OP | grep -cq 'Thread' +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-vstack + labels: + stern: include +spec: + restartPolicy: Never + containers: + - name: test + image: quay.io/helmpack/chart-testing:v3.3.1 + command: ["/bin/entrypoint.sh"] + volumeMounts: + - name: entrypoint-volume + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + volumes: + - name: entrypoint-volume + configMap: + defaultMode: 0700 + name: script-verify-vstack diff --git a/tests/e2e-udx/udx-java/50-assert.yaml b/tests/e2e-udx/udx-java/50-assert.yaml new file mode 100644 index 000000000..8396f62e5 --- /dev/null +++ b/tests/e2e-udx/udx-java/50-assert.yaml @@ -0,0 +1,23 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-java-scalar-transform +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml new file mode 100644 index 000000000..67d31a27b --- /dev/null +++ b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml @@ -0,0 +1,79 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-verify-java-scalar-transform +data: + entrypoint.sh: |- + #!/bin/bash + set -o xtrace + set -o errexit + + VDB_NAME=v-udx-java + SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME + POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") + echo $POD_NAME + # UDX_OP=$(kubectl exec svc/v-udx-java-sc1 -i -- sh -c "cd /bin/UDx-Examples/build; ls -al;") + # kubectl exec $POD_NAME -i -- bash -c "sudo mkdir -p /home/dbadmin/host/UDx-Examples/" + # kubectl cp /bin/UDx-Examples/ $POD_NAME:/home/dbadmin/host/ + UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "vsql -U dbadmin -c \"SELECT SET_CONFIG_PARAMETER('JavaSideProcessMinHeapSizeMB',2048);\"") + echo "$UDX_OP" + UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /host/UDx-Examples; vsql -U dbadmin -f Java-and-C++/JavaFunctions.sql") + echo "$UDX_OP" + echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 2 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 3 && /CREATE FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 4 && /CREATE FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 5 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 6 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 7 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 8 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-java-scalar-transform + labels: + stern: include +spec: + restartPolicy: Never + containers: + - name: test + image: quay.io/helmpack/chart-testing:v3.3.1 + command: ["/bin/entrypoint.sh"] + volumeMounts: + - name: entrypoint-volume + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + # - name: lib-volume + # mountPath: /bin/UDx-Examples/build + # readOnly: true + # - name: sql-volume + # mountPath: /bin/UDx-Examples/Java-and-C++ + # readOnly: true + volumes: + - name: entrypoint-volume + configMap: + defaultMode: 0700 + name: script-verify-java-scalar-transform + # - name: lib-volume + # hostPath: + # path: /host/UDx-Examples/build + # type: Directory + # - name: sql-volume + # hostPath: + # path: /host/UDx-Examples/Java-and-C++ + # type: Directory diff --git a/tests/e2e-udx/udx-java/51-assert.yaml b/tests/e2e-udx/udx-java/51-assert.yaml new file mode 100644 index 000000000..9f5490949 --- /dev/null +++ b/tests/e2e-udx/udx-java/51-assert.yaml @@ -0,0 +1,23 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-java-uda +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-udx/udx-java/51-verify-java-uda.yaml b/tests/e2e-udx/udx-java/51-verify-java-uda.yaml new file mode 100644 index 000000000..245d0b3c3 --- /dev/null +++ b/tests/e2e-udx/udx-java/51-verify-java-uda.yaml @@ -0,0 +1,59 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-verify-java-uda +data: + entrypoint.sh: |- + #!/bin/bash + set -o xtrace + set -o errexit + + VDB_NAME=v-udx-java + SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME + POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") + echo $POD_NAME + # UDX_OP=$(kubectl exec svc/v-udx-java-sc1 -i -- sh -c "cd /bin/UDx-Examples/build; ls -al;") + # kubectl exec $POD_NAME -i -- bash -c "sudo mkdir -p /home/dbadmin/host/UDx-Examples/" + # kubectl cp /bin/UDx-Examples/ $POD_NAME:/home/dbadmin/host/ + UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /host/UDx-Examples; vsql -U dbadmin -f Java-and-C++/JavaUDAnalytics.sql") + echo "$UDX_OP" + echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 2 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 3 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 4 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-java-uda + labels: + stern: include +spec: + restartPolicy: Never + containers: + - name: test + image: quay.io/helmpack/chart-testing:v3.3.1 + command: ["/bin/entrypoint.sh"] + volumeMounts: + - name: entrypoint-volume + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + volumes: + - name: entrypoint-volume + configMap: + defaultMode: 0700 + name: script-verify-java-uda diff --git a/tests/e2e-udx/udx-java/52-assert.yaml b/tests/e2e-udx/udx-java/52-assert.yaml new file mode 100644 index 000000000..2aee61a28 --- /dev/null +++ b/tests/e2e-udx/udx-java/52-assert.yaml @@ -0,0 +1,23 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-java-udl +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-udx/udx-java/52-verify-java-udl.yaml b/tests/e2e-udx/udx-java/52-verify-java-udl.yaml new file mode 100644 index 000000000..52fcb2028 --- /dev/null +++ b/tests/e2e-udx/udx-java/52-verify-java-udl.yaml @@ -0,0 +1,60 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-verify-java-udl +data: + entrypoint.sh: |- + #!/bin/bash + set -o xtrace + set -o errexit + + VDB_NAME=v-udx-java + SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME + POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") + echo $POD_NAME + # UDX_OP=$(kubectl exec svc/v-udx-java-sc1 -i -- sh -c "cd /bin/UDx-Examples/build; ls -al;") + # kubectl exec $POD_NAME -i -- bash -c "sudo mkdir -p /home/dbadmin/host/UDx-Examples/" + # kubectl cp /bin/UDx-Examples/ $POD_NAME:/home/dbadmin/host/ + UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /host/UDx-Examples; vsql -U dbadmin -f Java-and-C++/JavaUDLFunctions.sql") + echo "$UDX_OP" + echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 2 && /CREATE SOURCE FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 3 && /CREATE PARSER FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 4 && /CREATE PARSER FUNCTION/ {print}' | wc -l | grep -cq '1' + echo "$UDX_OP" | awk 'FNR == 5 && /CREATE FILTER FUNCTION/ {print}' | wc -l | grep -cq '1' +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-java-udl + labels: + stern: include +spec: + restartPolicy: Never + containers: + - name: test + image: quay.io/helmpack/chart-testing:v3.3.1 + command: ["/bin/entrypoint.sh"] + volumeMounts: + - name: entrypoint-volume + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + volumes: + - name: entrypoint-volume + configMap: + defaultMode: 0700 + name: script-verify-java-udl diff --git a/tests/e2e-udx/udx-java/60-errors.yaml b/tests/e2e-udx/udx-java/60-errors.yaml new file mode 100644 index 000000000..2d2e094b3 --- /dev/null +++ b/tests/e2e-udx/udx-java/60-errors.yaml @@ -0,0 +1,18 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + labels: + control-plane: controller-manager diff --git a/tests/e2e-udx/udx-java/60-uninstall-operator.yaml b/tests/e2e-udx/udx-java/60-uninstall-operator.yaml new file mode 100644 index 000000000..674dcbecc --- /dev/null +++ b/tests/e2e-udx/udx-java/60-uninstall-operator.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: sh -c "cd ../../.. && make undeploy-operator NAMESPACE=$NAMESPACE" diff --git a/tests/e2e-udx/udx-java/65-assert.yaml b/tests/e2e-udx/udx-java/65-assert.yaml new file mode 100644 index 000000000..fe22a5139 --- /dev/null +++ b/tests/e2e-udx/udx-java/65-assert.yaml @@ -0,0 +1,19 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: clean-communal +status: + phase: Succeeded diff --git a/tests/e2e-udx/udx-java/65-delete-crd.yaml b/tests/e2e-udx/udx-java/65-delete-crd.yaml new file mode 100644 index 000000000..27b82bb6a --- /dev/null +++ b/tests/e2e-udx/udx-java/65-delete-crd.yaml @@ -0,0 +1,22 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: vertica.com/v1beta1 + kind: VerticaDB + - apiVersion: v1 + kind: PersistentVolumeClaim +commands: + - command: bash -c "kustomize build clean-communal/overlay | kubectl -n $NAMESPACE apply -f - " diff --git a/tests/e2e-udx/udx-java/65-errors.yaml b/tests/e2e-udx/udx-java/65-errors.yaml new file mode 100644 index 000000000..415cba0a7 --- /dev/null +++ b/tests/e2e-udx/udx-java/65-errors.yaml @@ -0,0 +1,24 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: verticadb-operator +--- +apiVersion: vertica.com/v1beta1 +kind: VerticaDB diff --git a/tests/e2e-udx/udx-java/99-delete-ns.yaml b/tests/e2e-udx/udx-java/99-delete-ns.yaml new file mode 100644 index 000000000..4cf81e196 --- /dev/null +++ b/tests/e2e-udx/udx-java/99-delete-ns.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl delete ns $NAMESPACE diff --git a/tests/e2e-udx/udx-java/README.txt b/tests/e2e-udx/udx-java/README.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/tests/e2e-udx/udx-java/README.txt @@ -0,0 +1 @@ + diff --git a/tests/e2e-udx/udx-java/setup-vdb/base/kustomization.yaml b/tests/e2e-udx/udx-java/setup-vdb/base/kustomization.yaml new file mode 100644 index 000000000..01b927990 --- /dev/null +++ b/tests/e2e-udx/udx-java/setup-vdb/base/kustomization.yaml @@ -0,0 +1,15 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: + - setup-vdb.yaml diff --git a/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml b/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml new file mode 100644 index 000000000..547eceecc --- /dev/null +++ b/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml @@ -0,0 +1,38 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1beta1 +kind: VerticaDB +metadata: + name: v-udx-java +spec: + image: kustomize-vertica-image + communal: + includeUIDInPath: true + local: + requestSize: 100Mi + dbName: vertdb + encryptSpreadComm: vertica + subclusters: + - name: sc1 + size: 2 + kSafety: "0" + certSecrets: [] + imagePullSecrets: [] + volumes: + - name: downloads + hostPath: + path: /host + volumeMounts: + - mountPath: /host + name: downloads diff --git a/tests/e2e-udx/udx-java/verify-vsql-access/base/kustomization.yaml b/tests/e2e-udx/udx-java/verify-vsql-access/base/kustomization.yaml new file mode 100644 index 000000000..913a739b6 --- /dev/null +++ b/tests/e2e-udx/udx-java/verify-vsql-access/base/kustomization.yaml @@ -0,0 +1,15 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: + - verify-vsql-access.yaml diff --git a/tests/e2e-udx/udx-java/verify-vsql-access/base/verify-vsql-access.yaml b/tests/e2e-udx/udx-java/verify-vsql-access/base/verify-vsql-access.yaml new file mode 100644 index 000000000..781e03fff --- /dev/null +++ b/tests/e2e-udx/udx-java/verify-vsql-access/base/verify-vsql-access.yaml @@ -0,0 +1,57 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Verifies access to the external service through port 5433. It does this +# by connecting using vsql + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-verify-udx-java-test +data: + test.sh: |- + #!/bin/bash + set -o errexit + set -o xtrace + + CLUSTER_IP=v-udx-java-sc1 + vsql -h $CLUSTER_IP -c "drop table if exists t1" + vsql -h $CLUSTER_IP -c "create table t1 (c1 int)" + vsql -h $CLUSTER_IP -c "insert into t1 values (1)" + vsql -h $CLUSTER_IP -c "select * from t1" + vsql -h $CLUSTER_IP -c "drop table t1" + +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-verify-udx-java + labels: + stern: include +spec: + restartPolicy: Never + containers: + - name: test + image: kustomize-vertica-image + command: ["/home/dbadmin/test.sh"] + volumeMounts: + - name: test-script + mountPath: /home/dbadmin/test.sh + readOnly: true + subPath: test.sh + volumes: + - name: test-script + configMap: + defaultMode: 0777 + name: script-verify-udx-java-test From ec3b0a057392055917bd6b8c8a93c59233df5106 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Mon, 15 Aug 2022 17:43:54 -0400 Subject: [PATCH 09/30] java udx e2e test working; vertica image updated --- .../v1beta1_verticadb_host_mount_java.yaml | 2 +- docker-vertica/Dockerfile | 6 +- docker-vertica/Makefile | 2 - docker-vertica/docker-entrypoint.sh | 14 +- docker-vertica/packages/cleanup.sh | 9 +- tests/e2e-udx/udx-java/15-assert.yaml | 4 +- tests/e2e-udx/udx-java/17-assert.yaml | 10 +- tests/e2e-udx/udx-java/28-assert.yaml | 28 ---- .../e2e-udx/udx-java/28-verify-ip-family.yaml | 67 ---------- tests/e2e-udx/udx-java/35-assert.yaml | 28 ---- .../udx-java/35-verify-memory-limit.yaml | 73 ---------- tests/e2e-udx/udx-java/40-assert.yaml | 23 ---- .../udx-java/40-verify-spread-encryption.yaml | 49 ------- tests/e2e-udx/udx-java/45-assert.yaml | 23 ---- tests/e2e-udx/udx-java/45-upload-files.yaml | 14 ++ tests/e2e-udx/udx-java/45-verify-vstack.yaml | 49 ------- .../50-verify-java-scalar-transform.yaml | 22 ++- .../e2e-udx/udx-java/51-verify-java-uda.yaml | 17 +-- .../e2e-udx/udx-java/52-verify-java-udl.yaml | 16 +-- .../expected-outputs/JavaFunctionsOut.txt | 125 ++++++++++++++++++ .../expected-outputs/JavaUDAnalyticsOut.txt | 59 +++++++++ .../expected-outputs/JavaUDLFunctionsOut.txt | 63 +++++++++ .../udx-java/setup-vdb/base/setup-vdb.yaml | 16 +-- 23 files changed, 318 insertions(+), 401 deletions(-) delete mode 100644 tests/e2e-udx/udx-java/28-assert.yaml delete mode 100644 tests/e2e-udx/udx-java/28-verify-ip-family.yaml delete mode 100644 tests/e2e-udx/udx-java/35-assert.yaml delete mode 100644 tests/e2e-udx/udx-java/35-verify-memory-limit.yaml delete mode 100644 tests/e2e-udx/udx-java/40-assert.yaml delete mode 100644 tests/e2e-udx/udx-java/40-verify-spread-encryption.yaml delete mode 100644 tests/e2e-udx/udx-java/45-assert.yaml create mode 100644 tests/e2e-udx/udx-java/45-upload-files.yaml delete mode 100644 tests/e2e-udx/udx-java/45-verify-vstack.yaml create mode 100644 tests/e2e-udx/udx-java/expected-outputs/JavaFunctionsOut.txt create mode 100644 tests/e2e-udx/udx-java/expected-outputs/JavaUDAnalyticsOut.txt create mode 100644 tests/e2e-udx/udx-java/expected-outputs/JavaUDLFunctionsOut.txt diff --git a/config/samples/v1beta1_verticadb_host_mount_java.yaml b/config/samples/v1beta1_verticadb_host_mount_java.yaml index 514a4bae4..065bc9764 100644 --- a/config/samples/v1beta1_verticadb_host_mount_java.yaml +++ b/config/samples/v1beta1_verticadb_host_mount_java.yaml @@ -16,7 +16,7 @@ kind: VerticaDB metadata: name: verticadb-sample spec: - image: "vertica-k8s-fat:1.6.0" + image: "vertica-k8s-fat:1.7.0" communal: path: "s3://nimbusdb/db" endpoint: "http://minio" diff --git a/docker-vertica/Dockerfile b/docker-vertica/Dockerfile index d1bb0e658..c8aa5297c 100644 --- a/docker-vertica/Dockerfile +++ b/docker-vertica/Dockerfile @@ -93,9 +93,6 @@ ARG DBADMIN_UID=5000 # "apt-cache search jre | grep jre" ARG JRE_PKG=openjdk-8-jre-headless ARG MINIMAL -ENV MINIMAL=${MINIMAL:-""} -ARG USING_INIT="" -ENV USING_INIT=${USING_INIT:-""} COPY --from=builder /opt/vertica /opt/vertica COPY --from=builder /home/dbadmin /home/dbadmin @@ -140,8 +137,7 @@ RUN set -x \ && rm -rf /var/lib/apt/lists/* RUN set -x \ # Install jre if not minimal and not using init container - && if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") \ - && ($USING_INIT != "YES" && $USING_INIT != "yes") ]]; then \ + && if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") ]] ; then \ apt-get -y update \ && apt-get install -y --no-install-recommends $JRE_PKG \ && rm -rf /var/lib/apt/lists/* ; \ diff --git a/docker-vertica/Makefile b/docker-vertica/Makefile index 32cb62b96..ebd60e7ff 100644 --- a/docker-vertica/Makefile +++ b/docker-vertica/Makefile @@ -4,7 +4,6 @@ BASE_OS_VERSION?=focal-20220316 VERTICA_IMG?=vertica-k8s MINIMAL_VERTICA_IMG?= VERTICA_VERSION?=$(shell rpm --nosignature -qp --queryformat '%{VERSION}-%{RELEASE}' packages/$(VERTICA_RPM)) -USING_INIT?= all: docker-build-vertica @@ -19,5 +18,4 @@ docker-build-vertica: Dockerfile packages/package-checksum-patcher.py --build-arg VERTICA_RPM=${VERTICA_RPM} \ --build-arg BASE_OS_VERSION=${BASE_OS_VERSION} \ --build-arg BUILDER_OS_VERSION=${BUILDER_OS_VERSION} \ - --build-arg USING_INIT=${USING_INIT} \ -t ${VERTICA_IMG} . diff --git a/docker-vertica/docker-entrypoint.sh b/docker-vertica/docker-entrypoint.sh index ce812669a..b3ae76705 100755 --- a/docker-vertica/docker-entrypoint.sh +++ b/docker-vertica/docker-entrypoint.sh @@ -34,12 +34,12 @@ copy_config_files echo "Vertica container is now running" -if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") \ - && ($USING_INIT == "YES" || $USING_INIT == "yes") ]]; then - sudo mkdir -p /usr/lib/jvm - sudo cp -r /opt/jvm /usr/lib/ - # echo $(ls /usr/lib) - sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-se-8u41-ri/bin/java 1 -fi +# if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") \ +# && ($USING_INIT == "YES" || $USING_INIT == "yes") ]]; then +# sudo mkdir -p /usr/lib/jvm +# sudo cp -r /opt/jvm /usr/lib/ +# # echo $(ls /usr/lib) +# sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-se-8u41-ri/bin/java 1 +# fi sudo /usr/sbin/sshd -D diff --git a/docker-vertica/packages/cleanup.sh b/docker-vertica/packages/cleanup.sh index 1bb57669a..b548c2b16 100644 --- a/docker-vertica/packages/cleanup.sh +++ b/docker-vertica/packages/cleanup.sh @@ -24,7 +24,6 @@ # wander around in the image looking for things you can remove rm -r -f \ /opt/vertica/examples \ - /opt/vertica/sdk \ /opt/vertica/packages/*/examples \ /opt/vertica/oss/python*/lib/python*/test \ /opt/vertica/oss/python*/lib/python*/unittest/test \ @@ -32,7 +31,13 @@ rm -r -f \ /opt/vertica/oss/python*/lib/python*/site-packages/pip \ /opt/vertica/oss/python*/lib/python*/config-[0-9]* \ /opt/vertica/oss/python*/lib/python*/tkinter \ - /opt/vertica/oss/python*/lib/python*/idlelib + /opt/vertica/oss/python*/lib/python*/idlelib + + # (optional) minimal images remove sdk folder which enables building UDx libraries +if [ "$MINIMAL" = "YES" ] || [ "$MINIMAL" = "yes" ] +then + rm -r -f /opt/vertica/sdk +fi # cleanup many of the __pycache__ directories find /opt/vertica/oss/ -type d -name "__pycache__" -exec rm -rf {} + diff --git a/tests/e2e-udx/udx-java/15-assert.yaml b/tests/e2e-udx/udx-java/15-assert.yaml index 23cc51a4c..641fba39b 100644 --- a/tests/e2e-udx/udx-java/15-assert.yaml +++ b/tests/e2e-udx/udx-java/15-assert.yaml @@ -16,7 +16,7 @@ kind: StatefulSet metadata: name: v-udx-java-sc1 status: - replicas: 2 + replicas: 1 --- apiVersion: vertica.com/v1beta1 kind: VerticaDB @@ -24,4 +24,4 @@ metadata: name: v-udx-java status: subclusters: - - installCount: 2 + - installCount: 1 diff --git a/tests/e2e-udx/udx-java/17-assert.yaml b/tests/e2e-udx/udx-java/17-assert.yaml index af18170ed..4262621aa 100644 --- a/tests/e2e-udx/udx-java/17-assert.yaml +++ b/tests/e2e-udx/udx-java/17-assert.yaml @@ -16,8 +16,8 @@ kind: StatefulSet metadata: name: v-udx-java-sc1 status: - replicas: 2 - readyReplicas: 2 + replicas: 1 + readyReplicas: 1 --- apiVersion: vertica.com/v1beta1 kind: VerticaDB @@ -25,9 +25,9 @@ metadata: name: v-udx-java status: subclusters: - - installCount: 2 - addedToDBCount: 2 - upNodeCount: 2 + - installCount: 1 + addedToDBCount: 1 + upNodeCount: 1 --- # We enable spread encryption, so the cluster will restart as part of the create apiVersion: v1 diff --git a/tests/e2e-udx/udx-java/28-assert.yaml b/tests/e2e-udx/udx-java/28-assert.yaml deleted file mode 100644 index 122e1b8bc..000000000 --- a/tests/e2e-udx/udx-java/28-assert.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# 3 minute timeout mainly to wait for the docker image pull -timeout: 180 ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-ip-family -status: - containerStatuses: - - name: test - state: - terminated: - exitCode: 0 diff --git a/tests/e2e-udx/udx-java/28-verify-ip-family.yaml b/tests/e2e-udx/udx-java/28-verify-ip-family.yaml deleted file mode 100644 index a2df6ad15..000000000 --- a/tests/e2e-udx/udx-java/28-verify-ip-family.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Verifies external access through the service to the agent port 5444. It -# does this by invoking the REST API and doing basic sanity on what it -# received. - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: script-verify-ip-family -data: - entrypoint.sh: |- - #!/bin/bash - set -o errexit - set -o xtrace - - VDB_NAME=v-udx-java - SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME - POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") - - IP_FAMILY=$(kubectl exec $POD_NAME -i -- cat /opt/vertica/config/admintools.conf | grep "ipv6 = ") - POD_IP=$(kubectl get po $POD_NAME --no-headers -o custom-columns=":status.podIP") - - if [[ "$POD_IP" == *":"* ]]; then - if [[ "$IP_FAMILY" != "ipv6 = True" ]]; then - exit 5 - fi - else - if [[ "$IP_FAMILY" == "ipv6 = True" ]]; then - exit 5 - fi - fi ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-ip-family - labels: - stern: include -spec: - restartPolicy: OnFailure - containers: - - name: test - image: bitnami/kubectl:1.20.4 - command: ["/bin/entrypoint.sh"] - volumeMounts: - - name: entrypoint-volume - mountPath: /bin/entrypoint.sh - readOnly: true - subPath: entrypoint.sh - volumes: - - name: entrypoint-volume - configMap: - defaultMode: 0777 - name: script-verify-ip-family diff --git a/tests/e2e-udx/udx-java/35-assert.yaml b/tests/e2e-udx/udx-java/35-assert.yaml deleted file mode 100644 index 2bbddc671..000000000 --- a/tests/e2e-udx/udx-java/35-assert.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# 3 minute timeout mainly to wait for the docker image pull -timeout: 180 ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-memory-limit -status: - containerStatuses: - - name: test - state: - terminated: - exitCode: 0 diff --git a/tests/e2e-udx/udx-java/35-verify-memory-limit.yaml b/tests/e2e-udx/udx-java/35-verify-memory-limit.yaml deleted file mode 100644 index e6d7bf161..000000000 --- a/tests/e2e-udx/udx-java/35-verify-memory-limit.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Verify that Vertica node correctly reads information from /etc/podinfo/memory-limit. The pre-req for -# this test to run is to have an existing Vertica database running - -apiVersion: v1 -kind: ConfigMap -metadata: - name: script-verify-memory-limit -data: - entrypoint.sh: |- - #!/bin/bash - set -o xtrace - set -o errexit - - VDB_NAME=v-udx-java - SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME - POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") - MEM_LIMIT_BYTES=$(kubectl exec $POD_NAME -i -- cat /etc/podinfo/memory-limit) - echo "Memory limit is ${MEM_LIMIT_BYTES} bytes" - - cat << EOF > /tmp/vsql_mem_limit_cmds.in - select total_memory_bytes from host_resources order by host_name limit 1; - select case when round((rs.max_memory_size_kb * 1024)/hr.total_memory_bytes, 2) < 0.95 then 'passed' else 'failed' end from resource_pool_status as rs, host_resources as hr where rs.pool_name = 'general' limit 1; - EOF - - kubectl cp /tmp/vsql_mem_limit_cmds.in $POD_NAME:/home/dbadmin/vsql_mem_limit_cmds.in - kubectl exec $POD_NAME -i -- sh -c "cd /home/dbadmin && vsql --echo-queries --file vsql_mem_limit_cmds.in 2>&1 | tee vsql_mem_limit_cmds.out" - - cat << EOF > /tmp/vsql_mem_limit_cmds_verify.sh - set -e - cd /home/dbadmin - cat vsql_mem_limit_cmds.out - grep $MEM_LIMIT_BYTES vsql_mem_limit_cmds.out - grep "passed" vsql_mem_limit_cmds.out - EOF - - kubectl cp /tmp/vsql_mem_limit_cmds_verify.sh $POD_NAME:/home/dbadmin/vsql_mem_limit_cmds_verify.sh - kubectl exec $POD_NAME -i -- sh /home/dbadmin/vsql_mem_limit_cmds_verify.sh ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-memory-limit - labels: - stern: include -spec: - restartPolicy: Never - containers: - - name: test - image: quay.io/helmpack/chart-testing:v3.3.1 - command: ["/bin/entrypoint.sh"] - volumeMounts: - - name: entrypoint-volume - mountPath: /bin/entrypoint.sh - readOnly: true - subPath: entrypoint.sh - volumes: - - name: entrypoint-volume - configMap: - defaultMode: 0700 - name: script-verify-memory-limit diff --git a/tests/e2e-udx/udx-java/40-assert.yaml b/tests/e2e-udx/udx-java/40-assert.yaml deleted file mode 100644 index 53e3ed40f..000000000 --- a/tests/e2e-udx/udx-java/40-assert.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-spread-encryption -status: - containerStatuses: - - name: test - state: - terminated: - exitCode: 0 diff --git a/tests/e2e-udx/udx-java/40-verify-spread-encryption.yaml b/tests/e2e-udx/udx-java/40-verify-spread-encryption.yaml deleted file mode 100644 index 0cdce22d2..000000000 --- a/tests/e2e-udx/udx-java/40-verify-spread-encryption.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ConfigMap -metadata: - name: script-verify-spread-encryption -data: - entrypoint.sh: |- - #!/bin/bash - set -o xtrace - set -o errexit - - SEC_CHK=$(kubectl exec svc/v-udx-java-sc1 -- vsql -c "SELECT SECURITY_CONFIG_CHECK('NETWORK');") - echo $SEC_CHK - echo $SEC_CHK | grep -cq 'Spread encryption is enabled' ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-spread-encryption - labels: - stern: include -spec: - restartPolicy: Never - containers: - - name: test - image: quay.io/helmpack/chart-testing:v3.3.1 - command: ["/bin/entrypoint.sh"] - volumeMounts: - - name: entrypoint-volume - mountPath: /bin/entrypoint.sh - readOnly: true - subPath: entrypoint.sh - volumes: - - name: entrypoint-volume - configMap: - defaultMode: 0700 - name: script-verify-spread-encryption diff --git a/tests/e2e-udx/udx-java/45-assert.yaml b/tests/e2e-udx/udx-java/45-assert.yaml deleted file mode 100644 index 8ebd5474e..000000000 --- a/tests/e2e-udx/udx-java/45-assert.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-vstack -status: - containerStatuses: - - name: test - state: - terminated: - exitCode: 0 diff --git a/tests/e2e-udx/udx-java/45-upload-files.yaml b/tests/e2e-udx/udx-java/45-upload-files.yaml new file mode 100644 index 000000000..dd8227101 --- /dev/null +++ b/tests/e2e-udx/udx-java/45-upload-files.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl -n $NAMESPACE cp ../../../bin/udx-libs/build v-udx-java-sc1-0:/opt/vertica/sdk/examples + - command: kubectl -n $NAMESPACE cp ./expected-outputs v-udx-java-sc1-0:/opt/vertica/sdk/examples + - command: kubectl -n $NAMESPACE exec v-udx-java-sc1-0 -- ls /opt/vertica/sdk/examples + - command: kubectl -n $NAMESPACE exec v-udx-java-sc1-0 -- ls /opt/vertica/sdk/examples/build + # - command: kubectl -n $NAMESPACE cp run-vdb-gen.sh v-vdb-gen-sc1-0:/tmp/run-vdb-gen.sh + # - command: kubectl -n $NAMESPACE exec v-vdb-gen-sc1-0 -- chmod +x /tmp/run-vdb-gen.sh + # - script: sh -c "kubectl -n $NAMESPACE exec v-vdb-gen-sc1-0 -- sh -c \"/tmp/run-vdb-gen.sh $NAMESPACE $(cd ../../.. && make echo-images | grep ^VERTICA_IMG= | cut -d'=' -f2) $(kubectl get cm e2e -n $NAMESPACE -o jsonpath='{.data.caFileSecretName}')\" > /tmp/$NAMESPACE-vdb-gen.yaml" + # - command: cat /tmp/$NAMESPACE-vdb-gen.yaml + # # Apply the generated CR + # - command: kubectl -n $NAMESPACE apply -f /tmp/$NAMESPACE-vdb-gen.yaml + # - command: rm /tmp/$NAMESPACE-vdb-gen.yaml \ No newline at end of file diff --git a/tests/e2e-udx/udx-java/45-verify-vstack.yaml b/tests/e2e-udx/udx-java/45-verify-vstack.yaml deleted file mode 100644 index cc1a08d68..000000000 --- a/tests/e2e-udx/udx-java/45-verify-vstack.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ConfigMap -metadata: - name: script-verify-vstack -data: - entrypoint.sh: |- - #!/bin/bash - set -o xtrace - set -o errexit - - VSTACK_OP=$(kubectl exec svc/v-udx-java-sc1 -- sudo /opt/vertica/bin/vstack) - echo $VSTACK_OP - echo $VSTACK_OP | grep -cq 'Thread' ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-vstack - labels: - stern: include -spec: - restartPolicy: Never - containers: - - name: test - image: quay.io/helmpack/chart-testing:v3.3.1 - command: ["/bin/entrypoint.sh"] - volumeMounts: - - name: entrypoint-volume - mountPath: /bin/entrypoint.sh - readOnly: true - subPath: entrypoint.sh - volumes: - - name: entrypoint-volume - configMap: - defaultMode: 0700 - name: script-verify-vstack diff --git a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml index 67d31a27b..f6f12030c 100644 --- a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml +++ b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml @@ -25,21 +25,19 @@ data: SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") echo $POD_NAME - # UDX_OP=$(kubectl exec svc/v-udx-java-sc1 -i -- sh -c "cd /bin/UDx-Examples/build; ls -al;") - # kubectl exec $POD_NAME -i -- bash -c "sudo mkdir -p /home/dbadmin/host/UDx-Examples/" - # kubectl cp /bin/UDx-Examples/ $POD_NAME:/home/dbadmin/host/ UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "vsql -U dbadmin -c \"SELECT SET_CONFIG_PARAMETER('JavaSideProcessMinHeapSizeMB',2048);\"") echo "$UDX_OP" - UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /host/UDx-Examples; vsql -U dbadmin -f Java-and-C++/JavaFunctions.sql") + UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples; vsql -U dbadmin -f JavaFunctions.sql 2>&1") echo "$UDX_OP" - echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 2 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 3 && /CREATE FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 4 && /CREATE FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 5 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 6 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 7 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 8 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' + kubectl exec $POD_NAME -i -- bash -c "echo \"$UDX_OP\" | diff - /opt/vertica/sdk/examples/expected-outputs/JavaFunctionsOut.txt" + # echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 2 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 3 && /CREATE FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 4 && /CREATE FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 5 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 6 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 7 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 8 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' --- apiVersion: v1 kind: Pod diff --git a/tests/e2e-udx/udx-java/51-verify-java-uda.yaml b/tests/e2e-udx/udx-java/51-verify-java-uda.yaml index 245d0b3c3..4151a832b 100644 --- a/tests/e2e-udx/udx-java/51-verify-java-uda.yaml +++ b/tests/e2e-udx/udx-java/51-verify-java-uda.yaml @@ -25,15 +25,16 @@ data: SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") echo $POD_NAME - # UDX_OP=$(kubectl exec svc/v-udx-java-sc1 -i -- sh -c "cd /bin/UDx-Examples/build; ls -al;") - # kubectl exec $POD_NAME -i -- bash -c "sudo mkdir -p /home/dbadmin/host/UDx-Examples/" - # kubectl cp /bin/UDx-Examples/ $POD_NAME:/home/dbadmin/host/ - UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /host/UDx-Examples; vsql -U dbadmin -f Java-and-C++/JavaUDAnalytics.sql") + UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples; vsql -U dbadmin -f JavaUDAnalytics.sql 2>&1") echo "$UDX_OP" - echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 2 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 3 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 4 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' + # the row orders in the output might be unstable, so sort the entire output and then compare + kubectl exec $POD_NAME -i -- bash -c "echo \"$UDX_OP\" | diff - /opt/vertica/sdk/examples/expected-outputs/JavaUDAnalyticsOut.txt" \ + || kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples/expected-outputs; sort ./JavaUDAnalyticsOut.txt > ./JavaUDAnalyticsOutSorted.txt; \ + echo \"$UDX_OP\" | sort | diff - /opt/vertica/sdk/examples/expected-outputs/JavaUDAnalyticsOutSorted.txt" + # echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 2 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 3 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 4 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' --- apiVersion: v1 kind: Pod diff --git a/tests/e2e-udx/udx-java/52-verify-java-udl.yaml b/tests/e2e-udx/udx-java/52-verify-java-udl.yaml index 52fcb2028..ba1cf5a5d 100644 --- a/tests/e2e-udx/udx-java/52-verify-java-udl.yaml +++ b/tests/e2e-udx/udx-java/52-verify-java-udl.yaml @@ -25,16 +25,14 @@ data: SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") echo $POD_NAME - # UDX_OP=$(kubectl exec svc/v-udx-java-sc1 -i -- sh -c "cd /bin/UDx-Examples/build; ls -al;") - # kubectl exec $POD_NAME -i -- bash -c "sudo mkdir -p /home/dbadmin/host/UDx-Examples/" - # kubectl cp /bin/UDx-Examples/ $POD_NAME:/home/dbadmin/host/ - UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /host/UDx-Examples; vsql -U dbadmin -f Java-and-C++/JavaUDLFunctions.sql") + UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples; vsql -U dbadmin -f JavaUDLFunctions.sql 2>&1") echo "$UDX_OP" - echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 2 && /CREATE SOURCE FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 3 && /CREATE PARSER FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 4 && /CREATE PARSER FUNCTION/ {print}' | wc -l | grep -cq '1' - echo "$UDX_OP" | awk 'FNR == 5 && /CREATE FILTER FUNCTION/ {print}' | wc -l | grep -cq '1' + kubectl exec $POD_NAME -i -- bash -c "echo \"$UDX_OP\" | diff - /opt/vertica/sdk/examples/expected-outputs/JavaUDLFunctionsOut.txt" + # echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 2 && /CREATE SOURCE FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 3 && /CREATE PARSER FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 4 && /CREATE PARSER FUNCTION/ {print}' | wc -l | grep -cq '1' + # echo "$UDX_OP" | awk 'FNR == 5 && /CREATE FILTER FUNCTION/ {print}' | wc -l | grep -cq '1' --- apiVersion: v1 kind: Pod diff --git a/tests/e2e-udx/udx-java/expected-outputs/JavaFunctionsOut.txt b/tests/e2e-udx/udx-java/expected-outputs/JavaFunctionsOut.txt new file mode 100644 index 000000000..2e149ed88 --- /dev/null +++ b/tests/e2e-udx/udx-java/expected-outputs/JavaFunctionsOut.txt @@ -0,0 +1,125 @@ +CREATE LIBRARY +CREATE LIBRARY +CREATE FUNCTION +CREATE FUNCTION +CREATE TRANSFORM FUNCTION +CREATE TRANSFORM FUNCTION +CREATE TRANSFORM FUNCTION +CREATE TRANSFORM FUNCTION +CREATE TABLE +CREATE TABLE + c1 | c2 | add2ints +----+----+---------- + 1 | 2 | 3 + 3 | 2 | 5 + 5 | 2 | 7 + 1 | 4 | 5 + 2 | 2 | 4 +(5 rows) + + c1 | c2 | addanyints +----+----+------------ + 1 | 2 | 3 + 3 | 2 | 5 + 5 | 2 | 7 + 1 | 4 | 5 + 2 | 2 | 4 +(5 rows) + + c1 | c2 | c3 | addanyints +----+----+----+------------ + 1 | 2 | 3 | 6 + 3 | 2 | 9 | 14 + 5 | 2 | 8 | 15 + 1 | 4 | 5 | 10 + 2 | 2 | 5 | 9 +(5 rows) + + col1 | col2 +------+------ + 1 | 2 + 1 | 4 +(2 rows) + + col0 | col1 +------+------ + 1 | 2 + 1 | 4 +(2 rows) + + col0 | col1 +------+------ + 1 | 2 +(1 row) + + col0 | col1 +------+------ + 1 | 2 + 1 | 4 +(2 rows) + + col0 | col1 +------+------ + a | b + a | c +(2 rows) + + col0 +------ + 2 + 2 +(2 rows) + +CREATE TABLE + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 1 +(1 row) + +COMMIT +CREATE TABLE + doc_id | text +--------+------------------------------------------------------------------------------------------ + 100 | Vertica Analytic database + 300 | Vertica in database analytic functions combined with UD functions include many UD things +(2 rows) + + doc_id | text +--------+------------------------------------------------------------------------------------------ + 300 | Vertica in database analytic functions combined with UD functions include many UD things +(1 row) + + doc_id | text +--------+------------------------------------------------------------------------------------------------------------------------- + 200 | Analytic database functions with User Defined Functions UD functions for short + 300 | Vertica in database analytic functions combined with UD functions include many UD things + 400 | UDx Framework include UD scalar functions analytic functions UD single and multiphase transforms UD aggregates UD Loads +(3 rows) + + doc_id | text | score +--------+-------------------------------------------------------------------------------------------------------------------------+------------------- + 400 | UDx Framework include UD scalar functions analytic functions UD single and multiphase transforms UD aggregates UD Loads | 1.15072828980712 + 300 | Vertica in database analytic functions combined with UD functions include many UD things | 0.575364144903562 +(2 rows) + +DROP TABLE +DROP TABLE +DROP TABLE +DROP TABLE +DROP LIBRARY +DROP LIBRARY diff --git a/tests/e2e-udx/udx-java/expected-outputs/JavaUDAnalyticsOut.txt b/tests/e2e-udx/udx-java/expected-outputs/JavaUDAnalyticsOut.txt new file mode 100644 index 000000000..52baf344a --- /dev/null +++ b/tests/e2e-udx/udx-java/expected-outputs/JavaUDAnalyticsOut.txt @@ -0,0 +1,59 @@ +CREATE LIBRARY +CREATE ANALYTIC FUNCTION +CREATE ANALYTIC FUNCTION +CREATE ANALYTIC FUNCTION +CREATE TABLE + x | y | z | an_rank +---+----+---+--------- + 3 | 9 | 1 | 1 + 3 | 8 | 2 | 2 + 3 | 8 | 3 | 3 + 2 | 7 | 4 | 4 + 2 | 7 | 5 | 5 + 2 | 9 | 6 | 6 + 1 | 11 | 7 | 7 + 1 | 10 | 8 | 8 + 1 | 10 | 9 | 9 +(9 rows) + + x | y | z | an_rank +---+----+---+--------- + 1 | 11 | 7 | 1 + 3 | 9 | 1 | 1 + 1 | 10 | 8 | 1 + 1 | 10 | 9 | 2 + 3 | 8 | 2 | 1 + 3 | 8 | 3 | 2 + 2 | 9 | 6 | 1 + 2 | 7 | 4 | 1 + 2 | 7 | 5 | 2 +(9 rows) + + z | an_lag +---+-------- + 1 | + 2 | + 3 | 1 + 4 | 2 + 5 | 3 + 6 | 4 + 7 | 5 + 8 | 6 + 9 | 7 +(9 rows) + + z | an_lead +---+--------- + 1 | 3 + 2 | 4 + 3 | 5 + 4 | 6 + 5 | 7 + 6 | 8 + 7 | 9 + 8 | + 9 | +(9 rows) + +DROP TABLE +DROP LIBRARY diff --git a/tests/e2e-udx/udx-java/expected-outputs/JavaUDLFunctionsOut.txt b/tests/e2e-udx/udx-java/expected-outputs/JavaUDLFunctionsOut.txt new file mode 100644 index 000000000..9bfb451ef --- /dev/null +++ b/tests/e2e-udx/udx-java/expected-outputs/JavaUDLFunctionsOut.txt @@ -0,0 +1,63 @@ +CREATE LIBRARY +CREATE SOURCE FUNCTION +CREATE PARSER FUNCTION +CREATE PARSER FUNCTION +CREATE FILTER FUNCTION +CREATE TABLE + Rows Loaded +------------- + 10 +(1 row) + + i +--- + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 +(10 rows) + +TRUNCATE TABLE + i +--- + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 +(10 rows) + +TRUNCATE TABLE + i +--- + 2 + 2 + 2 + 2 + 2 +(5 rows) + +TRUNCATE TABLE + Rows Loaded +------------- + 10000000 +(1 row) + + count | count | avg +----------+----------+----------- + 10000000 | 10000000 | 4999999.5 +(1 row) + +DROP TABLE +DROP LIBRARY diff --git a/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml b/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml index 547eceecc..96f9d0c74 100644 --- a/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml +++ b/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml @@ -25,14 +25,14 @@ spec: encryptSpreadComm: vertica subclusters: - name: sc1 - size: 2 + size: 1 kSafety: "0" certSecrets: [] imagePullSecrets: [] - volumes: - - name: downloads - hostPath: - path: /host - volumeMounts: - - mountPath: /host - name: downloads + # volumes: + # - name: downloads + # hostPath: + # path: /host + # volumeMounts: + # - mountPath: /host + # name: downloads From a757f360d6d9d3b6b1415955aeda7483ff1cfbe8 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Tue, 16 Aug 2022 11:41:07 -0400 Subject: [PATCH 10/30] revert types --- api/v1beta1/verticadb_types.go | 47 ---------------------------------- 1 file changed, 47 deletions(-) diff --git a/api/v1beta1/verticadb_types.go b/api/v1beta1/verticadb_types.go index c456c2c2e..03e2da019 100644 --- a/api/v1beta1/verticadb_types.go +++ b/api/v1beta1/verticadb_types.go @@ -303,17 +303,6 @@ type VerticaDBSpec struct { // no effect. EncryptSpreadComm string `json:"encryptSpreadComm,omitempty"` - // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" - // Descriptors of the init containers as well as corresponding volumes that - // will be used to prepare runtime environments for programs in the main - // container, e.g. UDx language runtimes. Multiple init containers may be - // specified and will be started in the same order as their descriptors. - // Multiple (emptyDir) volumes may be specified for each init container, - // and each volume may be mounted at different locations in the init - // container as well as the main container. - Init []InitDescriptor `json:"init,omitempty"` - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:hidden" // Allows you to set any additional securityContext for the Vertica server // container. We merge the values with the securityContext generated by the @@ -323,42 +312,6 @@ type VerticaDBSpec struct { SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` } -// InitDescriptor is used to provide specifications for an init container -// which prepares runtime environments for programs inside the main container. -type InitDescriptor struct { - // +kubebuilder:validation:Required - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" - // The name of the init container, must be unique within the Pod. - Name string `json:"name,omitempty"` - // +kubebuilder:validation:Required - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" - // The Docker image used for the init container. - Image string `json:"image,omitempty"` - // +kubebuilder:validation:Required - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" - // The commands that will be executed by "/bin/sh" when the init - // container starts. - Command string `json:"command,omitempty"` - // +kubebuilder:validation:Required - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" - // A list of pairs of paths that specify mount points of volumes attached - // to the init container. - Paths []InitPaths `json:"paths,omitempty"` -} - -// InitPaths defines a pair of paths which are mount points of a volume -// in an init container and the main container respectively. -type InitPaths struct { - // +kubebuilder:validation:Required - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" - // The path inside the init container where a volume is mounted - InitPath string `json:"initPath,omitempty"` - // +kubebuilder:validation:Required - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" - // The path inside the main container where a volume is mounted - ServerPath string `json:"serverPath,omitempty"` -} - // LocalObjectReference is used instead of corev1.LocalObjectReference and behaves the same. // This is useful for the Openshift web console. This structure is used in some // VerticaDB spec fields to define a list of secrets but, with the k8s', From 2566d882cc1a9f12e63e9a2944c28324a61150b7 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Tue, 16 Aug 2022 11:43:54 -0400 Subject: [PATCH 11/30] revert builder --- pkg/builder/builder.go | 72 ++---------------------------------------- 1 file changed, 2 insertions(+), 70 deletions(-) diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index cb293d94c..032b0f190 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -3,7 +3,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); You may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +20,6 @@ import ( "path/filepath" "regexp" "sort" - "strconv" "strings" vapi "github.com/vertica/vertica-kubernetes/api/v1beta1" @@ -116,12 +117,6 @@ func buildVolumeMounts(vdb *vapi.VerticaDB) []corev1.VolumeMount { volMnts = append(volMnts, buildCertSecretVolumeMounts(vdb)...) volMnts = append(volMnts, vdb.Spec.VolumeMounts...) - if len(vdb.Spec.Init) > 0 { - for _, ic := range vdb.Spec.Init { - volMnts = append(volMnts, buildMainContainerUDXVolumeMounts(ic)...) - } - } - return volMnts } @@ -187,11 +182,6 @@ func buildVolumes(vdb *vapi.VerticaDB) []corev1.Volume { } vols = append(vols, buildCertSecretVolumes(vdb)...) vols = append(vols, vdb.Spec.Volumes...) - if len(vdb.Spec.Init) > 0 { - for _, ic := range vdb.Spec.Init { - vols = append(vols, buildUDXVolumes(ic)...) - } - } return vols } @@ -385,70 +375,12 @@ func buildPodSpec(vdb *vapi.VerticaDB, sc *vapi.Subcluster, saName string) corev Tolerations: sc.Tolerations, ImagePullSecrets: GetK8sLocalObjectReferenceArray(vdb.Spec.ImagePullSecrets), Containers: makeContainers(vdb, sc), - InitContainers: makeInitContainers(vdb), Volumes: buildVolumes(vdb), TerminationGracePeriodSeconds: &termGracePeriod, ServiceAccountName: saName, } } -func makeInitContainers(vdb *vapi.VerticaDB) []corev1.Container { - var initContainers []corev1.Container - for _, ic := range vdb.Spec.Init { - initContainers = append(initContainers, corev1.Container{ - Image: ic.Image, - ImagePullPolicy: corev1.PullIfNotPresent, - Name: ic.Name, - VolumeMounts: buildInitContainerUDXVolumeMounts(ic), - Command: []string{"/bin/sh"}, - Args: []string{"-c", ic.Command}, - }) - } - return initContainers -} - -func getInitContainerVolumeName(ic vapi.InitDescriptor, idx int) string { - return ic.Name + "-volume-" + strconv.Itoa(idx) -} - -func buildUDXVolumeMounts(ic vapi.InitDescriptor, where string) []corev1.VolumeMount { - var volumeMounts []corev1.VolumeMount - for idx, pathPair := range ic.Paths { - var mountPath string - if where == "initContainer" { - mountPath = pathPair.InitPath - } else if where == "mainContainer" { - mountPath = pathPair.ServerPath - } - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: getInitContainerVolumeName(ic, idx), - MountPath: mountPath, - }) - } - return volumeMounts -} - -func buildUDXVolumes(ic vapi.InitDescriptor) []corev1.Volume { - var volumes []corev1.Volume - for idx := range ic.Paths { - volumes = append(volumes, corev1.Volume{ - Name: getInitContainerVolumeName(ic, idx), - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }) - } - return volumes -} - -func buildInitContainerUDXVolumeMounts(ic vapi.InitDescriptor) []corev1.VolumeMount { - return buildUDXVolumeMounts(ic, "initContainer") -} - -func buildMainContainerUDXVolumeMounts(ic vapi.InitDescriptor) []corev1.VolumeMount { - return buildUDXVolumeMounts(ic, "mainContainer") -} - // makeServerContainer builds the spec for the server container func makeServerContainer(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.Container { envVars := translateAnnotationsToEnvVars(vdb) From e7f47d22b33bea6761ae02871441b865017df4f4 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Tue, 16 Aug 2022 11:52:51 -0400 Subject: [PATCH 12/30] discard docker-udx-java --- docker-udx-java/Dockerfile | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 docker-udx-java/Dockerfile diff --git a/docker-udx-java/Dockerfile b/docker-udx-java/Dockerfile deleted file mode 100644 index d68f33218..000000000 --- a/docker-udx-java/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM phusion/baseimage:focal-1.2.0 as downloader - -CMD ["/sbin/my_init"] - -WORKDIR /tmp/ - -# ---- Option 1: download from web - -RUN set -x \ -&& apt-get update \ -&& apt-get install wget \ -&& wget https://download.java.net/openjdk/jdk8u41/ri/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz \ -&& tar xzvf openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz - -# ---- Option 2: copy from local - -# COPY ./java-se-8u41-ri /tmp/ - -############################################################################################## -FROM alpine:latest - -COPY --from=downloader /tmp/java-se-8u41-ri/ /tmp/java-se-8u41-ri/ - -CMD ["/bin/sh"] \ No newline at end of file From 716c2c3aada3cb0ba29af4a360a131de667edfef Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Tue, 16 Aug 2022 11:59:30 -0400 Subject: [PATCH 13/30] revert Makefile --- Makefile | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index d850d4064..0bf970357 100644 --- a/Makefile +++ b/Makefile @@ -84,10 +84,6 @@ export OPERATOR_IMG # Image URL to use for building/pushing of the vertica server VERTICA_IMG ?= $(IMG_REPO)vertica-k8s:$(TAG) export VERTICA_IMG -# Image URL to use for building/pushing of the java udx init container -UDX_JAVA_IMG ?= $(IMG_REPO)udx-java:$(TAG) -export UDX_JAVA_IMG - # This is the base image to use for some upgrade tests. We will always # upgrade to VERTICA_IMG, so BASE_VERTICA_IMG must be some image from a # version earlier than VERTICA_IMG. @@ -336,6 +332,7 @@ docker-build-vertica: docker-vertica/Dockerfile ## Build vertica server docker i cd docker-vertica \ && make VERTICA_IMG=${VERTICA_IMG} MINIMAL_VERTICA_IMG=${MINIMAL_VERTICA_IMG} +.PHONY: docker-push docker-push-vertica: ## Push vertica server docker image ifeq ($(shell $(KIND_CHECK)), 0) docker push ${VERTICA_IMG} @@ -343,17 +340,6 @@ else scripts/push-to-kind.sh -i ${VERTICA_IMG} endif -.PHONY: docker-build-udx-java -docker-build-udx-java: docker-udx-java/Dockerfile ## Build docker image for java udx init container - docker build -t ${UDX_JAVA_IMG} -f docker-udx-java/Dockerfile docker-udx-java - -docker-push-udx-java: ## Push docker image for java udx init container -ifeq ($(shell $(KIND_CHECK)), 0) - docker push ${UDX_JAVA_IMG} -else - scripts/push-to-kind.sh -i ${UDX_JAVA_IMG} -endif - .PHONY: bundle bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. ifneq ($(DEPLOY_WITH), $(filter $(DEPLOY_WITH), olm random)) @@ -377,9 +363,9 @@ docker-build-olm-catalog: opm ## Build an OLM catalog that includes our bundle ( docker-push-olm-catalog: docker push $(OLM_CATALOG_IMG) -docker-build: docker-build-vertica docker-build-operator docker-build-vlogger docker-build-udx-java docker-build-bundle ## Build all docker images except OLM catalog +docker-build: docker-build-vertica docker-build-operator docker-build-vlogger docker-build-bundle ## Build all docker images except OLM catalog -docker-push: docker-push-vertica docker-push-operator docker-push-vlogger docker-push-udx-java docker-push-bundle ## Push all docker images except OLM catalog +docker-push: docker-push-vertica docker-push-operator docker-push-vlogger docker-push-bundle ## Push all docker images except OLM catalog echo-images: ## Print the names of all of the images used @echo "OPERATOR_IMG=$(OPERATOR_IMG)" From fc84201a73eefe9a39165641f08f8f226de025f1 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Tue, 16 Aug 2022 12:09:02 -0400 Subject: [PATCH 14/30] revert .gitignore --- .gitignore | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.gitignore b/.gitignore index 2fb791d12..68dfb17f3 100644 --- a/.gitignore +++ b/.gitignore @@ -68,12 +68,3 @@ config/release-manifests # bundle files are all auto generated, so omit them docker-bundle/Dockerfile bundle/ - - -docker-udx-java/java-se-8u41-ri -docker-udx-java/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz - -docker-vertica-udx-java - -tests/e2e-udx/*/UDx-Examples -tests/e2e-udx/udx-java/examples \ No newline at end of file From 75b3391051b5f2d6002d08abb103a82a94184636 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Tue, 16 Aug 2022 12:31:51 -0400 Subject: [PATCH 15/30] cleanup Dockerfile --- docker-vertica/Dockerfile | 6 ++---- docker-vertica/dbadmin/.bash_profile | 1 - tests/e2e-udx/udx-java/45-upload-files.yaml | 13 ++++--------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/docker-vertica/Dockerfile b/docker-vertica/Dockerfile index c8aa5297c..430199588 100644 --- a/docker-vertica/Dockerfile +++ b/docker-vertica/Dockerfile @@ -136,7 +136,7 @@ RUN set -x \ sudo \ && rm -rf /var/lib/apt/lists/* RUN set -x \ - # Install jre if not minimal and not using init container + # Install jre if not minimal && if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") ]] ; then \ apt-get -y update \ && apt-get install -y --no-install-recommends $JRE_PKG \ @@ -153,14 +153,12 @@ RUN set -x \ # Allow passwordless sudo access from dbadmin && echo "dbadmin ALL=(ALL) NOPASSWD: ALL" | tee -a /etc/sudoers \ && echo "dbadmin - nofile 65536" >> /etc/security/limits.conf \ - # Set JAVA_HOME environment variable if not minimal + # Set JAVA_HOME environment variable if not minimal, this will be loaded to all shells && if [[ $MINIMAL != "YES" && $MINIMAL != "yes" ]]; then \ - # echo "export JAVA_HOME=/usr" >> /home/dbadmin/.bashrc; \ echo "JAVA_HOME=/usr" >> /etc/environment; \ fi ENTRYPOINT [ "/usr/local/bin/docker-entrypoint.sh" ] -# ENTRYPOINT "MINIMAL=${MINIMAL} USING_INIT=${USING_INIT} /usr/local/bin/docker-entrypoint.sh" # vertica port EXPOSE 5433 diff --git a/docker-vertica/dbadmin/.bash_profile b/docker-vertica/dbadmin/.bash_profile index a18d4a051..13f594aef 100644 --- a/docker-vertica/dbadmin/.bash_profile +++ b/docker-vertica/dbadmin/.bash_profile @@ -12,4 +12,3 @@ export PATH export LANG=en_US.UTF-8 export TZ=UTC -# export JAVA_HOME=/usr diff --git a/tests/e2e-udx/udx-java/45-upload-files.yaml b/tests/e2e-udx/udx-java/45-upload-files.yaml index dd8227101..05e1f81a9 100644 --- a/tests/e2e-udx/udx-java/45-upload-files.yaml +++ b/tests/e2e-udx/udx-java/45-upload-files.yaml @@ -1,14 +1,9 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: + # upload the built library binaries - command: kubectl -n $NAMESPACE cp ../../../bin/udx-libs/build v-udx-java-sc1-0:/opt/vertica/sdk/examples + # upload the expected outputs of executing sql files - command: kubectl -n $NAMESPACE cp ./expected-outputs v-udx-java-sc1-0:/opt/vertica/sdk/examples - - command: kubectl -n $NAMESPACE exec v-udx-java-sc1-0 -- ls /opt/vertica/sdk/examples - - command: kubectl -n $NAMESPACE exec v-udx-java-sc1-0 -- ls /opt/vertica/sdk/examples/build - # - command: kubectl -n $NAMESPACE cp run-vdb-gen.sh v-vdb-gen-sc1-0:/tmp/run-vdb-gen.sh - # - command: kubectl -n $NAMESPACE exec v-vdb-gen-sc1-0 -- chmod +x /tmp/run-vdb-gen.sh - # - script: sh -c "kubectl -n $NAMESPACE exec v-vdb-gen-sc1-0 -- sh -c \"/tmp/run-vdb-gen.sh $NAMESPACE $(cd ../../.. && make echo-images | grep ^VERTICA_IMG= | cut -d'=' -f2) $(kubectl get cm e2e -n $NAMESPACE -o jsonpath='{.data.caFileSecretName}')\" > /tmp/$NAMESPACE-vdb-gen.yaml" - # - command: cat /tmp/$NAMESPACE-vdb-gen.yaml - # # Apply the generated CR - # - command: kubectl -n $NAMESPACE apply -f /tmp/$NAMESPACE-vdb-gen.yaml - # - command: rm /tmp/$NAMESPACE-vdb-gen.yaml \ No newline at end of file + # - command: kubectl -n $NAMESPACE exec v-udx-java-sc1-0 -- ls /opt/vertica/sdk/examples + # - command: kubectl -n $NAMESPACE exec v-udx-java-sc1-0 -- ls /opt/vertica/sdk/examples/build \ No newline at end of file From 859fae19b2ea768e633a453ec9297877071fc995 Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Tue, 16 Aug 2022 12:33:16 -0400 Subject: [PATCH 16/30] install python symlink in entrypoint --- docker-vertica/docker-entrypoint.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docker-vertica/docker-entrypoint.sh b/docker-vertica/docker-entrypoint.sh index b3ae76705..db953b1e4 100755 --- a/docker-vertica/docker-entrypoint.sh +++ b/docker-vertica/docker-entrypoint.sh @@ -34,12 +34,7 @@ copy_config_files echo "Vertica container is now running" -# if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") \ -# && ($USING_INIT == "YES" || $USING_INIT == "yes") ]]; then -# sudo mkdir -p /usr/lib/jvm -# sudo cp -r /opt/jvm /usr/lib/ -# # echo $(ls /usr/lib) -# sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-se-8u41-ri/bin/java 1 -# fi +# Create a symlink to python3 interpreter in vertica +sudo update-alternatives --install /usr/bin/python python /opt/vertica/oss/python3/bin/python3 1 sudo /usr/sbin/sshd -D From 6c1db2269428321bbdf65c52553599071cbe629c Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Wed, 17 Aug 2022 13:53:37 -0400 Subject: [PATCH 17/30] delete unnecessary tests --- tests/e2e-udx/udx-java/30-assert.yaml | 27 --------- .../udx-java/30-verify-vsql-access.yaml | 20 ------- .../50-verify-java-scalar-transform.yaml | 8 --- .../e2e-udx/udx-java/51-verify-java-uda.yaml | 4 -- .../e2e-udx/udx-java/52-verify-java-udl.yaml | 5 -- .../base/kustomization.yaml | 15 ----- .../base/verify-vsql-access.yaml | 57 ------------------- 7 files changed, 136 deletions(-) delete mode 100644 tests/e2e-udx/udx-java/30-assert.yaml delete mode 100644 tests/e2e-udx/udx-java/30-verify-vsql-access.yaml delete mode 100644 tests/e2e-udx/udx-java/verify-vsql-access/base/kustomization.yaml delete mode 100644 tests/e2e-udx/udx-java/verify-vsql-access/base/verify-vsql-access.yaml diff --git a/tests/e2e-udx/udx-java/30-assert.yaml b/tests/e2e-udx/udx-java/30-assert.yaml deleted file mode 100644 index b1446b7c1..000000000 --- a/tests/e2e-udx/udx-java/30-assert.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 180 ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-udx-java -status: - containerStatuses: - - name: test - state: - terminated: - exitCode: 0 diff --git a/tests/e2e-udx/udx-java/30-verify-vsql-access.yaml b/tests/e2e-udx/udx-java/30-verify-vsql-access.yaml deleted file mode 100644 index b2c5e44ef..000000000 --- a/tests/e2e-udx/udx-java/30-verify-vsql-access.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Verifies access to the external service through port 5433. It does this -# by connecting using vsql - -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - command: bash -c "kustomize build verify-vsql-access/overlay | kubectl -n $NAMESPACE apply -f - " diff --git a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml index f6f12030c..9331740aa 100644 --- a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml +++ b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml @@ -30,14 +30,6 @@ data: UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples; vsql -U dbadmin -f JavaFunctions.sql 2>&1") echo "$UDX_OP" kubectl exec $POD_NAME -i -- bash -c "echo \"$UDX_OP\" | diff - /opt/vertica/sdk/examples/expected-outputs/JavaFunctionsOut.txt" - # echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 2 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 3 && /CREATE FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 4 && /CREATE FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 5 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 6 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 7 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 8 && /CREATE TRANSFORM FUNCTION/ {print}' | wc -l | grep -cq '1' --- apiVersion: v1 kind: Pod diff --git a/tests/e2e-udx/udx-java/51-verify-java-uda.yaml b/tests/e2e-udx/udx-java/51-verify-java-uda.yaml index 4151a832b..1a6aeabd0 100644 --- a/tests/e2e-udx/udx-java/51-verify-java-uda.yaml +++ b/tests/e2e-udx/udx-java/51-verify-java-uda.yaml @@ -31,10 +31,6 @@ data: kubectl exec $POD_NAME -i -- bash -c "echo \"$UDX_OP\" | diff - /opt/vertica/sdk/examples/expected-outputs/JavaUDAnalyticsOut.txt" \ || kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples/expected-outputs; sort ./JavaUDAnalyticsOut.txt > ./JavaUDAnalyticsOutSorted.txt; \ echo \"$UDX_OP\" | sort | diff - /opt/vertica/sdk/examples/expected-outputs/JavaUDAnalyticsOutSorted.txt" - # echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 2 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 3 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 4 && /CREATE ANALYTIC FUNCTION/ {print}' | wc -l | grep -cq '1' --- apiVersion: v1 kind: Pod diff --git a/tests/e2e-udx/udx-java/52-verify-java-udl.yaml b/tests/e2e-udx/udx-java/52-verify-java-udl.yaml index ba1cf5a5d..29c40fd8f 100644 --- a/tests/e2e-udx/udx-java/52-verify-java-udl.yaml +++ b/tests/e2e-udx/udx-java/52-verify-java-udl.yaml @@ -28,11 +28,6 @@ data: UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples; vsql -U dbadmin -f JavaUDLFunctions.sql 2>&1") echo "$UDX_OP" kubectl exec $POD_NAME -i -- bash -c "echo \"$UDX_OP\" | diff - /opt/vertica/sdk/examples/expected-outputs/JavaUDLFunctionsOut.txt" - # echo "$UDX_OP" | awk 'FNR == 1 && /CREATE LIBRARY/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 2 && /CREATE SOURCE FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 3 && /CREATE PARSER FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 4 && /CREATE PARSER FUNCTION/ {print}' | wc -l | grep -cq '1' - # echo "$UDX_OP" | awk 'FNR == 5 && /CREATE FILTER FUNCTION/ {print}' | wc -l | grep -cq '1' --- apiVersion: v1 kind: Pod diff --git a/tests/e2e-udx/udx-java/verify-vsql-access/base/kustomization.yaml b/tests/e2e-udx/udx-java/verify-vsql-access/base/kustomization.yaml deleted file mode 100644 index 913a739b6..000000000 --- a/tests/e2e-udx/udx-java/verify-vsql-access/base/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resources: - - verify-vsql-access.yaml diff --git a/tests/e2e-udx/udx-java/verify-vsql-access/base/verify-vsql-access.yaml b/tests/e2e-udx/udx-java/verify-vsql-access/base/verify-vsql-access.yaml deleted file mode 100644 index 781e03fff..000000000 --- a/tests/e2e-udx/udx-java/verify-vsql-access/base/verify-vsql-access.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Verifies access to the external service through port 5433. It does this -# by connecting using vsql - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: script-verify-udx-java-test -data: - test.sh: |- - #!/bin/bash - set -o errexit - set -o xtrace - - CLUSTER_IP=v-udx-java-sc1 - vsql -h $CLUSTER_IP -c "drop table if exists t1" - vsql -h $CLUSTER_IP -c "create table t1 (c1 int)" - vsql -h $CLUSTER_IP -c "insert into t1 values (1)" - vsql -h $CLUSTER_IP -c "select * from t1" - vsql -h $CLUSTER_IP -c "drop table t1" - ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-verify-udx-java - labels: - stern: include -spec: - restartPolicy: Never - containers: - - name: test - image: kustomize-vertica-image - command: ["/home/dbadmin/test.sh"] - volumeMounts: - - name: test-script - mountPath: /home/dbadmin/test.sh - readOnly: true - subPath: test.sh - volumes: - - name: test-script - configMap: - defaultMode: 0777 - name: script-verify-udx-java-test From e860bb1c958c6bacaad7145d04f7a3b6258b800e Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Wed, 17 Aug 2022 14:21:41 -0400 Subject: [PATCH 18/30] add python udx tests --- .../PythonScalarFunctionsOut.txt | 56 +++++ .../PythonTransformFunctionsOut.txt | 206 ++++++++++++++++++ .../setup-vdb/base/kustomization.yaml | 15 ++ .../udx-python/setup-vdb/base/setup-vdb.yaml | 31 +++ 4 files changed, 308 insertions(+) create mode 100644 tests/e2e-udx/udx-python/expected-outputs/PythonScalarFunctionsOut.txt create mode 100644 tests/e2e-udx/udx-python/expected-outputs/PythonTransformFunctionsOut.txt create mode 100644 tests/e2e-udx/udx-python/setup-vdb/base/kustomization.yaml create mode 100644 tests/e2e-udx/udx-python/setup-vdb/base/setup-vdb.yaml diff --git a/tests/e2e-udx/udx-python/expected-outputs/PythonScalarFunctionsOut.txt b/tests/e2e-udx/udx-python/expected-outputs/PythonScalarFunctionsOut.txt new file mode 100644 index 000000000..e53de14ca --- /dev/null +++ b/tests/e2e-udx/udx-python/expected-outputs/PythonScalarFunctionsOut.txt @@ -0,0 +1,56 @@ +CREATE LIBRARY +CREATE FUNCTION +CREATE FUNCTION +CREATE FUNCTION +CREATE FUNCTION +CREATE FUNCTION +CREATE TABLE + add2ints +---------- + 3 + 7 + 11 +(3 rows) + +DROP TABLE +CREATE TABLE + copy2letters +-------------- + pl + ve + an + ve + 9. +(5 rows) + +DROP TABLE +CREATE TABLE + add2hours +-------------------------- + 1900-12-28 15:26:13.0129 + 2000-08-06 01:01:44.0111 + 2018-04-20 02:00:00 +(3 rows) + +DROP TABLE +CREATE TABLE + add2decimals +-------------- + 12.1000 + 3.3000 + 7.7000 +(3 rows) + +DROP TABLE +CREATE TABLE + addprefix +------------- + __platform + __vertica + __analytics + __version + __9.1 +(5 rows) + +DROP TABLE +DROP LIBRARY diff --git a/tests/e2e-udx/udx-python/expected-outputs/PythonTransformFunctionsOut.txt b/tests/e2e-udx/udx-python/expected-outputs/PythonTransformFunctionsOut.txt new file mode 100644 index 000000000..e373a669f --- /dev/null +++ b/tests/e2e-udx/udx-python/expected-outputs/PythonTransformFunctionsOut.txt @@ -0,0 +1,206 @@ +CREATE LIBRARY +CREATE TRANSFORM FUNCTION +CREATE TRANSFORM FUNCTION +CREATE TABLE + tokens +-------- + Word + The + quick + brown + fox + jumped + over + the + lazy + dog +(10 rows) + +DROP TABLE +CREATE TABLE + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 1 +(1 row) + +COMMIT + url | tokens +-----------------+----------- + www.amazon.com | Online + www.amazon.com | retail + www.amazon.com | merchant + www.amazon.com | and + www.amazon.com | provider + www.amazon.com | of + www.amazon.com | cloud + www.amazon.com | services + www.hp.com | Leading + www.hp.com | provider + www.hp.com | of + www.hp.com | computer + www.hp.com | hardware + www.hp.com | and + www.hp.com | imaging + www.hp.com | solutions + www.vertica.com | World's + www.vertica.com | fastest + www.vertica.com | analytic + www.vertica.com | database +(20 rows) + +DROP TABLE +CREATE TABLE + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 2 +(1 row) + + OUTPUT +-------- + 4 +(1 row) + + a | b | c +---+---+--- + 1 | 1 | 2 + 2 | 2 | 2 + 2 | 2 | 2 + 2 | 2 | 2 + 3 | 2 | 4 +(5 rows) + + a | b | c +---+---+--- + 2 | 2 | 2 + 2 | 2 | 2 + 1 | 1 | 2 + 4 | 4 | 5 + 3 | 2 | 4 + 3 | 2 | 4 +(6 rows) + +DROP TABLE +DROP LIBRARY +CREATE TABLE +CREATE LIBRARY +CREATE TRANSFORM FUNCTION + average | ignored_rows | total_rows +---------+--------------+------------ + 2.5 | 3 | 7 +(1 row) + +DROP LIBRARY +DROP TABLE +CREATE TABLE + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 1 +(1 row) + + OUTPUT +-------- + 1 +(1 row) + +COMMIT +CREATE LIBRARY +CREATE TRANSFORM FUNCTION + term | doc_id | term_freq | corp_freq +------------+--------+-----------+----------- + Loads | 400 | 1 | + Loads | | | 1 + and | 400 | 1 | + and | | | 1 + combined | 300 | 1 | + combined | | | 1 + Defined | 200 | 1 | + Defined | | | 1 + User | 200 | 1 | + User | | | 1 + for | 200 | 1 | + for | | | 1 + in | 300 | 1 | + in | | | 1 + short | 200 | 1 | + short | | | 1 + single | 400 | 1 | + single | | | 1 + Vertica | 100 | 1 | + Vertica | 300 | 1 | + Vertica | | | 2 + scalar | 400 | 1 | + scalar | | | 1 + database | 100 | 1 | + database | 200 | 1 | + database | 300 | 1 | + database | | | 3 + include | 300 | 1 | + include | 400 | 1 | + include | | | 2 + transforms | 400 | 1 | + transforms | | | 1 + with | 200 | 1 | + with | 300 | 1 | + with | | | 2 + Framework | 400 | 1 | + Framework | | | 1 + Functions | 200 | 1 | + Functions | | | 1 + functions | 200 | 2 | + functions | 300 | 2 | + functions | 400 | 2 | + functions | | | 3 + many | 300 | 1 | + many | | | 1 + multiphase | 400 | 1 | + multiphase | | | 1 + things | 300 | 1 | + things | | | 1 + Analytic | 100 | 1 | + Analytic | 200 | 1 | + Analytic | | | 2 + UD | 200 | 1 | + UD | 300 | 2 | + UD | 400 | 4 | + UD | | | 3 + UDx | 400 | 1 | + UDx | | | 1 + aggregates | 400 | 1 | + aggregates | | | 1 + analytic | 300 | 1 | + analytic | 400 | 1 | + analytic | | | 2 +(63 rows) + +DROP LIBRARY +DROP TABLE diff --git a/tests/e2e-udx/udx-python/setup-vdb/base/kustomization.yaml b/tests/e2e-udx/udx-python/setup-vdb/base/kustomization.yaml new file mode 100644 index 000000000..01b927990 --- /dev/null +++ b/tests/e2e-udx/udx-python/setup-vdb/base/kustomization.yaml @@ -0,0 +1,15 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: + - setup-vdb.yaml diff --git a/tests/e2e-udx/udx-python/setup-vdb/base/setup-vdb.yaml b/tests/e2e-udx/udx-python/setup-vdb/base/setup-vdb.yaml new file mode 100644 index 000000000..bddb332dc --- /dev/null +++ b/tests/e2e-udx/udx-python/setup-vdb/base/setup-vdb.yaml @@ -0,0 +1,31 @@ +# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1beta1 +kind: VerticaDB +metadata: + name: v-udx-python +spec: + image: kustomize-vertica-image + communal: + includeUIDInPath: true + local: + requestSize: 100Mi + dbName: vertdb + encryptSpreadComm: vertica + subclusters: + - name: sc1 + size: 1 + kSafety: "0" + certSecrets: [] + imagePullSecrets: [] \ No newline at end of file From 652b677dad92b651989d4c175450f13d4fd7813d Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Wed, 17 Aug 2022 14:40:51 -0400 Subject: [PATCH 19/30] temporarily remove python udx tests --- .../PythonScalarFunctionsOut.txt | 56 ----- .../PythonTransformFunctionsOut.txt | 206 ------------------ .../setup-vdb/base/kustomization.yaml | 15 -- .../udx-python/setup-vdb/base/setup-vdb.yaml | 31 --- 4 files changed, 308 deletions(-) delete mode 100644 tests/e2e-udx/udx-python/expected-outputs/PythonScalarFunctionsOut.txt delete mode 100644 tests/e2e-udx/udx-python/expected-outputs/PythonTransformFunctionsOut.txt delete mode 100644 tests/e2e-udx/udx-python/setup-vdb/base/kustomization.yaml delete mode 100644 tests/e2e-udx/udx-python/setup-vdb/base/setup-vdb.yaml diff --git a/tests/e2e-udx/udx-python/expected-outputs/PythonScalarFunctionsOut.txt b/tests/e2e-udx/udx-python/expected-outputs/PythonScalarFunctionsOut.txt deleted file mode 100644 index e53de14ca..000000000 --- a/tests/e2e-udx/udx-python/expected-outputs/PythonScalarFunctionsOut.txt +++ /dev/null @@ -1,56 +0,0 @@ -CREATE LIBRARY -CREATE FUNCTION -CREATE FUNCTION -CREATE FUNCTION -CREATE FUNCTION -CREATE FUNCTION -CREATE TABLE - add2ints ----------- - 3 - 7 - 11 -(3 rows) - -DROP TABLE -CREATE TABLE - copy2letters --------------- - pl - ve - an - ve - 9. -(5 rows) - -DROP TABLE -CREATE TABLE - add2hours --------------------------- - 1900-12-28 15:26:13.0129 - 2000-08-06 01:01:44.0111 - 2018-04-20 02:00:00 -(3 rows) - -DROP TABLE -CREATE TABLE - add2decimals --------------- - 12.1000 - 3.3000 - 7.7000 -(3 rows) - -DROP TABLE -CREATE TABLE - addprefix -------------- - __platform - __vertica - __analytics - __version - __9.1 -(5 rows) - -DROP TABLE -DROP LIBRARY diff --git a/tests/e2e-udx/udx-python/expected-outputs/PythonTransformFunctionsOut.txt b/tests/e2e-udx/udx-python/expected-outputs/PythonTransformFunctionsOut.txt deleted file mode 100644 index e373a669f..000000000 --- a/tests/e2e-udx/udx-python/expected-outputs/PythonTransformFunctionsOut.txt +++ /dev/null @@ -1,206 +0,0 @@ -CREATE LIBRARY -CREATE TRANSFORM FUNCTION -CREATE TRANSFORM FUNCTION -CREATE TABLE - tokens --------- - Word - The - quick - brown - fox - jumped - over - the - lazy - dog -(10 rows) - -DROP TABLE -CREATE TABLE - OUTPUT --------- - 1 -(1 row) - - OUTPUT --------- - 1 -(1 row) - - OUTPUT --------- - 1 -(1 row) - -COMMIT - url | tokens ------------------+----------- - www.amazon.com | Online - www.amazon.com | retail - www.amazon.com | merchant - www.amazon.com | and - www.amazon.com | provider - www.amazon.com | of - www.amazon.com | cloud - www.amazon.com | services - www.hp.com | Leading - www.hp.com | provider - www.hp.com | of - www.hp.com | computer - www.hp.com | hardware - www.hp.com | and - www.hp.com | imaging - www.hp.com | solutions - www.vertica.com | World's - www.vertica.com | fastest - www.vertica.com | analytic - www.vertica.com | database -(20 rows) - -DROP TABLE -CREATE TABLE - OUTPUT --------- - 1 -(1 row) - - OUTPUT --------- - 1 -(1 row) - - OUTPUT --------- - 2 -(1 row) - - OUTPUT --------- - 4 -(1 row) - - a | b | c ----+---+--- - 1 | 1 | 2 - 2 | 2 | 2 - 2 | 2 | 2 - 2 | 2 | 2 - 3 | 2 | 4 -(5 rows) - - a | b | c ----+---+--- - 2 | 2 | 2 - 2 | 2 | 2 - 1 | 1 | 2 - 4 | 4 | 5 - 3 | 2 | 4 - 3 | 2 | 4 -(6 rows) - -DROP TABLE -DROP LIBRARY -CREATE TABLE -CREATE LIBRARY -CREATE TRANSFORM FUNCTION - average | ignored_rows | total_rows ----------+--------------+------------ - 2.5 | 3 | 7 -(1 row) - -DROP LIBRARY -DROP TABLE -CREATE TABLE - OUTPUT --------- - 1 -(1 row) - - OUTPUT --------- - 1 -(1 row) - - OUTPUT --------- - 1 -(1 row) - - OUTPUT --------- - 1 -(1 row) - -COMMIT -CREATE LIBRARY -CREATE TRANSFORM FUNCTION - term | doc_id | term_freq | corp_freq -------------+--------+-----------+----------- - Loads | 400 | 1 | - Loads | | | 1 - and | 400 | 1 | - and | | | 1 - combined | 300 | 1 | - combined | | | 1 - Defined | 200 | 1 | - Defined | | | 1 - User | 200 | 1 | - User | | | 1 - for | 200 | 1 | - for | | | 1 - in | 300 | 1 | - in | | | 1 - short | 200 | 1 | - short | | | 1 - single | 400 | 1 | - single | | | 1 - Vertica | 100 | 1 | - Vertica | 300 | 1 | - Vertica | | | 2 - scalar | 400 | 1 | - scalar | | | 1 - database | 100 | 1 | - database | 200 | 1 | - database | 300 | 1 | - database | | | 3 - include | 300 | 1 | - include | 400 | 1 | - include | | | 2 - transforms | 400 | 1 | - transforms | | | 1 - with | 200 | 1 | - with | 300 | 1 | - with | | | 2 - Framework | 400 | 1 | - Framework | | | 1 - Functions | 200 | 1 | - Functions | | | 1 - functions | 200 | 2 | - functions | 300 | 2 | - functions | 400 | 2 | - functions | | | 3 - many | 300 | 1 | - many | | | 1 - multiphase | 400 | 1 | - multiphase | | | 1 - things | 300 | 1 | - things | | | 1 - Analytic | 100 | 1 | - Analytic | 200 | 1 | - Analytic | | | 2 - UD | 200 | 1 | - UD | 300 | 2 | - UD | 400 | 4 | - UD | | | 3 - UDx | 400 | 1 | - UDx | | | 1 - aggregates | 400 | 1 | - aggregates | | | 1 - analytic | 300 | 1 | - analytic | 400 | 1 | - analytic | | | 2 -(63 rows) - -DROP LIBRARY -DROP TABLE diff --git a/tests/e2e-udx/udx-python/setup-vdb/base/kustomization.yaml b/tests/e2e-udx/udx-python/setup-vdb/base/kustomization.yaml deleted file mode 100644 index 01b927990..000000000 --- a/tests/e2e-udx/udx-python/setup-vdb/base/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resources: - - setup-vdb.yaml diff --git a/tests/e2e-udx/udx-python/setup-vdb/base/setup-vdb.yaml b/tests/e2e-udx/udx-python/setup-vdb/base/setup-vdb.yaml deleted file mode 100644 index bddb332dc..000000000 --- a/tests/e2e-udx/udx-python/setup-vdb/base/setup-vdb.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: vertica.com/v1beta1 -kind: VerticaDB -metadata: - name: v-udx-python -spec: - image: kustomize-vertica-image - communal: - includeUIDInPath: true - local: - requestSize: 100Mi - dbName: vertdb - encryptSpreadComm: vertica - subclusters: - - name: sc1 - size: 1 - kSafety: "0" - certSecrets: [] - imagePullSecrets: [] \ No newline at end of file From 336af93e168d5e5d4acd01b471364cbf54eb8f58 Mon Sep 17 00:00:00 2001 From: Matt Spilchen Date: Thu, 18 Aug 2022 09:44:59 -0300 Subject: [PATCH 20/30] Add changie --- changes/unreleased/Added-20220818-094450.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/unreleased/Added-20220818-094450.yaml diff --git a/changes/unreleased/Added-20220818-094450.yaml b/changes/unreleased/Added-20220818-094450.yaml new file mode 100644 index 000000000..34577106f --- /dev/null +++ b/changes/unreleased/Added-20220818-094450.yaml @@ -0,0 +1,5 @@ +kind: Added +body: Support for Java, C++ and Python UDx's +time: 2022-08-18T09:44:50.678890201-03:00 +custom: + Issue: "248" From 2c3559ca18ef5deedaba6832766869d3cfced6bc Mon Sep 17 00:00:00 2001 From: jizhuoyu <44009608+jizhuoyu@users.noreply.github.com> Date: Thu, 18 Aug 2022 13:51:02 -0400 Subject: [PATCH 21/30] Update tests/e2e-udx/udx-java/README.txt Co-authored-by: spilchen --- tests/e2e-udx/udx-java/README.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-udx/udx-java/README.txt b/tests/e2e-udx/udx-java/README.txt index 8b1378917..3a0ccf12a 100644 --- a/tests/e2e-udx/udx-java/README.txt +++ b/tests/e2e-udx/udx-java/README.txt @@ -1 +1 @@ - +Tests Java UDx by compiling and running the samples from the Vertica server image From 2d6125a50f9bfccfdbffc1f2318bbfc1acbbf0b0 Mon Sep 17 00:00:00 2001 From: jizhuoyu <44009608+jizhuoyu@users.noreply.github.com> Date: Thu, 18 Aug 2022 13:51:31 -0400 Subject: [PATCH 22/30] Update tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml Co-authored-by: spilchen --- tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml b/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml index 96f9d0c74..b2d3dfbbf 100644 --- a/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml +++ b/tests/e2e-udx/udx-java/setup-vdb/base/setup-vdb.yaml @@ -29,10 +29,5 @@ spec: kSafety: "0" certSecrets: [] imagePullSecrets: [] - # volumes: - # - name: downloads - # hostPath: - # path: /host - # volumeMounts: - # - mountPath: /host - # name: downloads + volumes: [] + volumeMounts: [] From b833edfbf0ed27e2bd03e5f78b3eedb42e809254 Mon Sep 17 00:00:00 2001 From: jizhuoyu <44009608+jizhuoyu@users.noreply.github.com> Date: Thu, 18 Aug 2022 13:52:26 -0400 Subject: [PATCH 23/30] Update tests/e2e-udx/udx-java/52-verify-java-udl.yaml Co-authored-by: spilchen --- tests/e2e-udx/udx-java/52-verify-java-udl.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/e2e-udx/udx-java/52-verify-java-udl.yaml b/tests/e2e-udx/udx-java/52-verify-java-udl.yaml index 29c40fd8f..188aec058 100644 --- a/tests/e2e-udx/udx-java/52-verify-java-udl.yaml +++ b/tests/e2e-udx/udx-java/52-verify-java-udl.yaml @@ -21,10 +21,7 @@ data: set -o xtrace set -o errexit - VDB_NAME=v-udx-java - SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME - POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") - echo $POD_NAME + POD_NAME=v-udx-java-sc1-0 UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples; vsql -U dbadmin -f JavaUDLFunctions.sql 2>&1") echo "$UDX_OP" kubectl exec $POD_NAME -i -- bash -c "echo \"$UDX_OP\" | diff - /opt/vertica/sdk/examples/expected-outputs/JavaUDLFunctionsOut.txt" From 314c78283f0d4a7a07e1f3ae96bf7156003b2011 Mon Sep 17 00:00:00 2001 From: jizhuoyu <44009608+jizhuoyu@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:36:12 -0400 Subject: [PATCH 24/30] Update tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml Co-authored-by: spilchen --- .../e2e-udx/udx-java/50-verify-java-scalar-transform.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml index 9331740aa..e3310d7db 100644 --- a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml +++ b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml @@ -21,12 +21,8 @@ data: set -o xtrace set -o errexit - VDB_NAME=v-udx-java - SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME - POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") - echo $POD_NAME - UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "vsql -U dbadmin -c \"SELECT SET_CONFIG_PARAMETER('JavaSideProcessMinHeapSizeMB',2048);\"") - echo "$UDX_OP" + POD_NAME=v-udx-java-sc1-0 + kubectl exec $POD_NAME -i -- bash -c "vsql -U dbadmin -c \"SELECT SET_CONFIG_PARAMETER('JavaSideProcessMinHeapSizeMB',2048);\"" UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples; vsql -U dbadmin -f JavaFunctions.sql 2>&1") echo "$UDX_OP" kubectl exec $POD_NAME -i -- bash -c "echo \"$UDX_OP\" | diff - /opt/vertica/sdk/examples/expected-outputs/JavaFunctionsOut.txt" From 9afba3d4c41d9eb8f15cc274d9bfda424ccc6885 Mon Sep 17 00:00:00 2001 From: jizhuoyu <44009608+jizhuoyu@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:36:40 -0400 Subject: [PATCH 25/30] Update tests/e2e-udx/udx-java/51-verify-java-uda.yaml Co-authored-by: spilchen --- tests/e2e-udx/udx-java/51-verify-java-uda.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/e2e-udx/udx-java/51-verify-java-uda.yaml b/tests/e2e-udx/udx-java/51-verify-java-uda.yaml index 1a6aeabd0..f2be42c94 100644 --- a/tests/e2e-udx/udx-java/51-verify-java-uda.yaml +++ b/tests/e2e-udx/udx-java/51-verify-java-uda.yaml @@ -21,10 +21,7 @@ data: set -o xtrace set -o errexit - VDB_NAME=v-udx-java - SELECTOR=app.kubernetes.io/name=vertica,app.kubernetes.io/instance=$VDB_NAME - POD_NAME=$(kubectl get pods --selector=$SELECTOR -o jsonpath="{.items[0].metadata.name}") - echo $POD_NAME + POD_NAME=v-udx-java-sc1-0 UDX_OP=$(kubectl exec $POD_NAME -i -- bash -c "cd /opt/vertica/sdk/examples; vsql -U dbadmin -f JavaUDAnalytics.sql 2>&1") echo "$UDX_OP" # the row orders in the output might be unstable, so sort the entire output and then compare From 5703d45082539caf0806c01e0febd9cdd9264450 Mon Sep 17 00:00:00 2001 From: jizhuoyu <44009608+jizhuoyu@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:37:10 -0400 Subject: [PATCH 26/30] Update tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml Co-authored-by: spilchen --- tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml index e3310d7db..7843c35c0 100644 --- a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml +++ b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml @@ -44,12 +44,6 @@ spec: mountPath: /bin/entrypoint.sh readOnly: true subPath: entrypoint.sh - # - name: lib-volume - # mountPath: /bin/UDx-Examples/build - # readOnly: true - # - name: sql-volume - # mountPath: /bin/UDx-Examples/Java-and-C++ - # readOnly: true volumes: - name: entrypoint-volume configMap: From af62c3c4b749f45a65776c0c314182eeeb28dcfb Mon Sep 17 00:00:00 2001 From: jizhuoyu <44009608+jizhuoyu@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:37:27 -0400 Subject: [PATCH 27/30] Update tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml Co-authored-by: spilchen --- .../e2e-udx/udx-java/50-verify-java-scalar-transform.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml index 7843c35c0..d011a7fa2 100644 --- a/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml +++ b/tests/e2e-udx/udx-java/50-verify-java-scalar-transform.yaml @@ -49,11 +49,3 @@ spec: configMap: defaultMode: 0700 name: script-verify-java-scalar-transform - # - name: lib-volume - # hostPath: - # path: /host/UDx-Examples/build - # type: Directory - # - name: sql-volume - # hostPath: - # path: /host/UDx-Examples/Java-and-C++ - # type: Directory From 2ddcc91fa0c41d5c649848909cb5b2b9fb1de400 Mon Sep 17 00:00:00 2001 From: jizhuoyu <44009608+jizhuoyu@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:43:39 -0400 Subject: [PATCH 28/30] Update scripts/setup-kustomize.sh Co-authored-by: spilchen --- scripts/setup-kustomize.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup-kustomize.sh b/scripts/setup-kustomize.sh index fdd8b3fe2..4af0e447e 100755 --- a/scripts/setup-kustomize.sh +++ b/scripts/setup-kustomize.sh @@ -742,7 +742,7 @@ for tdir in e2e/*/*/base e2e-extra/*/*/base e2e-server-upgrade/*/*/base e2e-oper do create_vdb_pod_kustomization $(dirname $tdir) $(basename $(realpath $tdir/../..)) done -for tdir in e2e/* e2e-extra/* e2e-disabled/* e2e-server-upgrade/* e2e-operator-upgrade-overlays/* e2e-udx/* e2e-udx/* +for tdir in e2e/* e2e-extra/* e2e-disabled/* e2e-server-upgrade/* e2e-operator-upgrade-overlays/* e2e-udx/* do clean_communal_kustomization $tdir done From 8761e443f6a5bb0e57f70570685e3b21df72b55e Mon Sep 17 00:00:00 2001 From: jizhuoyu <44009608+jizhuoyu@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:44:25 -0400 Subject: [PATCH 29/30] Update docker-vertica/docker-entrypoint.sh Co-authored-by: spilchen --- docker-vertica/docker-entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-vertica/docker-entrypoint.sh b/docker-vertica/docker-entrypoint.sh index db953b1e4..c58dd536f 100755 --- a/docker-vertica/docker-entrypoint.sh +++ b/docker-vertica/docker-entrypoint.sh @@ -1,5 +1,6 @@ #!/bin/bash -set -ex +set -o errexit +set -o xtrace start_cron(){ # daemonizes, no need for & From e6a073d9923458b4636d7fc1ab69e0b711fececd Mon Sep 17 00:00:00 2001 From: Zhuoyu Ji Date: Thu, 18 Aug 2022 16:55:42 -0400 Subject: [PATCH 30/30] change according to review comments --- .../samples/v1beta1_verticadb_host_mount.yaml | 42 --------------- .../v1beta1_verticadb_host_mount_java.yaml | 42 --------------- .../samples/v1beta1_verticadb_new_design.yaml | 52 ------------------- docker-vertica/Dockerfile | 13 ++--- docker-vertica/docker-entrypoint.sh | 3 -- docker-vertica/packages/cleanup.sh | 9 +--- tests/e2e-udx/udx-java/45-upload-files.yaml | 4 +- 7 files changed, 8 insertions(+), 157 deletions(-) delete mode 100644 config/samples/v1beta1_verticadb_host_mount.yaml delete mode 100644 config/samples/v1beta1_verticadb_host_mount_java.yaml delete mode 100644 config/samples/v1beta1_verticadb_new_design.yaml diff --git a/config/samples/v1beta1_verticadb_host_mount.yaml b/config/samples/v1beta1_verticadb_host_mount.yaml deleted file mode 100644 index 94ec08ac4..000000000 --- a/config/samples/v1beta1_verticadb_host_mount.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: vertica.com/v1beta1 -kind: VerticaDB -metadata: - name: verticadb-sample -spec: - image: "vertica-k8s:1.6.0" - communal: - path: "s3://nimbusdb/db" - endpoint: "http://minio" - credentialSecret: s3-auth - includeUIDInPath: true - volumes: - - name: downloads - hostPath: - path: /host - volumeMounts: - - mountPath: /host - name: downloads - kSafety: "0" - subclusters: - - name: defaultsubcluster - size: 1 - # The CPU resource setting is here is a sample. We set it so that this - # will work with the sample VerticaAutoscaler resource. The actual amount - # should be sized according to: - # https://www.vertica.com/kb/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters/Content/Hardware/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters.htm - resources: - requests: - cpu: 500m diff --git a/config/samples/v1beta1_verticadb_host_mount_java.yaml b/config/samples/v1beta1_verticadb_host_mount_java.yaml deleted file mode 100644 index 065bc9764..000000000 --- a/config/samples/v1beta1_verticadb_host_mount_java.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: vertica.com/v1beta1 -kind: VerticaDB -metadata: - name: verticadb-sample -spec: - image: "vertica-k8s-fat:1.7.0" - communal: - path: "s3://nimbusdb/db" - endpoint: "http://minio" - credentialSecret: s3-auth - includeUIDInPath: true - volumes: - - name: downloads - hostPath: - path: /host - volumeMounts: - - mountPath: /host - name: downloads - kSafety: "0" - subclusters: - - name: defaultsubcluster - size: 1 - # The CPU resource setting is here is a sample. We set it so that this - # will work with the sample VerticaAutoscaler resource. The actual amount - # should be sized according to: - # https://www.vertica.com/kb/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters/Content/Hardware/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters.htm - resources: - requests: - cpu: 500m diff --git a/config/samples/v1beta1_verticadb_new_design.yaml b/config/samples/v1beta1_verticadb_new_design.yaml deleted file mode 100644 index e5d3acee2..000000000 --- a/config/samples/v1beta1_verticadb_new_design.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# (c) Copyright [2021-2022] Micro Focus or one of its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: vertica.com/v1beta1 -kind: VerticaDB -metadata: - name: verticadb-sample -spec: - image: "vertica-k8s:1.6.0" - communal: - path: "s3://nimbusdb/db" - endpoint: "http://minio" - credentialSecret: s3-auth - includeUIDInPath: true - volumes: - - name: downloads - hostPath: - path: /host - volumeMounts: - - mountPath: /host - name: downloads - init: - - name: java - image: "udx-java:1.6.0" - # in the image: 2 part, 1: download jre; 2: FROM: copy jvm folder from first stage to second stage - command: "time cp -r /tmp/java-se-8u41-ri/ /export/java-se-8u41-ri/" - paths: - - initPath: /export/java-se-8u41-ri/ - serverPath: /opt/jvm/ - # annotations: - # JAVA_TOOL_OPTIONS: "-Xms2G -Xmx2G" - kSafety: "0" - subclusters: - - name: defaultsubcluster - size: 1 - # The CPU resource setting is here is a sample. We set it so that this - # will work with the sample VerticaAutoscaler resource. The actual amount - # should be sized according to: - # https://www.vertica.com/kb/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters/Content/Hardware/Recommendations-for-Sizing-Vertica-Nodes-and-Clusters.htm - resources: - requests: - cpu: 500m diff --git a/docker-vertica/Dockerfile b/docker-vertica/Dockerfile index e2f4b3689..914898747 100644 --- a/docker-vertica/Dockerfile +++ b/docker-vertica/Dockerfile @@ -135,15 +135,10 @@ RUN set -x \ procps \ sysstat \ sudo \ - && rm -rf /var/lib/apt/lists/* -RUN set -x \ # Install jre if not minimal && if [[ ($MINIMAL != "YES" && $MINIMAL != "yes") ]] ; then \ - apt-get -y update \ - && apt-get install -y --no-install-recommends $JRE_PKG \ - && rm -rf /var/lib/apt/lists/* ; \ - fi -RUN set -x \ + apt-get install -y --no-install-recommends $JRE_PKG; \ + fi \ && rm -rf /var/lib/apt/lists/* \ # Make the "en_US.UTF-8" locale so vertica will be utf-8 enabled by default && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ @@ -157,7 +152,9 @@ RUN set -x \ # Set JAVA_HOME environment variable if not minimal, this will be loaded to all shells && if [[ $MINIMAL != "YES" && $MINIMAL != "yes" ]]; then \ echo "JAVA_HOME=/usr" >> /etc/environment; \ - fi + fi \ + # Create a symlink to python3 interpreter in vertica + && update-alternatives --install /usr/bin/python python /opt/vertica/oss/python3/bin/python3 1 ENTRYPOINT [ "/usr/local/bin/docker-entrypoint.sh" ] diff --git a/docker-vertica/docker-entrypoint.sh b/docker-vertica/docker-entrypoint.sh index c58dd536f..56af581ba 100755 --- a/docker-vertica/docker-entrypoint.sh +++ b/docker-vertica/docker-entrypoint.sh @@ -35,7 +35,4 @@ copy_config_files echo "Vertica container is now running" -# Create a symlink to python3 interpreter in vertica -sudo update-alternatives --install /usr/bin/python python /opt/vertica/oss/python3/bin/python3 1 - sudo /usr/sbin/sshd -D diff --git a/docker-vertica/packages/cleanup.sh b/docker-vertica/packages/cleanup.sh index b548c2b16..3df012c15 100644 --- a/docker-vertica/packages/cleanup.sh +++ b/docker-vertica/packages/cleanup.sh @@ -33,12 +33,6 @@ rm -r -f \ /opt/vertica/oss/python*/lib/python*/tkinter \ /opt/vertica/oss/python*/lib/python*/idlelib - # (optional) minimal images remove sdk folder which enables building UDx libraries -if [ "$MINIMAL" = "YES" ] || [ "$MINIMAL" = "yes" ] -then - rm -r -f /opt/vertica/sdk -fi - # cleanup many of the __pycache__ directories find /opt/vertica/oss/ -type d -name "__pycache__" -exec rm -rf {} + @@ -56,7 +50,7 @@ strip /opt/vertica/packages/*/lib/*.so* 2> /dev/null /opt/vertica/oss/python*/bin/python[0-9] \ /tmp/package-checksum-patcher.py /opt/vertica/packages/* -# (optional) minimal images remove packages that aren't auto installed +# (optional) minimal images remove packages that aren't auto installed as well as the sdk folder if [ "$MINIMAL" = "YES" ] || [ "$MINIMAL" = "yes" ] then cd /opt/vertica/packages @@ -64,4 +58,5 @@ then do rm -rf $i done + rm -r -f /opt/vertica/sdk fi diff --git a/tests/e2e-udx/udx-java/45-upload-files.yaml b/tests/e2e-udx/udx-java/45-upload-files.yaml index 05e1f81a9..1ef9f28f0 100644 --- a/tests/e2e-udx/udx-java/45-upload-files.yaml +++ b/tests/e2e-udx/udx-java/45-upload-files.yaml @@ -4,6 +4,4 @@ commands: # upload the built library binaries - command: kubectl -n $NAMESPACE cp ../../../bin/udx-libs/build v-udx-java-sc1-0:/opt/vertica/sdk/examples # upload the expected outputs of executing sql files - - command: kubectl -n $NAMESPACE cp ./expected-outputs v-udx-java-sc1-0:/opt/vertica/sdk/examples - # - command: kubectl -n $NAMESPACE exec v-udx-java-sc1-0 -- ls /opt/vertica/sdk/examples - # - command: kubectl -n $NAMESPACE exec v-udx-java-sc1-0 -- ls /opt/vertica/sdk/examples/build \ No newline at end of file + - command: kubectl -n $NAMESPACE cp ./expected-outputs v-udx-java-sc1-0:/opt/vertica/sdk/examples \ No newline at end of file