From edc4eb90a9333145f92214bbae8f028c6f0f217e Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Tue, 30 May 2023 20:41:01 -0400 Subject: [PATCH 01/11] feat: sklearnserver rock integration Summary of changes: - Added ROCK integrity tests. - Updated ROCK image definition. - Added tox with unit and integration tests. --- sklearnserver/rockcraft.yaml | 19 +++++---- sklearnserver/tests/test_rock.py | 43 ++++++++++++++++++++ sklearnserver/tox.ini | 69 ++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 sklearnserver/tests/test_rock.py create mode 100644 sklearnserver/tox.ini diff --git a/sklearnserver/rockcraft.yaml b/sklearnserver/rockcraft.yaml index 959eeb5..8773fa2 100644 --- a/sklearnserver/rockcraft.yaml +++ b/sklearnserver/rockcraft.yaml @@ -3,7 +3,7 @@ summary: An image for Seldon SKLearn Server description: | This image is used as part of the Charmed Kubeflow product. The SKLearn Server serves models which have been stored as pickles. -version: v1.16.0_1 # version format: _ +version: v1.16.0_20.04_1 # -_ license: Apache-2.0 base: ubuntu:20.04 services: @@ -78,15 +78,20 @@ parts: # but it does need to match pebble's workdir install -D -m 755 ${CRAFT_STAGE}/microservice/SKLearnServer.py microservice/SKLearnServer.py - # security requirement - mkdir -p ${CRAFT_PART_INSTALL}/usr/share/rocks - (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query" && \ - dpkg-query -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W) \ - > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query - non-root-user: plugin: nil overlay-script: | # Create a user in the $CRAFT_OVERLAY chroot groupadd -R $CRAFT_OVERLAY -g 1001 ubuntu useradd -R $CRAFT_OVERLAY -M -r -u 1001 -g ubuntu ubuntu + + security-team-requirement: + plugin: nil + after: [sklearnserver] + override-build: | + # security requirement + # there are no packages installed in `bare` base which is used in this rock + # `--root` option is not available in dpkg-query version which is packaged with 20.04 + mkdir -p ${CRAFT_PART_INSTALL}/usr/share/rocks + (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query") \ + > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query diff --git a/sklearnserver/tests/test_rock.py b/sklearnserver/tests/test_rock.py new file mode 100644 index 0000000..a5cfa98 --- /dev/null +++ b/sklearnserver/tests/test_rock.py @@ -0,0 +1,43 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Tests for required artifacts to be present in the ROCK image. +# + +from pathlib import Path + +import os +import logging +import random +import pytest +import string +import subprocess +import yaml +from pytest_operator.plugin import OpsTest + +def read_rock_info(): + ROCKCRAFT = yaml.safe_load(Path("rockcraft.yaml").read_text()) + name = ROCKCRAFT["name"] + version = ROCKCRAFT["version"] + arch = list(ROCKCRAFT["platforms"].keys())[0] + return f"{name}_{version}_{arch}:{version}" + +@pytest.fixture() +def rock_test_env(): + """Yields a temporary directory and random docker container name, then cleans them up after.""" + container_name = "".join([str(i) for i in random.choices(string.ascii_lowercase, k=8)]) + yield container_name + + try: + subprocess.run(["docker", "rm", container_name]) + except Exception: + pass + +@pytest.mark.abort_on_fail +def test_rock(ops_test: OpsTest, rock_test_env): + """Test rock.""" + container_name = rock_test_env + LOCAL_ROCK_IMAGE = read_rock_info() + + # verify that all artifacts are in correct locations + subprocess.run(["docker", "run", LOCAL_ROCK_IMAGE, "exec", "ls", "-la", "/microservice/SKLearnServer.py"], check=True) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini new file mode 100644 index 0000000..9c4b6a2 --- /dev/null +++ b/sklearnserver/tox.ini @@ -0,0 +1,69 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +[tox] +skipsdist = True +skip_missing_interpreters = True + +[testenv] +setenv = + PYTHONPATH={toxinidir} + PYTHONBREAKPOINT=ipdb.set_trace + CHARM_REPO=https://github.com/canonical/seldon-core-operator.git + CHARM_BRANCH=main + LOCAL_CHARM_DIR=charm_repo + +[testenv:unit] +passenv = * +allowlist_externals = + bash + tox + rockcraft +deps = + juju~=2.9.0 + pytest + pytest-operator + ops +commands = + # build and pack rock + rockcraft pack + bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ + VERSION=$(yq eval .version rockcraft.yaml) && \ + ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \ + ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \ + sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION' + # run rock tests + pytest -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests + +[testenv:integration] +passenv = * +allowlist_externals = + bash + git + rm + tox + rockcraft +deps = + juju~=2.9.0 + pytest + pytest-operator + ops +commands = + # build and pack rock + #rockcraft pack + # clone related charm + rm -rf {env:LOCAL_CHARM_DIR} + git clone --branch {env:CHARM_BRANCH} {env:CHARM_REPO} {env:LOCAL_CHARM_DIR} + # upload rock to docker and microk8s cache, replace charm's container with local rock reference + bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ + VERSION=$(yq eval .version rockcraft.yaml) && \ + ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \ + ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \ + sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \ + docker save $ROCK > $ROCK.tar && \ + microk8s ctr image import $ROCK.tar && \ + predictor_servers=$(yq e ".data.predictor_servers" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2) && \ + predictor_servers=$(jq --arg jq_rock $ROCK -r '\''.SKLEARN_SERVER.protocols.seldon.image=$jq_rock'\'' <<< $predictor_servers) && \ + predictor_servers=$(jq --arg jq_version $VERSION -r '\''.SKLEARN_SERVER.protocols.seldon.defaultImageVersion=$jq_version'\'' <<< $predictor_servers) && \ + yq e -i ".data.predictor_servers=$predictor_servers" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2' + # run charm integration test with rock + tox -c {env:LOCAL_CHARM_DIR} -e integration From c248e5effcb5f58f4a2a2bb656b565eea6f35735 Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Thu, 1 Jun 2023 17:03:28 -0400 Subject: [PATCH 02/11] feat: sklearn server rock tests Summary of changes: - Addressed review comments. - Modified to run tests. --- sklearnserver/tests/test_rock.py | 11 +++-------- sklearnserver/tox.ini | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/sklearnserver/tests/test_rock.py b/sklearnserver/tests/test_rock.py index a5cfa98..b087051 100644 --- a/sklearnserver/tests/test_rock.py +++ b/sklearnserver/tests/test_rock.py @@ -4,6 +4,7 @@ # Tests for required artifacts to be present in the ROCK image. # +from charmed_kubeflow_chisme.rock import CheckRock from pathlib import Path import os @@ -15,13 +16,6 @@ import yaml from pytest_operator.plugin import OpsTest -def read_rock_info(): - ROCKCRAFT = yaml.safe_load(Path("rockcraft.yaml").read_text()) - name = ROCKCRAFT["name"] - version = ROCKCRAFT["version"] - arch = list(ROCKCRAFT["platforms"].keys())[0] - return f"{name}_{version}_{arch}:{version}" - @pytest.fixture() def rock_test_env(): """Yields a temporary directory and random docker container name, then cleans them up after.""" @@ -36,8 +30,9 @@ def rock_test_env(): @pytest.mark.abort_on_fail def test_rock(ops_test: OpsTest, rock_test_env): """Test rock.""" + check_rock = CheckRock("rockcraft.yaml") container_name = rock_test_env - LOCAL_ROCK_IMAGE = read_rock_info() + LOCAL_ROCK_IMAGE = check_rock.get_image_name() # verify that all artifacts are in correct locations subprocess.run(["docker", "run", LOCAL_ROCK_IMAGE, "exec", "ls", "-la", "/microservice/SKLearnServer.py"], check=True) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini index 9c4b6a2..68b2baf 100644 --- a/sklearnserver/tox.ini +++ b/sklearnserver/tox.ini @@ -19,6 +19,7 @@ allowlist_externals = tox rockcraft deps = + charmed_kubeflow_chisme juju~=2.9.0 pytest pytest-operator @@ -26,11 +27,12 @@ deps = commands = # build and pack rock rockcraft pack - bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ - VERSION=$(yq eval .version rockcraft.yaml) && \ - ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \ - ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \ + bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ + print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ + VERSION=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ + print(CheckRock("rockcraft.yaml").get_version())'\'') && \ sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION' + # run rock tests pytest -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests @@ -43,21 +45,22 @@ allowlist_externals = tox rockcraft deps = + charmed_kubeflow_chisme juju~=2.9.0 pytest pytest-operator ops commands = # build and pack rock - #rockcraft pack + rockcraft pack # clone related charm rm -rf {env:LOCAL_CHARM_DIR} git clone --branch {env:CHARM_BRANCH} {env:CHARM_REPO} {env:LOCAL_CHARM_DIR} # upload rock to docker and microk8s cache, replace charm's container with local rock reference - bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ - VERSION=$(yq eval .version rockcraft.yaml) && \ - ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \ - ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \ + bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ + print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ + VERSION=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ + print(CheckRock("rockcraft.yaml").get_version())'\'') && \ sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \ docker save $ROCK > $ROCK.tar && \ microk8s ctr image import $ROCK.tar && \ From 90d8083bdc09671d0b78c0d37cc2671c9f8b094a Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Thu, 1 Jun 2023 17:03:28 -0400 Subject: [PATCH 03/11] feat: sklearn server rock tests Summary of changes: - Addressed review comments. - Modified to run tests. --- .github/workflows/build_and_publish_rock.yaml | 41 +++++++++++++++++++ sklearnserver/tests/test_rock.py | 11 ++--- sklearnserver/tox.ini | 20 +++++---- 3 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/build_and_publish_rock.yaml diff --git a/.github/workflows/build_and_publish_rock.yaml b/.github/workflows/build_and_publish_rock.yaml new file mode 100644 index 0000000..a1dcf68 --- /dev/null +++ b/.github/workflows/build_and_publish_rock.yaml @@ -0,0 +1,41 @@ +# Workflow to publish ROCK +name: Publish ROCK + +on: + workflow_call: + inputs: + rock_dir: + description: Directory in the repo that contains ROCK image definition + default: '' + required: false + type: string + secrets: + CKF_REGISTRY_USERNAME: + required: true + CKF_REGISTRY_PASSWORD: + required: true +env: + CKF_REGISTRY: ${{ vars.CKF_REGISTRY }} + +jobs: + build-and-publish-rock: + name: Build and publish ROCK + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + steps: + - name: Build and test ROCK + run: | + cd ${{ inputs.rock_dir }} + tox -e unit + tox -e integration + - name: Publish ROCK + run: | + NAME=$(yq eval .name rockcraft.yaml) + VERSION=$(yq eval .version rockcraft.yaml) + ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') + ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" + sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION + docker tag $ROCK:$VERSION ${{ inputs.CKF_REGISTRY }}/$ROCK:$VERSION + docker login -u ${{ inputs.CKF_REGISTRY_USERNAME }} -p ${{inputs.CKF_REGISTRY_PASSWORD }} ${{inputs.CKF_REGISTRY }} + docker push ${{ inputs.CKF_REGISTRY }}/$ROCK:$VERSION diff --git a/sklearnserver/tests/test_rock.py b/sklearnserver/tests/test_rock.py index a5cfa98..b087051 100644 --- a/sklearnserver/tests/test_rock.py +++ b/sklearnserver/tests/test_rock.py @@ -4,6 +4,7 @@ # Tests for required artifacts to be present in the ROCK image. # +from charmed_kubeflow_chisme.rock import CheckRock from pathlib import Path import os @@ -15,13 +16,6 @@ import yaml from pytest_operator.plugin import OpsTest -def read_rock_info(): - ROCKCRAFT = yaml.safe_load(Path("rockcraft.yaml").read_text()) - name = ROCKCRAFT["name"] - version = ROCKCRAFT["version"] - arch = list(ROCKCRAFT["platforms"].keys())[0] - return f"{name}_{version}_{arch}:{version}" - @pytest.fixture() def rock_test_env(): """Yields a temporary directory and random docker container name, then cleans them up after.""" @@ -36,8 +30,9 @@ def rock_test_env(): @pytest.mark.abort_on_fail def test_rock(ops_test: OpsTest, rock_test_env): """Test rock.""" + check_rock = CheckRock("rockcraft.yaml") container_name = rock_test_env - LOCAL_ROCK_IMAGE = read_rock_info() + LOCAL_ROCK_IMAGE = check_rock.get_image_name() # verify that all artifacts are in correct locations subprocess.run(["docker", "run", LOCAL_ROCK_IMAGE, "exec", "ls", "-la", "/microservice/SKLearnServer.py"], check=True) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini index 9c4b6a2..7876ff4 100644 --- a/sklearnserver/tox.ini +++ b/sklearnserver/tox.ini @@ -19,6 +19,7 @@ allowlist_externals = tox rockcraft deps = + charmed_kubeflow_chisme juju~=2.9.0 pytest pytest-operator @@ -26,10 +27,10 @@ deps = commands = # build and pack rock rockcraft pack - bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ - VERSION=$(yq eval .version rockcraft.yaml) && \ - ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \ - ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \ + bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ + print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ + VERSION=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ + print(CheckRock("rockcraft.yaml").get_version())'\'') && \ sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION' # run rock tests pytest -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests @@ -43,21 +44,22 @@ allowlist_externals = tox rockcraft deps = + charmed_kubeflow_chisme juju~=2.9.0 pytest pytest-operator ops commands = # build and pack rock - #rockcraft pack + rockcraft pack # clone related charm rm -rf {env:LOCAL_CHARM_DIR} git clone --branch {env:CHARM_BRANCH} {env:CHARM_REPO} {env:LOCAL_CHARM_DIR} # upload rock to docker and microk8s cache, replace charm's container with local rock reference - bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ - VERSION=$(yq eval .version rockcraft.yaml) && \ - ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \ - ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \ + bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ + print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ + VERSION=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ + print(CheckRock("rockcraft.yaml").get_version())'\'') && \ sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \ docker save $ROCK > $ROCK.tar && \ microk8s ctr image import $ROCK.tar && \ From 8b0b42c3b3e4e00df73f3a68e35e274eacdf920e Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Thu, 1 Jun 2023 17:14:08 -0400 Subject: [PATCH 04/11] fix: remove incorrect files from commit --- .github/workflows/build_and_publish_rock.yaml | 27 -------- seldon-core-operator/tox.ini | 68 ------------------- 2 files changed, 95 deletions(-) delete mode 100644 .github/workflows/build_and_publish_rock.yaml delete mode 100644 seldon-core-operator/tox.ini diff --git a/.github/workflows/build_and_publish_rock.yaml b/.github/workflows/build_and_publish_rock.yaml deleted file mode 100644 index e38bf03..0000000 --- a/.github/workflows/build_and_publish_rock.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Workflow to publish ROCK -name: Publish ROCK - -on: - workflow_dispatch: - inputs: - rock_dir: - description: Directory in the repo that contains ROCK image definition - default: '' - required: true - type: string - secrets: - CKF_REGISTRY_USERNAME: - required: true - CKF_REGISTRY_PASSWORD: - required: true - -jobs: - build-and-publish-rock: - name: Build and publish ROCK - uses: canonical/charmed-kubeflow-workflows/.github/workflows/build_and_publish_rock.yaml@main - secrets: - CKF_REGISTRY_USERNAME: ${{ secrets.CKF_REGISTRY_USERNAME }} - CKF_REGISTRY_PASSWORD: ${{ secrets.CKF_REGISTRY_PASSWORD }} - with: - rock_dir: ${{ inputs.rock_dir }} - CKF_REGISTRY: ${{ vars.CKF_REGISTRY }} diff --git a/seldon-core-operator/tox.ini b/seldon-core-operator/tox.ini deleted file mode 100644 index 0579a45..0000000 --- a/seldon-core-operator/tox.ini +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. -[tox] -skipsdist = True -skip_missing_interpreters = True - -[testenv] -setenv = - PYTHONPATH={toxinidir} - PYTHONBREAKPOINT=ipdb.set_trace - CHARM_REPO=https://github.com/canonical/seldon-core-operator.git - CHARM_BRANCH=main - LOCAL_CHARM_DIR=charm_repo - -[testenv:unit] -passenv = * -allowlist_externals = - bash - tox - rockcraft -deps = - charmed_kubeflow_chisme - juju~=2.9.0 - pytest - pytest-operator - ops -commands = - # build and pack rock - rockcraft pack - bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ - print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ - VERSION=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ - print(CheckRock("rockcraft.yaml").get_version())'\'') && \ - sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \ - docker save $ROCK > $ROCK.tar' - # run rock tests - pytest -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests - -[testenv:integration] -passenv = * -allowlist_externals = - bash - git - rm - tox - rockcraft -deps = - charmed_kubeflow_chisme - juju~=2.9.0 - pytest - pytest-operator - ops -commands = - # build and pack rock - rockcraft pack - # clone related charm - rm -rf {env:LOCAL_CHARM_DIR} - git clone --branch {env:CHARM_BRANCH} {env:CHARM_REPO} {env:LOCAL_CHARM_DIR} - # upload rock to docker and microk8s cache, replace charm's container with local rock reference - bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ - print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ - VERSION=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ - sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \ - docker save $ROCK > $ROCK.tar && \ - microk8s ctr image import $ROCK.tar && \ - yq e -i ".resources.oci-image.upstream-source=\"$ROCK:$VERSION\"" {env:LOCAL_CHARM_DIR}/metadata.yaml' - # run charm integration test with rock - tox -c {env:LOCAL_CHARM_DIR} -e integration From c45aa743d05c3c9c924460b52afe171039064ee7 Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Thu, 1 Jun 2023 20:20:49 -0400 Subject: [PATCH 05/11] fix: added handling of jinja2 templating --- sklearnserver/tox.ini | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini index 68b2baf..4a07d8d 100644 --- a/sklearnserver/tox.ini +++ b/sklearnserver/tox.ini @@ -44,6 +44,7 @@ allowlist_externals = rm tox rockcraft + sed deps = charmed_kubeflow_chisme juju~=2.9.0 @@ -56,6 +57,8 @@ commands = # clone related charm rm -rf {env:LOCAL_CHARM_DIR} git clone --branch {env:CHARM_BRANCH} {env:CHARM_REPO} {env:LOCAL_CHARM_DIR} + # replace jinja2 templated value with yq safe placeholder + sed -i "s/namespace: {{ namespace }}/namespace: YQ_SAFE/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2 # upload rock to docker and microk8s cache, replace charm's container with local rock reference bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ @@ -68,5 +71,7 @@ commands = predictor_servers=$(jq --arg jq_rock $ROCK -r '\''.SKLEARN_SERVER.protocols.seldon.image=$jq_rock'\'' <<< $predictor_servers) && \ predictor_servers=$(jq --arg jq_version $VERSION -r '\''.SKLEARN_SERVER.protocols.seldon.defaultImageVersion=$jq_version'\'' <<< $predictor_servers) && \ yq e -i ".data.predictor_servers=$predictor_servers" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2' + # replace yq safe placeholder with original value + sed -i "s/namespace: YQ_SAFE/namespace: {{ namespace }}/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2 # run charm integration test with rock - tox -c {env:LOCAL_CHARM_DIR} -e integration + tox -c {env:LOCAL_CHARM_DIR} -e integration -- --keep-models From 235c8299ef8d26b444afdaefcf7dd85864848fe9 Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Thu, 1 Jun 2023 20:20:49 -0400 Subject: [PATCH 06/11] fix: added handling of jinja2 templating --- sklearnserver/tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini index 68b2baf..8170615 100644 --- a/sklearnserver/tox.ini +++ b/sklearnserver/tox.ini @@ -44,6 +44,7 @@ allowlist_externals = rm tox rockcraft + sed deps = charmed_kubeflow_chisme juju~=2.9.0 @@ -56,6 +57,8 @@ commands = # clone related charm rm -rf {env:LOCAL_CHARM_DIR} git clone --branch {env:CHARM_BRANCH} {env:CHARM_REPO} {env:LOCAL_CHARM_DIR} + # replace jinja2 templated value with yq safe placeholder + sed -i "s/namespace: {{ namespace }}/namespace: YQ_SAFE/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2 # upload rock to docker and microk8s cache, replace charm's container with local rock reference bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ @@ -68,5 +71,7 @@ commands = predictor_servers=$(jq --arg jq_rock $ROCK -r '\''.SKLEARN_SERVER.protocols.seldon.image=$jq_rock'\'' <<< $predictor_servers) && \ predictor_servers=$(jq --arg jq_version $VERSION -r '\''.SKLEARN_SERVER.protocols.seldon.defaultImageVersion=$jq_version'\'' <<< $predictor_servers) && \ yq e -i ".data.predictor_servers=$predictor_servers" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2' + # replace yq safe placeholder with original value + sed -i "s/namespace: YQ_SAFE/namespace: {{ namespace }}/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2 # run charm integration test with rock tox -c {env:LOCAL_CHARM_DIR} -e integration From ff82e1bb5b5cc4fd0b6bcbf4f12f9561e0c7b2a4 Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Thu, 1 Jun 2023 20:23:31 -0400 Subject: [PATCH 07/11] fix; removed keep-models --- sklearnserver/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini index 4a07d8d..8170615 100644 --- a/sklearnserver/tox.ini +++ b/sklearnserver/tox.ini @@ -74,4 +74,4 @@ commands = # replace yq safe placeholder with original value sed -i "s/namespace: YQ_SAFE/namespace: {{ namespace }}/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2 # run charm integration test with rock - tox -c {env:LOCAL_CHARM_DIR} -e integration -- --keep-models + tox -c {env:LOCAL_CHARM_DIR} -e integration From fc4dd79e34deca9af96a38924999a8e4ddf757ed Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Tue, 6 Jun 2023 14:33:25 -0400 Subject: [PATCH 08/11] feat: updated tox to edit configmap Summary of changes: - Updated tox.ini to properly update configmap template. --- sklearnserver/tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini index 8170615..9871460 100644 --- a/sklearnserver/tox.ini +++ b/sklearnserver/tox.ini @@ -69,8 +69,7 @@ commands = microk8s ctr image import $ROCK.tar && \ predictor_servers=$(yq e ".data.predictor_servers" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2) && \ predictor_servers=$(jq --arg jq_rock $ROCK -r '\''.SKLEARN_SERVER.protocols.seldon.image=$jq_rock'\'' <<< $predictor_servers) && \ - predictor_servers=$(jq --arg jq_version $VERSION -r '\''.SKLEARN_SERVER.protocols.seldon.defaultImageVersion=$jq_version'\'' <<< $predictor_servers) && \ - yq e -i ".data.predictor_servers=$predictor_servers" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2' + predictor_servers=$(jq --arg jq_version $VERSION -r '\''.SKLEARN_SERVER.protocols.seldon.defaultImageVersion=$jq_version'\'' <<< $predictor_servers) yq e -i ".data.predictor_servers=strenv(predictor_servers)" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2' # replace yq safe placeholder with original value sed -i "s/namespace: YQ_SAFE/namespace: {{ namespace }}/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2 # run charm integration test with rock From 2c6fc8d396e2007dfc2df63c74a05e4b5e938a62 Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Wed, 14 Jun 2023 17:24:07 -0400 Subject: [PATCH 09/11] feat: update rockcraft with new non root solution https://github.com/canonical/seldon-core-operator/issues/133 Summary of changes: - Updated rockcraft.yaml with new run-user option to run as non-root. - Updated import procedure. - Tested with integration tests on the branch. --- sklearnserver/rockcraft.yaml | 21 ++++++++------------- sklearnserver/tests/test_rock.py | 5 ++++- sklearnserver/tox.ini | 3 ++- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/sklearnserver/rockcraft.yaml b/sklearnserver/rockcraft.yaml index 8773fa2..324b1fb 100644 --- a/sklearnserver/rockcraft.yaml +++ b/sklearnserver/rockcraft.yaml @@ -1,25 +1,27 @@ +# Based on https://github.com/SeldonIO/seldon-core/tree/master/servers/sklearnserver/sklearnserver name: sklearnserver summary: An image for Seldon SKLearn Server description: | This image is used as part of the Charmed Kubeflow product. The SKLearn Server serves models which have been stored as pickles. -version: v1.16.0_20.04_1 # -_ +version: v1.16.0_20.04_1 # __ license: Apache-2.0 base: ubuntu:20.04 +run-user: _daemon_ services: sklearnserver: override: replace summary: "sklearnserver service" startup: enabled # Yet again, use a subshell to jam conda into a working state. Can't use bashrc, because it immediately - # exits if PS1 isn't set, so no-go from scripts - command: bash -c 'cd /microservice && export PATH=/opt/conda/bin/${PATH} && eval $(/opt/conda/bin/conda shell.bash hook 2> /dev/null) && source /opt/conda/etc/profile.d/conda.sh && conda activate && seldon-core-microservice $MODEL_NAME --service-type $SERVICE_TYPE --persistence $PERSISTENCE' + # exits if PS1 isn't set, so no-go from scripts. + command: bash -c 'cd /microservice && export PATH=/opt/conda/bin/:${PATH} && eval $(/opt/conda/bin/conda shell.bash hook 2> /dev/null) && source /opt/conda/etc/profile.d/conda.sh && conda activate && seldon-core-microservice ${MODEL_NAME} --service-type ${SERVICE_TYPE} &> /tmp/log.txt' environment: + # the following environment variables are taken from: + # https://github.com/SeldonIO/seldon-core/blob/master/servers/sklearnserver/environment + # NOTE: PERSISTENCE is omitted because it is depricated MODEL_NAME: "SKLearnServer" SERVICE_TYPE: "MODEL" - PERSISTENCE: "0" - user: ubuntu - platforms: amd64: @@ -78,13 +80,6 @@ parts: # but it does need to match pebble's workdir install -D -m 755 ${CRAFT_STAGE}/microservice/SKLearnServer.py microservice/SKLearnServer.py - non-root-user: - plugin: nil - overlay-script: | - # Create a user in the $CRAFT_OVERLAY chroot - groupadd -R $CRAFT_OVERLAY -g 1001 ubuntu - useradd -R $CRAFT_OVERLAY -M -r -u 1001 -g ubuntu ubuntu - security-team-requirement: plugin: nil after: [sklearnserver] diff --git a/sklearnserver/tests/test_rock.py b/sklearnserver/tests/test_rock.py index b087051..ff5e9c0 100644 --- a/sklearnserver/tests/test_rock.py +++ b/sklearnserver/tests/test_rock.py @@ -32,7 +32,10 @@ def test_rock(ops_test: OpsTest, rock_test_env): """Test rock.""" check_rock = CheckRock("rockcraft.yaml") container_name = rock_test_env - LOCAL_ROCK_IMAGE = check_rock.get_image_name() + LOCAL_ROCK_IMAGE = f"{check_rock.get_image_name()}:{check_rock.get_version()}" # verify that all artifacts are in correct locations subprocess.run(["docker", "run", LOCAL_ROCK_IMAGE, "exec", "ls", "-la", "/microservice/SKLearnServer.py"], check=True) + + # verify that rockcraft.yaml contains correct image name for PREDICTIVE_UNIT_IMAGE environment variable + #assert CheckRock.get_environment()["PREDICTIVE_UNIT_IMAGE"].contains(LOCAL_ROCK_IMAGE) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini index 9871460..8ed8612 100644 --- a/sklearnserver/tox.ini +++ b/sklearnserver/tox.ini @@ -66,7 +66,7 @@ commands = print(CheckRock("rockcraft.yaml").get_version())'\'') && \ sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \ docker save $ROCK > $ROCK.tar && \ - microk8s ctr image import $ROCK.tar && \ + microk8s ctr image import $ROCK.tar --digests=true && \ predictor_servers=$(yq e ".data.predictor_servers" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2) && \ predictor_servers=$(jq --arg jq_rock $ROCK -r '\''.SKLEARN_SERVER.protocols.seldon.image=$jq_rock'\'' <<< $predictor_servers) && \ predictor_servers=$(jq --arg jq_version $VERSION -r '\''.SKLEARN_SERVER.protocols.seldon.defaultImageVersion=$jq_version'\'' <<< $predictor_servers) yq e -i ".data.predictor_servers=strenv(predictor_servers)" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2' @@ -74,3 +74,4 @@ commands = sed -i "s/namespace: YQ_SAFE/namespace: {{ namespace }}/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2 # run charm integration test with rock tox -c {env:LOCAL_CHARM_DIR} -e integration + From 9bd3ee80b440d24529444f4793095089a9c03db8 Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Thu, 15 Jun 2023 11:26:47 -0400 Subject: [PATCH 10/11] feat: reverted to use shell and yq vs test class Summary of changes: - Reverted to use of shell commands and `yq` instead of CheckRock test class from chisme package. NOTE: Use of bash shell commands significantly reduces maintability of tox.ini --- sklearnserver/tox.ini | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini index 8ed8612..2575ce0 100644 --- a/sklearnserver/tox.ini +++ b/sklearnserver/tox.ini @@ -27,10 +27,10 @@ deps = commands = # build and pack rock rockcraft pack - bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ - print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ - VERSION=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ - print(CheckRock("rockcraft.yaml").get_version())'\'') && \ + bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ + VERSION=$(yq eval .version rockcraft.yaml) && \ + ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \ + ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \ sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION' # run rock tests @@ -60,10 +60,10 @@ commands = # replace jinja2 templated value with yq safe placeholder sed -i "s/namespace: {{ namespace }}/namespace: YQ_SAFE/" {env:LOCAL_CHARM_DIR}/src/templates/configmap.yaml.j2 # upload rock to docker and microk8s cache, replace charm's container with local rock reference - bash -c 'ROCK=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ - print(CheckRock("rockcraft.yaml").get_image_name())'\'') && \ - VERSION=$(python -c '\''from charmed_kubeflow_chisme.rock import CheckRock; \ - print(CheckRock("rockcraft.yaml").get_version())'\'') && \ + bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ + VERSION=$(yq eval .version rockcraft.yaml) && \ + ARCH=$(yq eval ".platforms | keys" rockcraft.yaml | awk -F " " '\''{ print $2 }'\'') && \ + ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \ sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \ docker save $ROCK > $ROCK.tar && \ microk8s ctr image import $ROCK.tar --digests=true && \ From 1a04e69fc1984210ebf285433f861ff8a10f0b8c Mon Sep 17 00:00:00 2001 From: Ivan Chvets Date: Thu, 15 Jun 2023 13:40:20 -0400 Subject: [PATCH 11/11] fix: remove chisme package --- sklearnserver/tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/sklearnserver/tox.ini b/sklearnserver/tox.ini index 2575ce0..8a86bb1 100644 --- a/sklearnserver/tox.ini +++ b/sklearnserver/tox.ini @@ -19,7 +19,6 @@ allowlist_externals = tox rockcraft deps = - charmed_kubeflow_chisme juju~=2.9.0 pytest pytest-operator @@ -46,7 +45,6 @@ allowlist_externals = rockcraft sed deps = - charmed_kubeflow_chisme juju~=2.9.0 pytest pytest-operator