[#1345] System tests for vulnerability

[#1328] Add agent service
[#1328] Generate GRPC API for Python
[#1328] Working GRPC client in Python
[#1328] Read the GetState GRPC response
[#1328] GRPC client class
[#1328] Add method to forward command to Kea
[#1328] Revert "Add agent service"
[#1328] Revert unnecessary changes
[#1328] Generate certs in separate container
[#1328] Fix linter issues
[#1328] Ensure the timestamps are correct
[#1328] Add unit tests
[#1328] Add missing prerequisite
[#1328] Add protoc to system test Dockerfile
[#1328] Disable IPv6 in system test CI
[#1328] Improve docstrings
[#1328] Fix problem with HA pair
[#1328] Fix system tests
[#1328] Fix linter issue
[#1328] Wrap with lamda
[#1328] Fix linter issue
[#1328] Lock asyncio version

.gitignore

@@ -56,14 +56,18 @@ docker/config/agent-kea-many-subnets/kea-dhcp4.conf
 /dist
 **/.bundle/config
 
+# Protoc generated files
+/backend/api/stork/backend/api/agent/protocol/agent.pb.go
+/tests/system/agent_pb2.py
+/tests/system/agent_grpc.py
+
 # tests venv and pycache
 /tests/system/venv/
 /tests/system/test-results/
 .vscode/
 /isc-stork-*.deb
 /isc-stork-*.rpm
 *.code-workspace
-/backend/api/stork/backend/api/agent/protocol/agent.pb.go
 root/
 tests/system/.env/
 test-results/

.gitlab-ci.yml

@@ -592,6 +592,8 @@ system_test:
       - $CI_PROJECT_NAME-compose-$CI_DEFAULT_BRANCH
   before_script:
     - ip -6 route del default
+    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
+    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
     - rake prepare:systemtest
   variables:
     # Alpine uses the libc-musl library, which isn't compatible with the NodeJS

doc/dev/ci-images.rst

@@ -148,11 +148,18 @@ The image names are the file names of their Dockerfiles.
 
 **Tag: 4**
 
-    Introduced in the #1353 ticket to provide new Alpine 3.18 image,
-    which includes updated Go 1.22.2 package.
+    - ``compose.Dockerfile``:
+
+        Introduced in #1328 ticket to add the missing ``protoc`` dependency.
+
+        - Added: protoc 24
+        - Update: NodeJS 20 and NPM 10
 
     - ``alpine.Dockerfile``:
 
+        Introduced in the #1353 ticket to provide new Alpine 3.18 image,
+        which includes updated Go 1.22.2 package.
+
         - Base: ``golang:1.22-alpine3.18``
         - Frozen all dependency versions
         - Updated to Ruby 3.2, Python 3.11, NPM 9.6, Make 4.4, Binutils-gold 2.40

docker/images/ci/compose.Dockerfile

@@ -5,5 +5,6 @@ RUN apk add --no-cache \
     python3~3.11 \
     openssl~3.1 \
     ruby-rake~13 \
-    nodejs~18.17 \
-    npm~9
+    nodejs~20.11 \
+    npm~10 \
+    protoc~24.4

rakelib/00_init.rake

@@ -1064,6 +1064,15 @@ file PYTEST => [PIP] do
 end
 add_hash_guard(PYTEST, pytests_requirements_file)
 
+PROTOC_GEN_PYTHON_GRPC = File.join(python_tools_dir, "bin", "protoc-gen-python_grpc")
+file PROTOC_GEN_PYTHON_GRPC => [PYTEST] do
+    sh "touch", "-c", PROTOC_GEN_PYTHON_GRPC
+    if !File.file?(PROTOC_GEN_PYTHON_GRPC)
+        # This plugin doesn't support version printing.
+        fail
+    end
+end
+
 PIP_COMPILE = File.join(python_tools_dir, "bin", "pip-compile")
 file PIP_COMPILE => [PIP] do
     sh PIP, "install", "pip-tools==#{pip_tools_ver}"

rakelib/10_codebase.rake

@@ -230,6 +230,29 @@ CLEAN.append *FileList["backend/**/*mock_test.go"]
 ### Python ###
 ##############
 
+agent_pb_python_file = "tests/system/agent_pb2.py"
+file agent_pb_python_file => [agent_proto_file, PROTOC, PROTOC_GEN_PYTHON_GRPC] do
+    Dir.chdir("backend/api") do
+        sh PROTOC,
+            "--proto_path=.",
+            "--python_out=.",
+            "--grpc_python_out=.",
+            "--plugin=protoc-gen-grpc_python=#{PROTOC_GEN_PYTHON_GRPC}",
+            "agent.proto"
+    end
+    sh "mv", *FileList["backend/api/*.py"], "tests/system"
+    sh "touch", "-c", agent_pb_python_file
+end
+CLEAN.append agent_pb_python_file
+
+agent_grpc_python_file = "tests/system/agent_grpc.py"
+file agent_grpc_python_file => [agent_pb_python_file] do
+    sh "touch", "-c", agent_grpc_python_file
+end
+CLEAN.append agent_grpc_python_file
+
+GRPC_PYTHON_API_FILES = [agent_pb_python_file, agent_grpc_python_file]
+
 OPEN_API_GENERATOR_PYTHON_DIR = "tests/system/openapi_client"
 file OPEN_API_GENERATOR_PYTHON_DIR => [JAVA, SWAGGER_FILE, OPENAPI_GENERATOR] do
     sh "rm", "-rf", OPEN_API_GENERATOR_PYTHON_DIR

rakelib/30_dev.rake

@@ -460,7 +460,7 @@ namespace :lint do
 
     namespace :python do
         desc 'Runs pylint, python linter tool'
-        task :pylint => [PYLINT, PYTEST, FLASK, OPEN_API_GENERATOR_PYTHON_DIR] do
+        task :pylint => [PYLINT, PYTEST, FLASK, OPEN_API_GENERATOR_PYTHON_DIR, *GRPC_PYTHON_API_FILES] do
             python_files, exit_code = Open3.capture2('git', 'ls-files', '*.py')
             python_files = python_files.split("\n").map{ |string| string.strip }
             puts "Running pylint:"

rakelib/50_system_tests.rake

@@ -144,7 +144,7 @@ desc 'Run system tests
     BIND9_VERSION - use specific BIND9 version - optional, format: MAJOR.MINOR
     POSTGRES_VERSION - use specific Postgres database version - optional
     EXIT_FIRST - exit on the first error - optional, default: false'
-task :systemtest => [PYTEST, DOCKER_COMPOSE, OPEN_API_GENERATOR_PYTHON_DIR, *volume_files, "systemtest:setup_version_envvars"] do
+task :systemtest => [PYTEST, DOCKER_COMPOSE, OPEN_API_GENERATOR_PYTHON_DIR, *GRPC_PYTHON_API_FILES, *volume_files, "systemtest:setup_version_envvars"] do
     opts = []
 
     if !ENV["TEST"].nil?
@@ -241,7 +241,7 @@ namespace :systemtest do
     end
 
     desc 'List the test cases'
-    task :list => [PYTEST, OPEN_API_GENERATOR_PYTHON_DIR] do
+    task :list => [PYTEST, OPEN_API_GENERATOR_PYTHON_DIR, *OPEN_API_GENERATOR_PYTHON_DIR] do
         Dir.chdir(system_tests_dir) do
             sh PYTEST, "--collect-only"
         end
@@ -366,6 +366,9 @@ namespace :gen do
 
         desc 'Generate Swagger API files'
         task :swagger => [OPEN_API_GENERATOR_PYTHON_DIR]
+
+        desc 'Generate GRPC API files'
+        task :grpc => GRPC_PYTHON_API_FILES
     end
 end
 

rakelib/init_deps/pytest.in

@@ -4,3 +4,6 @@ pyyaml
 python-dateutil
 pydantic
 urllib3
+protobuf
+grpclib
+asyncio

rakelib/init_deps/pytest.txt

@@ -6,14 +6,28 @@
 #
 annotated-types==0.7.0
     # via pydantic
+asyncio==3.4.3
+    # via -r rakelib/init_deps/pytest.in
 exceptiongroup==1.2.1
     # via pytest
+grpclib==0.4.7
+    # via -r rakelib/init_deps/pytest.in
+h2==4.1.0
+    # via grpclib
+hpack==4.0.0
+    # via h2
+hyperframe==6.0.1
+    # via h2
 iniconfig==2.0.0
     # via pytest
+multidict==6.0.5
+    # via grpclib
 packaging==24.0
     # via pytest
 pluggy==1.5.0
     # via pytest
+protobuf==4.25.3
+    # via -r rakelib/init_deps/pytest.in
 pydantic==2.7.2
     # via -r rakelib/init_deps/pytest.in
 pydantic-core==2.18.3

tests/system/core/compose.py

@@ -453,6 +453,31 @@ def port(self, service_name, port) -> Tuple[str, int]:
             mapped_host = self._default_mapped_hostname
         return mapped_host, mapped_port
 
+    def copy_from_service(self, service_name, source, destination):
+        """
+        Copies a file or directory from a service container to the host.
+
+        Parameters
+        ----------
+        service_name: str
+            Name of the docker compose service
+        source: str
+            The path to the file or directory in the container
+        destination: str
+            The path to the destination on the host
+        """
+        if not os.path.isabs(source):
+            raise ValueError("source must be an absolute path")
+        if not os.path.isabs(destination):
+            raise ValueError("destination must be an absolute path")
+
+        cp_cmd = self.docker_compose_command() + [
+            "cp",
+            f"{service_name}:{source}",
+            destination,
+        ]
+        self._call_command(cmd=cp_cmd)
+
     def get_service_ip_address(self, service_name, network_name, family):
         """
         Returns the assigned IP address for one of the services.