Some minor docker improvements

docker/Makefile

@@ -25,6 +25,8 @@
 PDO_REPO ?= ${PDO_SOURCE_ROOT}
 PDO_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 
+PDO_VERSION ?= $$( if [ -d repository ]; then cd repository; bin/get_version; else echo "latest"; fi )
+
 # PDO_USER_UID, PDO_GROUP_UID --
 #   The UID/GID for the user accounts that will be created in the container; this is
 #   important especially when the containers are being used for development; note
@@ -38,11 +40,9 @@ PDO_GROUP_UID ?= $(shell id -g)
 # Turns out that setting the script directory from the Makefile's name
 # does not work very well if the path contains spaces in the name; in
 # fact lots of things break very badly; set this explicitly
-SCRIPT_DIR ?= ${PDO_SOURCE_ROOT}/docker
+DOCKER_DIR ?= ${PDO_SOURCE_ROOT}/docker
 
 DOCKER_USERNAME = $(LOGNAME)
-DOCKER_BUILDARGS += --build-arg PDO_HOSTNAME=$(PDO_HOSTNAME)
-DOCKER_BUILDARGS += --build-arg PDO_LEDGER_URL=$(PDO_LEDGER_URL)
 DOCKER_BUILDARGS += --build-arg UID=$(PDO_USER_UID)
 DOCKER_BUILDARGS += --build-arg GID=$(PDO_GROUP_UID)
 DOCKER_ARGS = $(DOCKER_BUILDARGS)
@@ -56,32 +56,50 @@ TIMESTAMP := $(shell /bin/date "+%Y%m%d%H%M%S")
 all : $(addprefix build_,$(IMAGES))
 
 rebuild_% : repository
-	@ docker build $(DOCKER_ARGS) \
+	docker build $(DOCKER_ARGS) \
 		--build-arg REBUILD=$(TIMESTAMP) \
-		--build-arg PDO_VERSION=$(shell cd repository; bin/get_version) \
-		--tag pdo_$*:$(shell cd repository; bin/get_version) \
-		--file $(SCRIPT_DIR)/pdo_$*.dockerfile .
+		--build-arg PDO_VERSION=$(PDO_VERSION) \
+		--tag pdo_$*:$(PDO_VERSION) \
+		--file $(DOCKER_DIR)/pdo_$*.dockerfile .
 
 build_% : repository
-	@ docker build $(DOCKER_ARGS) \
-		--build-arg PDO_VERSION=$(shell cd repository; bin/get_version) \
-		--tag pdo_$*:$(shell cd repository; bin/get_version) \
-		--file $(SCRIPT_DIR)/pdo_$*.dockerfile .
+	docker build $(DOCKER_ARGS) \
+		--build-arg PDO_VERSION=$(PDO_VERSION) \
+		--tag pdo_$*:$(PDO_VERSION) \
+		--file $(DOCKER_DIR)/pdo_$*.dockerfile .
+
+# docker build dependencies
+build_client: build_base
+build_services: build_services_base
+build_services_base: build_base
+build_ccf: build_ccf_base
 
 clean_% :
-	@ docker rmi -f pdo_$*
+	docker rmi -f pdo_$*:$(PDO_VERSION)
+
+DOCKER_RUN_ARGS = -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer
+DOCKER_RUN_ARGS += --network host
+
+run_ccf : build_ccf stop_ccf
+	docker run $(DOCKER_RUN_ARGS) --name ccf_container -P -d pdo_ccf:$(PDO_VERSION)
+
+run_services : build_base build_services_base build_services stop_services
+	docker run $(DOCKER_RUN_ARGS) --name services_container -P -d pdo_services:$(PDO_VERSION)
 
-run_ccf : build_ccf
+run_client : build_base build_client stop_client
+	docker run $(DOCKER_RUN_ARGS) -it --name client_container pdo_client:$(PDO_VERSION) \
+		-c "stty cols $$(tput cols) rows $$(tput lines) && bash"
+
+stop_all : stop_ccf stop_services stop_client
+
+stop_ccf :
 	- docker rm -f ccf_container
-	@ docker run -v $(SCRIPT_DIR)/xfer/:/project/pdo/xfer --network host --name ccf_container -P -d pdo_ccf
 
-run_services : build_base build_services_base build_services
+stop_services :
 	- docker rm -f services_container
-	@ docker run -v $(SCRIPT_DIR)/xfer/:/project/pdo/xfer --network host --name services_container -P -d pdo_services
 
-run_client : build_base build_client
+stop_client :
 	- docker rm -f client_container
-	@ docker run -v $(SCRIPT_DIR)/xfer/:/project/pdo/xfer -it --network host --name client_container pdo_client
 
 # -----------------------------------------------------------------
 # We need a repository with the source for the branch we are going
@@ -91,10 +109,10 @@ run_client : build_base build_client
 # performance requirements are relatively low.
 # -----------------------------------------------------------------
 repository :
-	@ git clone --single-branch --branch $(PDO_BRANCH) --recurse-submodules $(PDO_REPO) repository
+	git clone --single-branch --branch $(PDO_BRANCH) --recurse-submodules $(PDO_REPO) repository
 
 clean_repository :
-	@ rm -rf repository
+	rm -rf repository
 
 # -----------------------------------------------------------------
 # Testing uses docker-compose to build a set of images that will be configured
@@ -109,14 +127,11 @@ TEST_FILES += -f services_base.yaml
 TEST_FILES += -f ccf_base.yaml
 TEST_FILES += -f test.yaml
 
-build_test : repository
-	PDO_USER_UID=$(PDO_USER_UID) PDO_GROUP_UID=$(PDO_GROUP_UID) \
-		docker-compose $(TEST_FILES) build \
-			--build-arg PDO_VERSION=$(shell cd repository; bin/get_version)
+build_test : repository build_services build_ccf build_client
 
-test : clean_config clean_repository build_base build_services_base build_ccf_base build_test
-	docker-compose $(TEST_FILES) up --abort-on-container-exit
-	docker-compose $(TEST_FILES) down
+test : clean_config clean_repository build_test stop_all
+	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) up --abort-on-container-exit
+	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) down
 
 # -----------------------------------------------------------------
 # Cleaning is a bit interesting because the containers don't go away
@@ -127,16 +142,21 @@ test : clean_config clean_repository build_base build_services_base build_ccf_ba
 _IMAGES_=$(shell docker images -a --filter=dangling=true -q)
 _CONTAINERS_=$(shell docker ps --filter=status=exited --filter=status=created -q)
 clean_images : $(addprefix clean_,$(IMAGES))
-	@ if [ ! -z "$(_CONTAINERS_)" ]; then docker rm -f $(_CONTAINERS_); fi
-	@ if [ ! -z "$(_IMAGES_)" ]; then docker rmi -f $(_IMAGES_); fi
+	if [ ! -z "$(_CONTAINERS_)" ]; then docker rm -f $(_CONTAINERS_); fi
+	if [ ! -z "$(_IMAGES_)" ]; then docker rmi -f $(_IMAGES_); fi
 
 clean_config :
-	@ rm -f $(SCRIPT_DIR)/xfer/ccf/keys/*.pem $(SCRIPT_DIR)/xfer/ccf/etc/*.toml
-	@ rm -f $(SCRIPT_DIR)/xfer/services/keys/*.pem $(SCRIPT_DIR)/xfer/services/etc/*.toml
-	@ rm -f $(SCRIPT_DIR)/xfer/services/etc/site.psh
+	rm -f $(DOCKER_DIR)/xfer/ccf/keys/*.pem $(DOCKER_DIR)/xfer/ccf/etc/*.toml
+	rm -f $(DOCKER_DIR)/xfer/services/keys/*.pem $(DOCKER_DIR)/xfer/services/etc/*.toml
+	rm -f $(DOCKER_DIR)/xfer/services/etc/site.psh
 
 clean : clean_images clean_config clean_repository
 
 .PHONY: clean clean_images clean_config clean_repository
 .PHONY: build_test test
 .PHONY: run_ccf run_client run_services
+
+# -----------------------------------------------------------------
+# Include additional targets to simplify development inside containers
+# -----------------------------------------------------------------
+include make.dev

docker/base.yaml

@@ -17,7 +17,7 @@ version: "3.4"
 
 services:
   base_container:
-    image: pdo_base
-    build:
-      context: .
-      dockerfile: pdo_base.dockerfile
+    image: pdo_base:${PDO_VERSION:-latest}
+    # build containers only via Makefile and directly via docker build
+    # as we use containers also outside of docker-compose to prevent
+    # double-build

docker/ccf_base.yaml

@@ -16,16 +16,10 @@ version: "3.4"
 
 services:
   ccf_container:
-    image: pdo_ccf
-    build:
-      context: .
-      dockerfile: pdo_ccf.dockerfile
-      args:
-        PDO_INTERPRETER: ${PDO_INTERPRETER:-wawaka}
-        PDO_LEDGER_TYPE: ccf
-        PDO_HOSTNAME: ${PDO_HOSTNAME}
-        PDO_LEDGER_URL: ${PDO_LEDGER_URL}
-        SGX_MODE: ${SGX_MODE:-SIM}
+    image: pdo_ccf:${PDO_VERSION:-latest}
+    # build containers only via Makefile and directly via docker build
+    # as we use containers also outside of docker-compose to prevent
+    # double-build
     container_name: ccf_container
     network_mode: "host"
     volumes:

docker/client_base.yaml

@@ -16,19 +16,9 @@ version: "3.4"
 
 services:
   client_container:
-    image: pdo_client
-    build:
-      context: .
-      dockerfile: pdo_client.dockerfile
-      args:
-        UID: ${PDO_USER_UID:-1000}
-        GID: ${PDO_GROUP_UID:-1000}
-        PDO_INTERPRETER: ${PDO_INTERPRETER:-wawaka}
-        PDO_LEDGER_TYPE: ccf
-        PDO_HOSTNAME: ${PDO_HOSTNAME}
-        PDO_LEDGER_URL: ${PDO_LEDGER_URL}
-        PDO_LOG_LEVEL: ${PDO_LOG_LEVEL:-info}
-        PDO_DEBUG_BUILD: ${PDO_DEBUG_BUILD:-0}
+    image: pdo_client:${PDO_VERSION:-latest}
+    # build containers only via Makefile and directly via docker build
+    # as we use containers also outside of docker-compose to prevent
     depends_on:
       - ccf_container
       - services_container

docker/make.dev

@@ -0,0 +1,73 @@
+# ------------------------------------------------------------------------------
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ------------------------------------------------------------------------------
+
+# This file, included in main Makefile, contains targets that are useful for running
+# docker containers for development. Note that these are highly specific to individual
+# developer tastes and can be modified as appropriate.
+
+DOCKER_DEV_CMD = stty cols $$(tput cols) rows $$(tput lines) && bash
+
+DOCKER_DEV_ARGS = $(DOCKER_RUN_ARGS)
+DOCKER_DEV_ARGS += -v $(PDO_SOURCE_ROOT)/docker/tools/:/project/pdo/tools
+DOCKER_DEV_ARGS += -v $(PDO_SOURCE_ROOT)/:/project/pdo/src
+DOCKER_DEV_ARGS += --entrypoint=/bin/bash -it --rm
+
+# sharing a pip download cache on the host can be beneficial if you have to
+# restart your container, e.g., for testing client contracts with large python
+# dependencies. By default, we create a local cache dir but you can have it anywhere,
+# e.g., in /tmp or even pointing ~/.cache/pip (although be aware that not pip does
+# not necessarily guarantee cross-version compatibility!)
+PIP_CACHE_DIR   ?= $(DOCKER_DIR)/cache/pip
+DOCKER_DEV_ARGS += -v $(PIP_CACHE_DIR):/project/pdo/.cache/pip
+
+# Docker runtime args specific to client
+DOCKER_CLIENT_DEV_ARGS ?= 
+# - if you want pdo-contracts mounted in client dev image, define (e.g., in make.loc)
+#   variable 'DOCKER_CLIENT_WITH_PDO_CONTRACTS'
+ifdef DOCKER_CLIENT_WITH_PDO_CONTRACTS
+	DOCKER_CLIENT_DEV_ARGS += -v $(DOCKER_DIR)/../../pdo-contracts.git:/project/pdo/contracts
+endif
+
+run_ccf_dev : build_ccf
+	docker run $(DOCKER_DEV_ARGS) --name ccf_container_dev \
+		-P pdo_ccf:$(PDO_VERSION) -c "$(DOCKER_DEV_CMD)"
+
+run_services_dev : build_base build_services_base build_services
+	docker run $(DOCKER_DEV_ARGS) --name services_container_dev \
+		-P pdo_services:$(PDO_VERSION) -c "$(DOCKER_DEV_CMD)"
+
+run_client_dev : build_base build_client
+	docker run $(DOCKER_CLIENT_DEV_ARGS) $(DOCKER_DEV_ARGS) --name client_container_dev pdo_client:$(PDO_VERSION) \
+		-c "$(DOCKER_DEV_CMD)"
+
+stop_all : stop_ccf_dev stop_services_dev stop_client_dev
+
+stop_ccf_dev :
+	- docker rm -f ccf_container_dev
+
+stop_services_dev :
+	- docker rm -f services_container_dev
+
+stop_client_dev :
+	- docker rm -f client_container_dev
+
+test_no_reset : build_test stop_all
+	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) up --abort-on-container-exit
+	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) down
+
+.PHONY: test_no_reset
+.PHONY: stop_ccf_dev stop_services_dev stop_client_dev
+.PHONY: run_ccf_dev run_services_dev run_client_dev

docker/pdo_ccf.dockerfile

@@ -25,9 +25,9 @@ FROM pdo_ccf_base:${PDO_VERSION}
 # -----------------------------------------------------------------
 # set up the PDO sources
 # -----------------------------------------------------------------
-ARG REBUILD 0
+ARG REBUILD=0
 
-ARG SGX_MODE SIM
+ARG SGX_MODE=SIM
 ENV SGX_MODE=$SGX_MODE
 
 ARG PDO_DEBUG_BUILD=0
@@ -57,13 +57,6 @@ RUN --mount=type=cache,uid=${UID},gid=${GID},target=/project/pdo/.cache/pip \
 # Network ports for running services
 EXPOSE 6600
 
-ARG PDO_HOSTNAME
-ENV PDO_HOSTNAME=$PDO_HOSTNAME
-
-ARG PDO_LEDGER_URL
-ENV PDO_LEDGER_URL=$PDO_LEDGER_URL
-
-# Note that the entry point when specified with exec syntax
 # can be extended through the docker run interface far more
 # easily than if you use the other specification format of
 # a single string

docker/pdo_ccf_base.dockerfile

@@ -39,7 +39,8 @@ RUN apt-get update \
         python3-virtualenv \
         virtualenv \
         net-tools \
-        wget
+        wget \
+        ${ADD_APT_PKGS}
 
 RUN echo "deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu ${UBUNTU_NAME} main" >> /etc/apt/sources.list
 RUN curl https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add -

docker/pdo_client.dockerfile

@@ -40,7 +40,7 @@ USER $UNAME
 # -----------------------------------------------------------------
 # set up the PDO sources
 # -----------------------------------------------------------------
-ARG REBUILD 0
+ARG REBUILD=0
 
 ARG PDO_DEBUG_BUILD=0
 ENV PDO_DEBUG_BUILD=${PDO_DEBUG_BUILD}
@@ -73,13 +73,5 @@ ARG GID=${UID}
 RUN --mount=type=cache,uid=${UID},gid=${GID},target=/project/pdo/.cache/pip \
     /project/pdo/tools/build_client.sh
 
-ARG PDO_HOSTNAME
-ENV PDO_HOSTNAME=$PDO_HOSTNAME
-
-ARG PDO_LEDGER_URL
-ENV PDO_LEDGER_URL=$PDO_LEDGER_URL
-
-# the client is set up for interactive access; the environment can be
-# set up by source /project/pdo/tools/start_client.sh with the arguments
-# to build a new client environment or copy one from the xfer directory
+RUN ln -s /project/pdo/tools/bashrc_client.sh /project/pdo/.bashrc
 ENTRYPOINT [ "/bin/bash" ]

docker/pdo_services.dockerfile

@@ -25,7 +25,7 @@ FROM pdo_services_base:${PDO_VERSION}
 # -----------------------------------------------------------------
 # set up the PDO sources
 # -----------------------------------------------------------------
-ARG REBUILD 0
+ARG REBUILD=0
 
 ARG PDO_DEBUG_BUILD=0
 ENV PDO_DEBUG_BUILD=${PDO_DEBUG_BUILD}
@@ -63,11 +63,6 @@ EXPOSE 7001 7002 7003 7004 7005
 EXPOSE 7101 7102 7103 7104 7105
 EXPOSE 7201 7202 7203 7204 7205
 
-ARG PDO_HOSTNAME
-ENV PDO_HOSTNAME=$PDO_HOSTNAME
-
-ARG PDO_LEDGER_URL
-ENV PDO_LEDGER_URL=$PDO_LEDGER_URL
 
 # Note that the entry point when specified with exec syntax
 # can be extended through the docker run interface far more

docker/services_base.yaml

@@ -17,18 +17,9 @@ version: "3.4"
 
 services:
   services_container:
-    image: pdo_services
-    build:
-      context: .
-      dockerfile: pdo_services.dockerfile
-      args:
-        PDO_INTERPRETER: ${PDO_INTERPRETER:-wawaka}
-        PDO_HOSTNAME: ${PDO_HOSTNAME}
-        PDO_LEDGER_TYPE: ccf
-        PDO_LEDGER_URL: ${PDO_LEDGER_URL}
-        SGX_MODE: ${SGX_MODE:-SIM}
-        PDO_LOG_LEVEL: ${PDO_LOG_LEVEL:-info}
-        PDO_DEBUG_BUILD: ${PDO_DEBUG_BUILD:-0}
+    image: pdo_services:${PDO_VERSION:-latest}
+    # build containers only via Makefile and directly via docker build
+    # as we use containers also outside of docker-compose to prevent
     container_name: services_container
     network_mode: "host"
     depends_on: