Skip to content

Commit

Permalink
Merge pull request #172 from willcl-ark/docker-images
Browse files Browse the repository at this point in the history
  • Loading branch information
willcl-ark authored Oct 30, 2023
2 parents ed252bb + ad3b5eb commit 417a6a7
Show file tree
Hide file tree
Showing 21 changed files with 363 additions and 191 deletions.
6 changes: 0 additions & 6 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,6 @@ jobs:
- uses: actions/checkout@v3
- uses: ./.github/actions
- run: ./test/scenarios_test.py
onion:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions
- run: ./test/v25_net_test.py
rpc:
runs-on: ubuntu-latest
steps:
Expand Down
68 changes: 68 additions & 0 deletions scripts/build_images.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#!/usr/bin/env bash

# Create a new builder to enable building multi-platform images
docker buildx create --use

# Image and Registry info
DOCKER_REGISTRY="bitcoindevproject/bitcoin-core"
BITCOIN_URL="https://bitcoincore.org/bin"
echo "DOCKER_REGISTRY=$DOCKER_REGISTRY"
echo "BITCOIN_URL=$BITCOIN_URL"

# Map internal architectures to the format used in the URLs
declare -A ARCH_MAP=(
["amd64"]="x86_64-linux-gnu"
["arm64"]="aarch64-linux-gnu"
["armhf"]="arm-linux-gnueabihf"
# ["amd64-darwin"]="x86_64-apple-darwin"
# ["arm64-darwin"]="arm64-apple-darwin"
)

# Tags and their supported architectures
declare -A VERSION_ARCH_MAP=(
["0.21.2"]="amd64 arm64 armhf"
["22.1"]="amd64 arm64 armhf"
["23.2"]="amd64 arm64 armhf"
["24.2"]="amd64 arm64 armhf"
["25.1"]="amd64 arm64 armhf"
)

if [ -d "src/templates" ]; then
cd src/templates || exit 1
else
echo "Directory src/templates does not exist. Please run this script from the project root."
exit 1
fi

# Loop through each tag and its architectures to build and push
for VERSION in "${!VERSION_ARCH_MAP[@]}"; do
IFS=' ' read -ra ARCHS <<< "${VERSION_ARCH_MAP[$VERSION]}"
IMAGES_LIST=() # Array to store images for manifest
for DOCKER_ARCH in "${ARCHS[@]}"; do
echo "VERSION=$VERSION"
echo "DOCKER_ARCH=$DOCKER_ARCH"

# Map the architecture to the URL format
URL_ARCH=${ARCH_MAP[$DOCKER_ARCH]}
echo "URL_ARCH=$URL_ARCH"

IMAGE_TAG="$VERSION-$DOCKER_ARCH"
IMAGE_FULL_NAME="$DOCKER_REGISTRY:$IMAGE_TAG"
echo "IMAGE_FULL_NAME=$IMAGE_FULL_NAME"

# Use Buildx to build the image for the specified architecture
docker buildx build --platform linux/"$DOCKER_ARCH" \
--provenance=false \
--build-arg ARCH="$URL_ARCH" \
--build-arg BITCOIN_VERSION="$VERSION" \
--build-arg BITCOIN_URL="$BITCOIN_URL" \
--tag "$IMAGE_FULL_NAME" \
. --push

IMAGES_LIST+=("$IMAGE_FULL_NAME")
done

# Create the manifest list for each version under the same repository
MANIFEST_TAG="$DOCKER_REGISTRY:$VERSION"
docker buildx imagetools create --tag "$MANIFEST_TAG" "${IMAGES_LIST[@]}"
done
48 changes: 24 additions & 24 deletions src/graphs/default.graphml
Original file line number Diff line number Diff line change
Expand Up @@ -5,52 +5,52 @@
<key attr.name="build_args" attr.type="string" for="node" id="build_args"/>
<graph edgedefault="directed">
<node id="0">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w0</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w0</data>
</node>
<node id="1">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w1</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w1</data>
</node>
<node id="2">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w2</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w2</data>
</node>
<node id="3">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w3</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w3</data>
</node>
<node id="4">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w4</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w4</data>
</node>
<node id="5">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w5</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w5</data>
</node>
<node id="6">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w6</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w6</data>
</node>
<node id="7">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w7</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w7</data>
</node>
<node id="8">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w8</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w8</data>
</node>
<node id="9">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w9</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w9</data>
</node>
<node id="10">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w10</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w10</data>
</node>
<node id="11">
<data key="version">25.0</data>
<data key="bitcoin_config">uacomment=w11</data>
<data key="version">25.1</data>
<data key="bitcoin_config">-uacomment=w11</data>
</node>
<!-- connect the nodes in a ring to start -->
<edge id="1" source="0" target="1"></edge>
Expand Down
108 changes: 33 additions & 75 deletions src/interfaces/docker_interface.py
Original file line number Diff line number Diff line change
@@ -1,31 +1,28 @@
import logging
import re
import shutil
import subprocess
import yaml

from datetime import datetime
from copy import deepcopy
from pathlib import Path
from typing import cast
import docker
from docker.models.containers import Container

from .interfaces import ContainerInterface
from warnet.utils import bubble_exception_str, parse_raw_messages
from services.tor import Tor
from services.fork_observer import ForkObserver
from services.fluentd import Fluentd
from templates import TEMPLATES
from warnet.tank import Tank, CONTAINER_PREFIX_BITCOIND
from warnet.utils import bubble_exception_str, parse_raw_messages, parse_bitcoin_conf, dump_bitcoin_conf, get_architecture, set_execute_permission
from warnet.utils import bubble_exception_str, parse_raw_messages, default_bitcoin_conf_args, set_execute_permission


DOCKER_COMPOSE_NAME = "docker-compose.yml"
DOCKERFILE_NAME = "Dockerfile"
TORRC_NAME = "torrc"
WARNET_ENTRYPOINT_NAME = "warnet_entrypoint.sh"
DOCKER_ENTRYPOINT_NAME = "docker_entrypoint.sh"
ENTRYPOINT_NAME = "entrypoint.sh"
DOCKER_REGISTRY = "bitcoindevproject/bitcoin-core"

logger = logging.getLogger("docker-interface")
logging.getLogger("docker.utils.config").setLevel(logging.WARNING)
Expand Down Expand Up @@ -121,12 +118,12 @@ def get_bitcoin_debug_log(self, container_name: str):
)
return cast(bytes, logs).decode('utf8') # cast for typechecker

def get_bitcoin_cli(self, container_name: str, method: str, params=None):
def get_bitcoin_cli(self, tank: Tank, method: str, params=None):
if params:
cmd = f"bitcoin-cli {method} {' '.join(map(str, params))}"
cmd = f"bitcoin-cli -regtest -rpcuser={tank.rpc_user} -rpcport={tank.rpc_port} -rpcpassword={tank.rpc_password} {method} {' '.join(map(str, params))}"
else:
cmd = f"bitcoin-cli {method}"
return self.exec_run(container_name, cmd, user="bitcoin")
cmd = f"bitcoin-cli -regtest -rpcuser={tank.rpc_user} -rpcport={tank.rpc_port} -rpcpassword={tank.rpc_password} {method}"
return self.exec_run(tank.container_name, cmd, user="bitcoin")

def get_messages(self, a_name: str, b_ipv4: str, bitcoin_network: str = "regtest"):
src_node = self.get_container(a_name)
Expand Down Expand Up @@ -179,13 +176,6 @@ def logs_grep(self, pattern: str, container_name: str):

return '\n'.join(matching_logs)

def _write_bitcoin_confs(self, warnet):
with open(TEMPLATES / "bitcoin.conf", "r") as file:
text = file.read()
base_bitcoin_conf = parse_bitcoin_conf(text)
for tank in warnet.tanks:
self.write_bitcoin_conf(tank,base_bitcoin_conf)

def _write_docker_compose(self, warnet):
compose = {
"version": "3.8",
Expand All @@ -209,7 +199,6 @@ def _write_docker_compose(self, warnet):
# Prometheus(warnet.network_name, self.config_dir),
# NodeExporter(warnet.network_name),
# Grafana(warnet.network_name),
Tor(warnet.network_name, TEMPLATES),
ForkObserver(warnet.network_name, warnet.fork_observer_config),
Fluentd(warnet.network_name, warnet.config_dir),
]
Expand All @@ -229,58 +218,28 @@ def _write_docker_compose(self, warnet):
)

def generate_deployment_file(self, warnet):
self._write_bitcoin_confs(warnet)
self._write_docker_compose(warnet)
warnet.deployment_file = warnet.config_dir / DOCKER_COMPOSE_NAME


def write_bitcoin_conf(self, tank, base_bitcoin_conf):
conf = deepcopy(base_bitcoin_conf)
options = tank.conf.split(",")
for option in options:
option = option.strip()
if option:
if "=" in option:
key, value = option.split("=")
else:
key, value = option, "1"
conf[tank.bitcoin_network].append((key, value))

conf[tank.bitcoin_network].append(("rpcuser", tank.rpc_user))
conf[tank.bitcoin_network].append(("rpcpassword", tank.rpc_password))
conf[tank.bitcoin_network].append(("rpcport", tank.rpc_port))

conf_file = dump_bitcoin_conf(conf)
path = tank.config_dir / f"bitcoin.conf"
logger.info(f"Wrote file {path}")
with open(path, "w") as file:
file.write(conf_file)
tank.conf_file = path

def copy_torrc(self, tank):
shutil.copyfile(Path(TEMPLATES) / TORRC_NAME, tank.config_dir/ TORRC_NAME)

def copy_entrypoints(self, tank):
shutil.copyfile(Path(TEMPLATES) / WARNET_ENTRYPOINT_NAME, tank.config_dir / WARNET_ENTRYPOINT_NAME)
set_execute_permission(tank.config_dir / WARNET_ENTRYPOINT_NAME)

shutil.copyfile(Path(TEMPLATES) / DOCKER_ENTRYPOINT_NAME, tank.config_dir / DOCKER_ENTRYPOINT_NAME)
set_execute_permission(tank.config_dir / DOCKER_ENTRYPOINT_NAME)

def copy_dockerfile(self, tank):
shutil.copyfile(Path(TEMPLATES) / DOCKERFILE_NAME, tank.config_dir / DOCKERFILE_NAME)
def default_config_args(self, tank):
defaults = default_bitcoin_conf_args()
defaults += f" -rpcuser={tank.rpc_user}"
defaults += f" -rpcpassword={tank.rpc_password}"
defaults += f" -rpcport={tank.rpc_port}"
return defaults

def copy_configs(self, tank):
self.copy_torrc(tank)
self.copy_entrypoints(tank)
self.copy_dockerfile(tank)
import shutil
shutil.copyfile(TEMPLATES / DOCKERFILE_NAME, tank.config_dir / DOCKERFILE_NAME)
shutil.copyfile(TEMPLATES / TORRC_NAME, tank.config_dir / TORRC_NAME)
shutil.copyfile(TEMPLATES / ENTRYPOINT_NAME, tank.config_dir / ENTRYPOINT_NAME)
set_execute_permission(tank.config_dir / ENTRYPOINT_NAME)

def add_services(self, tank, services):
assert tank.index is not None
assert tank.conf_file is not None
services[tank.container_name] = {}

self.copy_configs(tank)
logger.debug(f"{tank.version=}")

# Setup bitcoind, either release binary or build from source
if "/" and "#" in tank.version:
Expand All @@ -295,22 +254,18 @@ def add_services(self, tank, services):
"BUILD_ARGS": f"{tank.DEFAULT_BUILD_ARGS + tank.extra_build_args}",
},
}
services[tank.container_name]['build'] = build
self.copy_configs(tank)
else:
# assume it's a release version, get the binary
build = {
"context": str(tank.config_dir),
"dockerfile": str(tank.config_dir / DOCKERFILE_NAME),
"args": {
"ARCH": get_architecture(),
"BITCOIN_URL": "https://bitcoincore.org/bin",
"BITCOIN_VERSION": f"{tank.version}",
},
}
# Add the bitcoind service
image = f"{DOCKER_REGISTRY}:{tank.version}"
services[tank.container_name]['image'] = image
# Add common bitcoind service details
services[tank.container_name].update(
{
"container_name": tank.container_name,
"build": build,
"environment": {
"BITCOIN_ARGS": self.default_config_args(tank)
},
"networks": {
tank.network_name: {
"ipv4_address": f"{tank.ipv4}",
Expand All @@ -328,7 +283,7 @@ def add_services(self, tank, services):
"test": ["CMD", "pidof", "bitcoind"],
"interval": "5s", # Check every 5 seconds
"timeout": "1s", # Give the check 1 second to complete
"start_period": "5s", # Start checking after 2 seconds
"start_period": "5s", # Start checking after 5 seconds
"retries": 3
},
"logging": {
Expand Down Expand Up @@ -377,15 +332,18 @@ def warnet_from_deployment(self, warnet):

def tank_from_deployment(self, service, warnet):
rex = fr"{warnet.network_name}_{CONTAINER_PREFIX_BITCOIND}_([0-9]{{6}})"
# Not a tank, maybe a scaled service
if not "container_name" in service:
return None
match = re.match(rex, service["container_name"])
if match is None:
return None

index = int(match.group(1))
t = Tank(index, warnet.config_dir, warnet)
t._ipv4 = service["networks"][t.network_name]["ipv4_address"]
if "BITCOIN_VERSION" in service["build"]["args"]:
t.version = service["build"]["args"]["BITCOIN_VERSION"]
if "image" in service:
t.version = service["image"].split(":")[1]
else:
t.version = f"{service['build']['args']['REPO']}#{service['build']['args']['BRANCH']}"
return t
Expand Down
2 changes: 1 addition & 1 deletion src/services/tor.py → src/services/tor_da.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
DIRECTORY_AUTHORITY_IP = "100.20.15.18"


class Tor(BaseService):
class TorDA(BaseService):
def __init__(self, docker_network, templates):
super().__init__(docker_network)
self.templates = templates
Expand Down
16 changes: 16 additions & 0 deletions src/services/tor_relay.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from .base_service import BaseService

DOCKERFILE = "Dockerfile_tor_relay"


class TorRelay(BaseService):
def __init__(self, docker_network, templates):
super().__init__(docker_network)
self.templates = templates
self.service = {
"build": {
"context": str(self.templates),
"dockerfile": DOCKERFILE,
},
"networks": [ self.docker_network ],
}
Loading

0 comments on commit 417a6a7

Please sign in to comment.