From f5ddca2422321e13cdaba3eff65af4d195e94290 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 4 Dec 2024 16:54:29 -0500 Subject: [PATCH 1/4] Add debugging setup for vscode In order to use the debugger, you should Run any build as usual, but use substitute `debug/debug.sh` for `docker` in a docker invocation. The script uses "pgrep" and "socat", as such they are required to run the script. Example: ``` ./debug/debug.sh -f /tmp/moby-runc.yml --target=azlinux3/container --output=type=docker,name=hello:world . ``` Then, connect to the debugging setup using this `launch.json` config in vscode. You need the go plugin installed: ```json { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { "name": "Connect to server", "type": "go", "request": "attach", "mode": "remote", "port": 30157, "host": "127.0.0.1", "apiVersion": 2 } ] } ``` Signed-off-by: Peter Engelbert --- Dockerfile.debug | 37 ++++++++++++++++++++++++++++ cmd/frontend/debug.go | 43 ++++++++++++++++++++++++++++++++ cmd/frontend/main.go | 31 ++++++++++++++++------- cmd/frontend/no_debug.go | 9 +++++++ debug/debug.sh | 53 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 164 insertions(+), 9 deletions(-) create mode 100644 Dockerfile.debug create mode 100644 cmd/frontend/debug.go create mode 100644 cmd/frontend/no_debug.go create mode 100755 debug/debug.sh diff --git a/Dockerfile.debug b/Dockerfile.debug new file mode 100644 index 000000000..4b99a727d --- /dev/null +++ b/Dockerfile.debug @@ -0,0 +1,37 @@ +FROM --platform=${BUILDPLATFORM} mcr.microsoft.com/oss/go/microsoft/golang:1.22@sha256:9e5243001d0f43d6e1c556a0ce8eedc1bc80eb37b5210036b289d2c601bc08b6 AS go +FROM scratch AS dlv-ctx + +FROM go AS frontend-build +ARG HOSTDIR +WORKDIR ${HOSTDIR} +COPY . . +ENV CGO_ENABLED=0 +ARG TARGETARCH TARGETOS GOFLAGS=-trimpath +ENV GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOFLAGS=${GOFLAGS} +RUN \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + go build -tags=debug -gcflags="all=-N -l" -o /frontend ./cmd/frontend && \ + go build -o /dalec-redirectio ./cmd/dalec-redirectio + +FROM go AS dlv-build +ARG HOSTDIR +WORKDIR /dlv +ADD https://github.com/go-delve/delve.git#v1.23.1 . +ENV CGO_ENABLED=0 +ARG TARGETARCH TARGETOS GOFLAGS=-trimpath +ENV GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOFLAGS=${GOFLAGS} +RUN \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + make build + +FROM mcr.microsoft.com/azurelinux/distroless/base:3.0 AS frontend +ARG HOSTDIR +COPY . ${HOSTDIR} +COPY --from=frontend-build /frontend /frontend +COPY --from=frontend-build /dalec-redirectio /dalec-redirectio +COPY --from=dlv-build /dlv/dlv /usr/local/bin/dlv +LABEL moby.buildkit.frontend.network.none="true" +LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts" +ENTRYPOINT ["/frontend"] diff --git a/cmd/frontend/debug.go b/cmd/frontend/debug.go new file mode 100644 index 000000000..14aa80fc1 --- /dev/null +++ b/cmd/frontend/debug.go @@ -0,0 +1,43 @@ +//go:build debug + +package main + +import ( + "context" + "fmt" + "io" + "os/exec" + "os/signal" + "syscall" +) + +func waitForDebug(ctx context.Context) error { + pid := fmt.Sprintf("%d", syscall.Getpid()) + cmd := exec.Command( + "dlv", + "attach", + "--api-version=2", + "--headless", + "--listen=unix:/dlv.sock", + "--allow-non-terminal-interactive", + pid, + ) + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + + if err := cmd.Start(); err != nil { + return err + } + + go func() { + if err := cmd.Wait(); err != nil { + panic(err) + } + }() + + s, cancel := signal.NotifyContext(ctx, syscall.SIGCONT) + defer cancel() + <-s.Done() + + return nil +} diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go index 6940ea153..6731fb04c 100644 --- a/cmd/frontend/main.go +++ b/cmd/frontend/main.go @@ -1,6 +1,7 @@ package main import ( + "context" _ "embed" "os" @@ -10,6 +11,7 @@ import ( "github.com/Azure/dalec/frontend/debug" "github.com/Azure/dalec/frontend/ubuntu" "github.com/Azure/dalec/frontend/windows" + gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/frontend/gateway/grpcclient" "github.com/moby/buildkit/util/appcontext" "github.com/moby/buildkit/util/bklog" @@ -31,16 +33,27 @@ func main() { mux.Add(debug.DebugRoute, debug.Handle, nil) - if err := grpcclient.RunFromEnvironment(ctx, mux.Handler( - // copy/paster's beware: [frontend.WithTargetForwardingHandler] should not be set except for the root dalec frontend. - frontend.WithBuiltinHandler(azlinux.Mariner2TargetKey, azlinux.NewMariner2Handler()), - frontend.WithBuiltinHandler(azlinux.AzLinux3TargetKey, azlinux.NewAzlinux3Handler()), - frontend.WithBuiltinHandler(windows.DefaultTargetKey, windows.Handle), - ubuntu.Handlers, - debian.Handlers, - frontend.WithTargetForwardingHandler, - )); err != nil { + f := func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) { + if err := waitForDebug(ctx); err != nil { + return nil, err + } + + handlerFunc := mux.Handler( + // copy/paster's beware: [frontend.WithTargetForwardingHandler] should not be set except for the root dalec frontend. + frontend.WithBuiltinHandler(azlinux.Mariner2TargetKey, azlinux.NewMariner2Handler()), + frontend.WithBuiltinHandler(azlinux.AzLinux3TargetKey, azlinux.NewAzlinux3Handler()), + frontend.WithBuiltinHandler(windows.DefaultTargetKey, windows.Handle), + ubuntu.Handlers, + debian.Handlers, + frontend.WithTargetForwardingHandler, + ) + + return handlerFunc(ctx, client) + } + + if err := grpcclient.RunFromEnvironment(ctx, f); err != nil { bklog.L.WithError(err).Fatal("error running frontend") os.Exit(137) } + } diff --git a/cmd/frontend/no_debug.go b/cmd/frontend/no_debug.go new file mode 100644 index 000000000..4b5591125 --- /dev/null +++ b/cmd/frontend/no_debug.go @@ -0,0 +1,9 @@ +//go:build !debug + +package main + +import "context" + +func waitForDebug(ctx context.Context) error { + return nil +} diff --git a/debug/debug.sh b/debug/debug.sh new file mode 100755 index 000000000..3ecf4e4de --- /dev/null +++ b/debug/debug.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +if [ -z "$(command -v socat)" ]; then + echo you must have "'socat'" installed + exit 1 +fi + +if [ -z "$(command -v pgrep)" ]; then + echo you must have "'pgrep'" installed + exit 1 +fi + +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${PROJECT_DIR}" + +# Build frontend with debugging setup Note the host path for the dalec source +# and the in-container build path must be the same +REF="local/dalec/frontend:tmp" +docker build \ + -f Dockerfile.debug \ + -t "${REF}" \ + --build-arg=HOSTDIR="${PROJECT_DIR}" \ + . + +# Wait for frontend process to start, and forward the socket connection when the process has started +( + pid="" + while [ -z "$pid" ]; do + sleep 0.5 + pid="$(pgrep frontend)" + done + + socat_logfile="$(mktemp)" + socat -v UNIX:"/proc/${pid}/root/dlv.sock" TCP-LISTEN:30157,reuseaddr,fork 2>"$socat_logfile" & + scatpid="$!" + trap "kill -9 ${scatpid}" EXIT + + # Detect when vs code has connected, then send the CONT signal to the frontend + txt="" + while [ -z "$txt" ]; do + sleep 0.5 + if ! [ -f "$socat_logfile" ]; then + continue + fi + txt="$(head -n1 "$socat_logfile")" + done + + kill -s CONT "$pid" + wait "$scatpid" +) & + +# Run the build +exec docker build --build-arg=BUILDKIT_SYNTAX="${REF}" "$@" From 5729ff48f5bf8f1086ddec0122e98e7fb376c76e Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 10 Dec 2024 10:07:16 -0500 Subject: [PATCH 2/4] Use /proc/self/status to see if debugger attached Signed-off-by: Peter Engelbert --- cmd/frontend/debug.go | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/cmd/frontend/debug.go b/cmd/frontend/debug.go index 14aa80fc1..399a72294 100644 --- a/cmd/frontend/debug.go +++ b/cmd/frontend/debug.go @@ -3,11 +3,13 @@ package main import ( + "bufio" "context" "fmt" "io" + "os" "os/exec" - "os/signal" + "strings" "syscall" ) @@ -35,9 +37,28 @@ func waitForDebug(ctx context.Context) error { } }() - s, cancel := signal.NotifyContext(ctx, syscall.SIGCONT) - defer cancel() - <-s.Done() + tracerPid := "0" + for tracerPid == "0" { + b, err := os.Open("/proc/self/status") + if err != nil { + return err + } + + s := bufio.NewScanner(b) + for s.Scan() { + start, end, ok := strings.Cut(s.Text(), ":\t") + if !ok { + continue + } + + if start != "TracerPid" { + continue + } + + tracerPid = end + break + } + } return nil } From 3ad3a1a78c1285a54a7906e5b86caeaf463c01d4 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 10 Dec 2024 10:08:06 -0500 Subject: [PATCH 3/4] Remove signal sending from debug script Also add `set -eux` Signed-off-by: Peter Engelbert --- debug/debug.sh | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/debug/debug.sh b/debug/debug.sh index 3ecf4e4de..dca45a244 100755 --- a/debug/debug.sh +++ b/debug/debug.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -eux if [ -z "$(command -v socat)" ]; then echo you must have "'socat'" installed @@ -24,29 +25,18 @@ docker build \ # Wait for frontend process to start, and forward the socket connection when the process has started ( + set +x pid="" while [ -z "$pid" ]; do sleep 0.5 pid="$(pgrep frontend)" done + set -x socat_logfile="$(mktemp)" - socat -v UNIX:"/proc/${pid}/root/dlv.sock" TCP-LISTEN:30157,reuseaddr,fork 2>"$socat_logfile" & - scatpid="$!" - trap "kill -9 ${scatpid}" EXIT - - # Detect when vs code has connected, then send the CONT signal to the frontend - txt="" - while [ -z "$txt" ]; do - sleep 0.5 - if ! [ -f "$socat_logfile" ]; then - continue - fi - txt="$(head -n1 "$socat_logfile")" - done - - kill -s CONT "$pid" - wait "$scatpid" + socat -v UNIX:"/proc/${pid}/root/dlv.sock" TCP-LISTEN:30157,reuseaddr,fork 2>"$socat_logfile" + socat_pid="$!" + trap "kill -9 ${socat_pid}" EXIT ) & # Run the build From 6f4ea48a40ae26dea74fb2258a930bf3a909fcb7 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 8 Jan 2025 09:46:42 -0500 Subject: [PATCH 4/4] Check ptrace_scope procfile before proceeding Signed-off-by: Peter Engelbert --- debug/debug.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/debug/debug.sh b/debug/debug.sh index dca45a244..84ded1f26 100755 --- a/debug/debug.sh +++ b/debug/debug.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eux +PTRACE_SCOPE_PROCFILE="/proc/sys/kernel/yama/ptrace_scope" + if [ -z "$(command -v socat)" ]; then echo you must have "'socat'" installed exit 1 @@ -11,6 +13,16 @@ if [ -z "$(command -v pgrep)" ]; then exit 1 fi +if ! [ -f "$PTRACE_SCOPE_PROCFILE" ]; then + echo "unable to detect necessary procfile, attempting to continue..." +fi + +if [ "$(<"$PTRACE_SCOPE_PROCFILE")" != "0" ]; then + echo "you must set ${PTRACE_SCOPE_PROCFILE} to '0':" + echo "echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope" + exit 1 +fi + PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${PROJECT_DIR}"