Merge pull request #36 from nabla-containers/docker-network

Updated to work with docker prestart hooks for network setup

libcontainer/configs/config.go

@@ -1,5 +1,9 @@
 package configs
 
+import (
+	spec "github.com/opencontainers/runtime-spec/specs-go"
+)
+
 type Config struct {
 	Args   []string `json:"args"`
 	Rootfs string   `json:"rootfs"`
@@ -14,6 +18,9 @@ type Config struct {
 
 	// Network namespace
 	NetnsPath string `json:"netnspath"`
+
+	// Hooks configures callbacks for container lifecycle events.
+	Hooks *spec.Hooks `json:"hooks,omitempty"`
 }
 
 // HostUID returns the UID to run the nabla container as. Default is root.

libcontainer/configs/spec.go

@@ -40,6 +40,7 @@ func ParseSpec(s *specs.Spec) (*Config, error) {
 		Version:   s.Version,
 		NetnsPath: netnsPath,
 		Labels:    labels,
+		Hooks:     s.Hooks,
 	}
 
 	return &cfg, nil

libcontainer/container_nabla.go

@@ -202,13 +202,16 @@ func (c *nablaContainer) start(p *Process) error {
 
 	defer parentPipe.Close()
 	config := initConfig{
-		Root:      c.config.Rootfs,
-		Args:      c.config.Args,
-		FsPath:    c.fsPath,
-		Cwd:       c.config.Cwd,
-		Env:       c.config.Env,
-		TapName:   nablaTapName(c.id),
-		NetnsPath: c.config.NetnsPath,
+		Id:         c.id,
+		BundlePath: c.root,
+		Root:       c.config.Rootfs,
+		Args:       c.config.Args,
+		FsPath:     c.fsPath,
+		Cwd:        c.config.Cwd,
+		Env:        c.config.Env,
+		TapName:    nablaTapName(c.id),
+		NetnsPath:  c.config.NetnsPath,
+		Hooks:      c.config.Hooks,
 	}
 
 	enc := json.NewEncoder(parentPipe)

libcontainer/hooks.go

@@ -0,0 +1,68 @@
+package libcontainer
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	spec "github.com/opencontainers/runtime-spec/specs-go"
+	"os"
+	"os/exec"
+	"syscall"
+	"time"
+)
+
+func runHook(hook spec.Hook, cid, bundlePath string) error {
+	// Adapted from:
+	// github.com/kata-containers/runtime/cli/hook.go
+	state := spec.State{
+		Pid:    os.Getpid(),
+		Bundle: bundlePath,
+		ID:     cid,
+	}
+
+	stateJSON, err := json.Marshal(state)
+	if err != nil {
+		return err
+	}
+
+	var stdout, stderr bytes.Buffer
+	cmd := &exec.Cmd{
+		Path:   hook.Path,
+		Args:   hook.Args,
+		Env:    hook.Env,
+		Stdin:  bytes.NewReader(stateJSON),
+		Stdout: &stdout,
+		Stderr: &stderr,
+	}
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+
+	if hook.Timeout == nil {
+		if err := cmd.Wait(); err != nil {
+			return fmt.Errorf("%s: stdout: %s, stderr: %s", err, stdout.String(), stderr.String())
+		}
+	} else {
+		done := make(chan error, 1)
+		go func() {
+			done <- cmd.Wait()
+			close(done)
+		}()
+
+		select {
+		case err := <-done:
+			if err != nil {
+				return fmt.Errorf("%s: stdout: %s, stderr: %s", err, stdout.String(), stderr.String())
+			}
+		case <-time.After(time.Duration(*hook.Timeout) * time.Second):
+			if err := syscall.Kill(cmd.Process.Pid, syscall.SIGKILL); err != nil {
+				return err
+			}
+
+			return fmt.Errorf("Hook timeout")
+		}
+	}
+
+	return nil
+}

libcontainer/init_nabla.go

@@ -17,6 +17,7 @@ package libcontainer
 import (
 	"encoding/json"
 	"fmt"
+	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/vishvananda/netns"
 	"os"
 	"path/filepath"
@@ -41,16 +42,13 @@ func nablaRunArgs(cfg *initConfig) ([]string, error) {
 	}
 
 	args := []string{NablaRunncContBin,
+		"-k8s",
 		"-nabla-run", NablaRunBin,
 		"-tap", cfg.TapName,
 		"-cwd", cfg.Cwd,
 		"-volume", cfg.FsPath + ":/",
 		"-unikernel", filepath.Join(cfg.Root, cfg.Args[0])}
 
-	if cfg.NetnsPath != "" {
-		args = append(args, "-k8s")
-	}
-
 	for _, e := range cfg.Env {
 		args = append(args, "-env", e)
 	}
@@ -63,13 +61,16 @@ func nablaRunArgs(cfg *initConfig) ([]string, error) {
 }
 
 type initConfig struct {
-	Root      string   `json:"root"`
-	Args      []string `json:"args"`
-	FsPath    string   `json:"fspath"`
-	Cwd       string   `json:"cwd"`
-	Env       []string `json:"env"`
-	TapName   string   `json:"tap"`
-	NetnsPath string   `json:"netnspath"`
+	Id         string      `json:"id"`
+	BundlePath string      `json:"bundlepath"`
+	Root       string      `json:"root"`
+	Args       []string    `json:"args"`
+	FsPath     string      `json:"fspath"`
+	Cwd        string      `json:"cwd"`
+	Env        []string    `json:"env"`
+	TapName    string      `json:"tap"`
+	NetnsPath  string      `json:"netnspath"`
+	Hooks      *spec.Hooks `json:"hooks"`
 }
 
 func initNabla() error {
@@ -102,19 +103,8 @@ func initNabla() error {
 	// specific env vars.
 	os.Clearenv()
 
-	// wait for the fifo to be opened on the other side before
-	// exec'ing the users process.
-	fd, err := syscall.Openat(rootfd, execFifoFilename, os.O_WRONLY|syscall.O_CLOEXEC, 0)
-	if err != nil {
-		return newSystemErrorWithCause(err, "openat exec fifo")
-	}
-	if _, err := syscall.Write(fd, []byte("0")); err != nil {
-		return newSystemErrorWithCause(err, "write 0 exec fifo")
-	}
-	syscall.Close(fd)
-	syscall.Close(rootfd)
-
 	// Go into network namespace for temporary hack for CNI plugin using veth pairs
+	// K8s case
 	if config.NetnsPath != "" {
 		nsh, err := netns.GetFromPath(config.NetnsPath)
 		if err != nil {
@@ -124,8 +114,38 @@ func initNabla() error {
 		if err := netns.Set(nsh); err != nil {
 			return newSystemErrorWithCause(err, "unable to get set netns")
 		}
+	} else {
+		// Docker case for docker cli
+		// TODO: case on specific --docker-cli flag
+		nsh, err := netns.New()
+		if err != nil {
+			return newSystemErrorWithCause(err, "unable to create netns handle")
+		}
+
+		if err := netns.Set(nsh); err != nil {
+			return newSystemErrorWithCause(err, "unable to get set netns")
+		}
+	}
+	if config.Hooks != nil {
+		for _, hook := range config.Hooks.Prestart {
+			if err := runHook(hook, config.Id, config.BundlePath); err != nil {
+				return newSystemErrorWithCause(err, "unable to run prestart hook")
+			}
+		}
 	}
 
+	// wait for the fifo to be opened on the other side before
+	// exec'ing the users process.
+	fd, err := syscall.Openat(rootfd, execFifoFilename, os.O_WRONLY|syscall.O_CLOEXEC, 0)
+	if err != nil {
+		return newSystemErrorWithCause(err, "openat exec fifo")
+	}
+	if _, err := syscall.Write(fd, []byte("0")); err != nil {
+		return newSystemErrorWithCause(err, "write 0 exec fifo")
+	}
+	syscall.Close(fd)
+	syscall.Close(rootfd)
+
 	// Check if it is a pause container, if it is, just pause
 	if len(config.Args) == 1 && config.Args[0] == pauseNablaName {
 		select {}

runnc-cli/runnc.go

@@ -116,7 +116,6 @@ func main() {
 	app.Before = func(context *cli.Context) error {
 		if context.GlobalBool("debug") {
 			logrus.SetLevel(logrus.DebugLevel)
-			fmt.Fprintln(os.Stderr, "Setting debug level")
 		}
 		if path := context.GlobalString("log"); path != "" {
 			f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0666)

runnc-cli/state.go

@@ -63,9 +63,6 @@ instance of a container.`,
 		}
 		os.Stdout.Write(data)
 
-		// DEBUG
-		os.Stderr.Write(data)
-
 		return nil
 	},
 }