Merge pull request #35 from nabla-containers/k8s-network

Added networking hack for CNI namespaced veth pairs

hack/copy_binaries.sh

@@ -18,9 +18,17 @@
 # PERFORMANCE OF THIS SOFTWARE.
 
 BIN_PATH=/usr/local/bin/
+
+# We add binaries like runnc-cont and nabla-run to /opt/X since they are not 
+# to be consumed directly by the user.
+BIN_PATH2=/opt/runnc/bin/
+
 COPY_BINS=("build/runnc" "build/runnc-cont" "build/nabla-run")
 
+mkdir -p ${BIN_PATH2}
+
 for i in ${COPY_BINS[@]}; do
     echo $i
     cp $i ${BIN_PATH}/
+    cp $i ${BIN_PATH2}/
 done

libcontainer/configs/config.go

@@ -11,6 +11,9 @@ type Config struct {
 
 	// Labels are user defined metadata that is stored in the config and populated on the state
 	Labels []string `json:"labels"`
+
+	// Network namespace
+	NetnsPath string `json:"netnspath"`
 }
 
 // HostUID returns the UID to run the nabla container as. Default is root.

libcontainer/configs/spec.go

@@ -23,13 +23,23 @@ func ParseSpec(s *specs.Spec) (*Config, error) {
 		labels = append(labels, fmt.Sprintf("%s=%s", k, v))
 	}
 
+	netnsPath := ""
+	if s.Linux != nil {
+		for _, v := range s.Linux.Namespaces {
+			if v.Type == specs.NetworkNamespace {
+				netnsPath = v.Path
+			}
+		}
+	}
+
 	cfg := Config{
-		Args:    s.Process.Args,
-		Rootfs:  s.Root.Path,
-		Env:     s.Process.Env,
-		Cwd:     s.Process.Cwd,
-		Version: s.Version,
-		Labels:  labels,
+		Args:      s.Process.Args,
+		Rootfs:    s.Root.Path,
+		Env:       s.Process.Env,
+		Cwd:       s.Process.Cwd,
+		Version:   s.Version,
+		NetnsPath: netnsPath,
+		Labels:    labels,
 	}
 
 	return &cfg, nil

libcontainer/container_nabla.go

@@ -202,12 +202,13 @@ func (c *nablaContainer) start(p *Process) error {
 
 	defer parentPipe.Close()
 	config := initConfig{
-		Root:    c.config.Rootfs,
-		Args:    c.config.Args,
-		FsPath:  c.fsPath,
-		Cwd:     c.config.Cwd,
-		Env:     c.config.Env,
-		TapName: nablaTapName(c.id),
+		Root:      c.config.Rootfs,
+		Args:      c.config.Args,
+		FsPath:    c.fsPath,
+		Cwd:       c.config.Cwd,
+		Env:       c.config.Env,
+		TapName:   nablaTapName(c.id),
+		NetnsPath: c.config.NetnsPath,
 	}
 
 	enc := json.NewEncoder(parentPipe)

libcontainer/init_nabla.go

@@ -17,6 +17,7 @@ package libcontainer
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/vishvananda/netns"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -46,6 +47,10 @@ func nablaRunArgs(cfg *initConfig) ([]string, error) {
 		"-volume", cfg.FsPath + ":/",
 		"-unikernel", filepath.Join(cfg.Root, cfg.Args[0])}
 
+	if cfg.NetnsPath != "" {
+		args = append(args, "-k8s")
+	}
+
 	for _, e := range cfg.Env {
 		args = append(args, "-env", e)
 	}
@@ -58,12 +63,13 @@ func nablaRunArgs(cfg *initConfig) ([]string, error) {
 }
 
 type initConfig struct {
-	Root    string   `json:"root"`
-	Args    []string `json:"args"`
-	FsPath  string   `json:"fspath"`
-	Cwd     string   `json:"cwd"`
-	Env     []string `json:"env"`
-	TapName string   `json:"tap"`
+	Root      string   `json:"root"`
+	Args      []string `json:"args"`
+	FsPath    string   `json:"fspath"`
+	Cwd       string   `json:"cwd"`
+	Env       []string `json:"env"`
+	TapName   string   `json:"tap"`
+	NetnsPath string   `json:"netnspath"`
 }
 
 func initNabla() error {
@@ -108,6 +114,18 @@ func initNabla() error {
 	syscall.Close(fd)
 	syscall.Close(rootfd)
 
+	// Go into network namespace for temporary hack for CNI plugin using veth pairs
+	if config.NetnsPath != "" {
+		nsh, err := netns.GetFromPath(config.NetnsPath)
+		if err != nil {
+			return newSystemErrorWithCause(err, "unable to get netns handle")
+		}
+
+		if err := netns.Set(nsh); err != nil {
+			return newSystemErrorWithCause(err, "unable to get set netns")
+		}
+	}
+
 	// Check if it is a pause container, if it is, just pause
 	if len(config.Args) == 1 && config.Args[0] == pauseNablaName {
 		select {}

nabla-lib/network/network_linux.go

@@ -159,12 +159,12 @@ func CreateMacvtapInterfaceDocker(tapName *string, master string) (
 
 	netHandle, err := netlink.NewHandle()
 	if err != nil {
-		return nil, nil, nil, "", err
+		return nil, nil, nil, "", errors.Wrap(err, "Unable to create netlink handler")
 	}
 
 	err = SetupTunDev()
 	if err != nil {
-		return nil, nil, nil, "", err
+		return nil, nil, nil, "", errors.Wrap(err, "Unable to setup tun dev")
 	}
 
 	masterLink, err := netlink.LinkByName(master)
@@ -175,13 +175,13 @@ func CreateMacvtapInterfaceDocker(tapName *string, master string) (
 
 	macvtapLink, name, tapNameTmp, err := createMacvtapInterface(netHandle, masterLink)
 	if err != nil {
-		return nil, nil, nil, "", err
+		return nil, nil, nil, "", errors.Wrap(err, "Unable to create Macvtapint")
 	}
 	*tapName = tapNameTmp
 
 	addrs, err := netlink.AddrList(masterLink, netlink.FAMILY_V4)
 	if err != nil {
-		return nil, nil, nil, "", err
+		return nil, nil, nil, "", errors.Wrap(err, "Unable to get address list")
 	}
 	if len(addrs) == 0 {
 		return nil, nil, nil, "", fmt.Errorf("master should have an IP")
@@ -192,7 +192,7 @@ func CreateMacvtapInterfaceDocker(tapName *string, master string) (
 
 	routes, err := netlink.RouteList(masterLink, netlink.FAMILY_V4)
 	if err != nil {
-		return nil, nil, nil, "", err
+		return nil, nil, nil, "", errors.Wrap(err, "Unable to get route list")
 	}
 	if len(routes) == 0 {
 		return nil, nil, nil, "",
@@ -204,17 +204,17 @@ func CreateMacvtapInterfaceDocker(tapName *string, master string) (
 	// ip addr del $INET_STR dev master
 	err = netlink.AddrDel(masterLink, &masterAddr)
 	if err != nil {
-		return nil, nil, nil, "", err
+		return nil, nil, nil, "", errors.Wrap(err, "Unable to delete address of master")
 	}
 
 	err = netlink.LinkSetUp(macvtapLink)
 	if err != nil {
-		return nil, nil, nil, "", err
+		return nil, nil, nil, "", errors.Wrap(err, "Unable to set up tap link")
 	}
 
 	err = netlink.LinkSetUp(masterLink)
 	if err != nil {
-		return nil, nil, nil, "", err
+		return nil, nil, nil, "", errors.Wrap(err, "Unable to set up master link")
 	}
 
 	// The HardwareAddr Attr doesn't automatically get updated

runnc-cont/runnc_cont.go

@@ -86,6 +86,8 @@ func main() {
 		"Max memory size in MBs (defaults to 512)")
 	inDocker := flag.Bool("docker", false,
 		"Is this running in a Docker container")
+	inK8s := flag.Bool("k8s", false,
+		"Is this running in a K8s cluster")
 	volume := flag.String("volume", ":",
 		"'--volume <SRC>:<DST>'. "+
 			"<SRC> is the directory or device to mount, and <DST> "+
@@ -111,12 +113,12 @@ func main() {
 	cmdargs := strings.Join(flag.Args(), " ")
 
 	os.Exit(run(*nablarun, *unikernel, *tap, ip, ipNet.Mask, gw,
-		*inDocker, vol, cmdargs, envVars, *cwd, *mem))
+		*inDocker, *inK8s, vol, cmdargs, envVars, *cwd, *mem))
 }
 
 func run(nablarun string, unikernel string, tapName string,
 	ip net.IP, mask net.IPMask, gw net.IP,
-	inDocker bool, volume []string,
+	inDocker bool, inK8s bool, volume []string,
 	cmdargs string, envVars []string, cwd string, mem int) int {
 
 	disk, err := setupDisk(volume[0])
@@ -135,6 +137,16 @@ func run(nablarun string, unikernel string, tapName string,
 			fmt.Fprintf(os.Stderr, "Could not create %s: %v\n", tapName, err)
 			return 1
 		}
+	} else if inK8s {
+		// The tap device will get the IP assigned to the k8s nabla
+		// container veth pair.
+		// XXX: This is a workaround due to an error with MacvTap, error was :
+		// Could not create /dev/tap8863: open /sys/devices/virtual/net/macvtap8863/tap8863/dev: no such file or directory
+		ip, gw, mask, err = network.CreateTapInterfaceDocker(tapName, "eth0")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Could not create %s: %v\n", tapName, err)
+			return 1
+		}
 	} else {
 		err = network.CreateTapInterface(tapName, &gw, &mask)
 		if err != nil {