[WIP] Init containertest and dockerclient

Makefile

@@ -8,7 +8,7 @@ DOCKER_REPO = dyweb/go.ice
 # --- build vars ---
 
 # --- packages ---
-PKGST=./cli
+PKGST=./cli ./cmd ./dockerclient ./containertest
 PKGS=./cli/...
 # --- packages ---
 

README.md

@@ -29,6 +29,10 @@ Non Goals
 
 ## License
 
+MIT
+
+NOTE: code under [dockerclient/types](dockerclient/types) are copied from [moby](https://github.com/moby/moby/tree/master/api/types) and licensed under Apache-2.0
+
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fat15%2Fgo.ice.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fat15%2Fgo.ice?ref=badge_large)
 
 ## About

cmd/dk/main.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+
+	"github.com/docker/docker/api/types"
+	icli "github.com/dyweb/go.ice/cli"
+	"github.com/dyweb/go.ice/dockerclient"
+	dlog "github.com/dyweb/gommon/log"
+	"github.com/spf13/cobra"
+)
+
+const (
+	myname = "bh"
+)
+
+var logReg = dlog.NewRegistry()
+var log = logReg.Logger()
+
+var (
+	version   string
+	commit    string
+	buildTime string
+	buildUser string
+	goVersion = runtime.Version()
+)
+
+var buildInfo = icli.BuildInfo{Version: version, Commit: commit, BuildTime: buildTime, BuildUser: buildUser, GoVersion: goVersion}
+
+var cli *icli.Root
+
+func main() {
+	cli = icli.New(
+		icli.Name(myname),
+		icli.Description("BenchHub"),
+		icli.Version(buildInfo),
+	)
+	root := cli.Command()
+	psCmd := cobra.Command{
+		Use: "ps",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c := mustClient()
+			containers, err := c.ContainerList(context.Background(), types.ContainerListOptions{
+				All: true,
+			})
+			if err != nil {
+				return err
+			}
+			log.Infof("%d", len(containers))
+			return nil
+		},
+	}
+	pullCmd := cobra.Command{
+		Use: "pull",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			c := mustClient()
+			reader, err := c.ImagePull(context.Background(), "dyweb/go-dev:1.13.6", types.ImagePullOptions{})
+			if err != nil {
+				return err
+			}
+			// TODO: it's actually json stream ...
+			io.Copy(os.Stdout, reader)
+			reader.Close()
+			return nil
+		},
+	}
+	root.AddCommand(&psCmd)
+	root.AddCommand(&pullCmd)
+	if err := root.Execute(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func mustClient() *dockerclient.Client {
+	c, err := dockerclient.New("/var/run/docker.sock")
+	if err != nil {
+		log.Fatal(err)
+	}
+	return c
+}

containertest/pkg.go

@@ -0,0 +1,3 @@
+// Package containertest allows launching container using go code in go test.
+// See https://github.com/dyweb/go.ice/issues/56
+package containertest

doc/ROADMAP.md

@@ -1,5 +1,9 @@
 # Roadmap
 
+## v0.0.4
+
+- [ ] container test, used by BenchHub
+
 ## 0.1.x
 
 Use 0.1.x for v2 features, the v2 is actually v0.2.x since there weren't a usable v2

doc/log/2020/2020-03/2020-03-03-containertest.md

@@ -0,0 +1,13 @@
+# 2020-03-03 Container Test
+
+For https://github.com/dyweb/go.ice/issues/56, it is required by benchhub for testing relational database.
+
+## TODO
+
+- [ ] revive the [old docker client](https://github.com/dyweb/go.ice/tree/archive/2020-01-13/lib/dockerclient)
+- [ ] allow start/stop mysql container and wait for ready
+- [ ] create database from testdata
+
+It's a bit hard to have go mod working with docker, suggested way is https://github.com/moby/moby/issues/39302#issuecomment-504146736
+
+Moved part of dockerclient part, but maybe shell out is a better idea, compared with manually vendoring the part I need ...

dockerclient/client.go

@@ -0,0 +1,75 @@
+package dockerclient
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"github.com/docker/docker/api/types"
+	"github.com/dyweb/gommon/errors"
+	"github.com/dyweb/gommon/httpclient"
+	"github.com/dyweb/gommon/util/httputil"
+)
+
+type Client struct {
+	version string
+	h       *httpclient.Client
+}
+
+func New(host string) (*Client, error) {
+	if host != "" && !strings.Contains(host, ".sock") {
+		// standard docker command accept host without the protocol prefix and use tls flag to indicate https
+		if !strings.HasPrefix(host, "http://") {
+			host = "http://" + host
+		}
+	}
+	h, err := httpclient.New(
+		host,
+		httpclient.UseJSON(),
+		httpclient.WithErrorHandlerFunc(DecodeDockerError),
+	)
+	if err != nil {
+		return nil, err
+	}
+	return &Client{
+		version: DefaultVersion,
+		h:       h,
+	}, nil
+}
+
+func (dc *Client) Ping() (types.Ping, error) {
+	var ping types.Ping
+	res, err := dc.h.GetRaw(httpclient.Bkg(), "/_ping")
+	if err != nil {
+		return ping, err
+	}
+	defer httpclient.DrainAndClose(res)
+	ping.APIVersion = res.Header.Get("API-Version")
+	if res.Header.Get("Docker-Experimental") == "true" {
+		ping.Experimental = true
+	}
+	ping.OSType = res.Header.Get("OSType")
+	return ping, nil
+}
+
+func (dc *Client) Version() (types.Version, error) {
+	var v types.Version
+	return v, dc.h.Get(httpclient.Bkg(), "/version", &v)
+}
+
+func DecodeDockerError(status int, body []byte, res *http.Response) (decodedError error) {
+	e := ErrDocker{
+		Status: status,
+		Method: httputil.Method(res.Request.Method),
+		Url:    res.Request.URL.String(),
+		Path:   res.Request.URL.Path,
+		Body:   string(body),
+	}
+	// try to decode docker's error message, which is just a single string, well designed ...
+	var derr types.ErrorResponse
+	if err := json.Unmarshal(body, &derr); err != nil {
+		errors.Ignore(err)
+	}
+	e.Message = derr.Message
+	return &e
+}

dockerclient/container.go

@@ -0,0 +1,101 @@
+package dockerclient
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/network"
+
+	"github.com/dyweb/gommon/errors"
+	"github.com/dyweb/gommon/httpclient"
+)
+
+// TODO
+// - start
+// - stop
+// - kill
+
+type configWrapper struct {
+	*container.Config
+	HostConfig       *container.HostConfig
+	NetworkingConfig *network.NetworkingConfig
+}
+
+// https://docs.docker.com/engine/api/sdk/examples/#run-a-container
+// https://github.com/docker/cli/blob/master/cli/command/container/run.go
+func (dc *Client) ContainerCreate(ctx context.Context, config *container.Config,
+	hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig,
+	containerName string) (container.ContainerCreateCreatedBody, error) {
+	hCtx := httpclient.ConvertContext(ctx)
+	if containerName != "" {
+		hCtx.SetParam("name", containerName)
+	}
+	body := configWrapper{
+		Config:           config,
+		HostConfig:       hostConfig,
+		NetworkingConfig: networkingConfig,
+	}
+
+	var created container.ContainerCreateCreatedBody
+	err := dc.h.Post(hCtx, "/containers/create", body, &created)
+	return created, err
+}
+
+// https://github.com/docker/cli/blob/master/cli/command/container/list.go
+// https://github.com/moby/moby/blob/master/client/container_list.go
+// https://docs.docker.com/engine/reference/commandline/ps/#usage
+func (dc *Client) ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) {
+	hCtx := httpclient.ConvertContext(ctx)
+
+	if options.All {
+		hCtx.SetParam("all", "1")
+	}
+	if options.Limit != -1 {
+		hCtx.SetParam("limit", strconv.Itoa(options.Limit))
+	}
+	if options.Since != "" {
+		hCtx.SetParam("since", options.Since)
+	}
+	if options.Before != "" {
+		hCtx.SetParam("before", options.Before)
+	}
+	if options.Size {
+		hCtx.SetParam("size", "1")
+	}
+	if options.Filters.Len() > 0 {
+		if filterJSON, err := filters.ToJSON(options.Filters); err != nil {
+			return nil, err
+		} else {
+			hCtx.SetParam("filters", filterJSON)
+		}
+	}
+
+	var containers []types.Container
+	if err := dc.h.Get(hCtx, "/containers/json", &containers); err != nil {
+		return nil, err
+	}
+	return containers, nil
+}
+
+// TODO: signal should be typed
+// TODO: kill -l to list all the signals
+// https://www.linux.org/threads/kill-commands-and-signals.8881/
+// https://github.com/docker/cli/blob/master/cli/command/container/kill.go
+// https://github.com/moby/moby/blob/master/client/container_kill.go
+func (dc *Client) ContainerKill(ctx context.Context, containerId, signal string) error {
+	hCtx := httpclient.ConvertContext(ctx)
+
+	if signal == "" {
+		signal = "KILL"
+	}
+	if containerId == "" {
+		return errors.New("containerId is empty for container kill")
+	}
+	if err := dc.h.PostIgnoreRes(hCtx, "/containers/"+containerId+"/kill", nil); err != nil {
+		return err
+	}
+	return nil
+}

dockerclient/image.go

@@ -0,0 +1,79 @@
+package dockerclient
+
+import (
+	"context"
+	"io"
+	"strings"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/versions"
+	"github.com/dyweb/gommon/httpclient"
+)
+
+// image.go is merged all the image_*.go into one file
+
+// https://github.com/moby/moby/blob/master/client/image_pull.go difference between pull and create is pull try to auth
+// https://github.com/moby/moby/blob/master/client/image_create.go
+func (dc *Client) ImagePull(ctx context.Context, refStr string, options types.ImagePullOptions) (io.ReadCloser, error) {
+
+	hCtx := httpclient.ConvertContext(ctx)
+
+	ref, err := reference.ParseNormalizedNamed(refStr)
+	if err != nil {
+		return nil, err
+	}
+	hCtx.SetParam("fromImage", reference.FamiliarName(ref))
+	hCtx.SetParam("tag", getAPITagFromNamedRef(ref))
+	if options.Platform != "" {
+		hCtx.SetParam("platform", strings.ToLower(options.Platform))
+	}
+	// TODO: handle auth, this is needed to pull from private registry
+	res, err := dc.h.PostRaw(hCtx, "/images/create", nil)
+	if err != nil {
+		return nil, err
+	}
+	return res.Body, nil
+}
+
+// https://github.com/moby/moby/blob/master/client/image_list.go
+func (dc *Client) ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error) {
+	var images []types.ImageSummary
+
+	hCtx := httpclient.ConvertContext(ctx)
+	optionFilters := options.Filters
+	referenceFilters := optionFilters.Get("reference")
+	if versions.LessThan(dc.version, "1.25") && len(referenceFilters) > 0 {
+		hCtx.SetParam("filter", referenceFilters[0])
+		for _, filterValue := range referenceFilters {
+			optionFilters.Del("reference", filterValue)
+		}
+	}
+	if optionFilters.Len() > 0 {
+		filterJSON, err := filters.ToJSON(optionFilters)
+		if err != nil {
+			return images, err
+		}
+		hCtx.SetParam("filters", filterJSON)
+	}
+	if options.All {
+		hCtx.SetParam("all", "1")
+	}
+	return images, dc.h.Get(hCtx, "/images/json", &images)
+}
+
+// getAPITagFromNamedRef returns a tag from the specified reference.
+// This function is necessary as long as the docker "server" api expects
+// digests to be sent as tags and makes a distinction between the name
+// and tag/digest part of a reference.
+func getAPITagFromNamedRef(ref reference.Named) string {
+	if digested, ok := ref.(reference.Digested); ok {
+		return digested.Digest().String()
+	}
+	ref = reference.TagNameOnly(ref)
+	if tagged, ok := ref.(reference.Tagged); ok {
+		return tagged.Tag()
+	}
+	return ""
+}