Merge pull request #74 from seatgeek/dr-private-repo-prefix

Add Private Docker Registry Config Option

README.md

@@ -4,8 +4,6 @@
 
 This project will copy public DockerHub repositories to a private registry.
 
-It's possible to filter by docker tags, tag age and number of latest tags.
-
 <!-- TOC -->
 
 - [docker-mirror](#docker-mirror)
@@ -38,6 +36,20 @@ Make sure that your local Docker agent is logged into to `ECR` (`aws ecr get-log
 
 `docker-mirror` will look for your AWS credentials in all the default locations (`env`, `~/.aws/` and so forth like normal AWS tools do)
 
+### Configuration File
+
+There are several configuration options you can use in your `config.yaml` below. Please see the `config.yaml` file in the repository for a full example.
+
+- `ignore_tag:` This option sets tags that can be ignored on pulls. (i.e. `ignore_tag: - "*-alpine"`)
+
+- `match_tag:` This option sets the tags that you want to match on for pulls. (i.e. `match_tag: - "3*"`)
+
+- `max_tag_age:` This option sets the max tag age you wish to pull from. (i.e. `max_tag_age: 4w`)
+
+- `name:` This option sets the name of your repository. (i.e. `name: elasticsearch`)
+
+- `private_registry:` This option allows you to set a private Docker registry prefix for docker pulls. It will prefix any of your `name:` options with the `private_registry` name and a slash to allow you to customize where your images are being pulled through. This is particularly useful if you use a proxy to dockerhub. i.e. (`private_registry: "private-registry-name"`)
+
 ### Adding new mirror repository
 
 - add the new repository to the `config.yaml` file

config.yaml

@@ -4,7 +4,8 @@ target:
   prefix: "hub/"
 
 repositories:
-  - name: elasticsearch
+  - private_registry: "private-registry-name",
+    name: elasticsearch
     max_tag_age: 4w
     ignore_tag:
       - "*-alpine"

main.go

@@ -39,6 +39,7 @@ type TargetConfig struct {
 
 // Repository is a single docker hub repository to mirror
 type Repository struct {
+	PrivateRegistry string            `yaml:"private_registry"`
 	Name            string            `yaml:"name"`
 	MatchTags       []string          `yaml:"match_tag"`
 	DropTags        []string          `yaml:"ignore_tag"`
@@ -49,6 +50,11 @@ type Repository struct {
 	TargetPrefix    *string           `yaml:"target_prefix"`
 }
 
+func createDockerClient() (*docker.Client, error) {
+	client, err := docker.NewClientFromEnv()
+	return client, err
+}
+
 func main() {
 	// log level
 	if rawLevel := os.Getenv("LOG_LEVEL"); rawLevel != "" {
@@ -94,7 +100,8 @@ func main() {
 
 	// init Docker client
 	log.Info("Creating Docker client")
-	client, err := docker.NewClientFromEnv()
+	var client DockerClient
+	client, err = createDockerClient()
 	if err != nil {
 		log.Fatalf("Could not create Docker client: %s", err.Error())
 	}
@@ -132,7 +139,7 @@ func main() {
 
 	// start background workers
 	for i := 0; i < config.Workers; i++ {
-		go worker(&wg, workerCh, client, ecrManager)
+		go worker(&wg, workerCh, &client, ecrManager)
 	}
 
 	prefix := os.Getenv("PREFIX")
@@ -152,7 +159,7 @@ func main() {
 	log.Info("Done")
 }
 
-func worker(wg *sync.WaitGroup, workerCh chan Repository, dc *docker.Client, ecrm *ecrManager) {
+func worker(wg *sync.WaitGroup, workerCh chan Repository, dc *DockerClient, ecrm *ecrManager) {
 	log.Debug("Starting worker")
 
 	for {

mirror.go

@@ -48,8 +48,16 @@ func (l logWriter) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+type DockerClient interface {
+	Info() (*docker.DockerInfo, error)
+	TagImage(string, docker.TagImageOptions) error
+	PullImage(docker.PullImageOptions, docker.AuthConfiguration) error
+	PushImage(docker.PushImageOptions, docker.AuthConfiguration) error
+	RemoveImage(string) error
+}
+
 type mirror struct {
-	dockerClient *docker.Client  // docker client used to pull, tag and push images
+	dockerClient *DockerClient   // docker client used to pull, tag and push images
 	ecrManager   *ecrManager     // ECR manager, used to ensure the ECR repository exist
 	log          *log.Entry      // logrus logger with the relevant custom fields
 	repo         Repository      // repository the mirror
@@ -61,7 +69,6 @@ const defaultSleepDuration time.Duration = 60 * time.Second
 func (m *mirror) setup(repo Repository) (err error) {
 	m.log = log.WithField("full_repo", repo.Name)
 	m.repo = repo
-
 	// specific tag to mirror
 	if strings.Contains(repo.Name, ":") {
 		chunk := strings.SplitN(repo.Name, ":", 2)
@@ -175,7 +182,12 @@ func (m *mirror) pullImage(tag string) error {
 		authConfig.Password = os.Getenv("DOCKERHUB_PASSWORD")
 	}
 
-	return m.dockerClient.PullImage(pullOptions, authConfig)
+	if m.repo.PrivateRegistry != "" {
+		pullOptions.Repository = m.repo.PrivateRegistry + "/" + m.repo.Name
+		return (*m.dockerClient).PullImage(pullOptions, authConfig)
+	}
+
+	return (*m.dockerClient).PullImage(pullOptions, authConfig)
 }
 
 // (re)tag the (local) docker image with the target repository name
@@ -189,7 +201,7 @@ func (m *mirror) tagImage(tag string) error {
 		Force: true,
 	}
 
-	return m.dockerClient.TagImage(fmt.Sprintf("%s:%s", m.repo.Name, tag), tagOptions)
+	return (*m.dockerClient).TagImage(fmt.Sprintf("%s:%s", m.repo.Name, tag), tagOptions)
 }
 
 // push the local (re)tagged image to the target docker registry
@@ -210,20 +222,20 @@ func (m *mirror) pushImage(tag string) error {
 		return err
 	}
 
-	return m.dockerClient.PushImage(pushOptions, *creds)
+	return (*m.dockerClient).PushImage(pushOptions, *creds)
 }
 
 func (m *mirror) deleteImage(tag string) error {
 	repository := fmt.Sprintf("%s:%s", m.repo.Name, tag)
 	m.log.Info("Cleaning images: " + repository)
-	err := m.dockerClient.RemoveImage(repository)
+	err := (*m.dockerClient).RemoveImage(repository)
 	if err != nil {
 		return err
 	}
 
 	target := fmt.Sprintf("%s/%s:%s", config.Target.Registry, m.targetRepositoryName(), tag)
 	m.log.Info("Cleaning images: " + target)
-	err = m.dockerClient.RemoveImage(target)
+	err = (*m.dockerClient).RemoveImage(target)
 	if err != nil {
 		return err
 	}

mirror_test.go

@@ -4,6 +4,8 @@ import (
 	"strconv"
 	"testing"
 	"time"
+
+	docker "github.com/fsouza/go-dockerclient"
 )
 
 func TestGetSleepTime(t *testing.T) {
@@ -39,6 +41,101 @@ func TestGetSleepTime(t *testing.T) {
 
 }
 
+type ResponseContainer struct {
+	TagImageName               string
+	TagImageOptions            docker.TagImageOptions
+	PullImageOptions           docker.PullImageOptions
+	PullImageAuthConfiguration docker.AuthConfiguration
+	PushImageOptions           docker.PushImageOptions
+	PushImageAuthConfiguration docker.AuthConfiguration
+	RemoveImageName            string
+}
+
+type TestDockerClient struct {
+	ResponseContainer *ResponseContainer
+}
+
+func (t *TestDockerClient) Info() (*docker.DockerInfo, error) {
+	return &docker.DockerInfo{}, nil
+}
+
+func (t *TestDockerClient) TagImage(name string, opts docker.TagImageOptions) error {
+	t.ResponseContainer.TagImageName = name
+	t.ResponseContainer.TagImageOptions = opts
+	return nil
+}
+
+func (t *TestDockerClient) PullImage(opts docker.PullImageOptions, authConfig docker.AuthConfiguration) error {
+	t.ResponseContainer.PullImageOptions = opts
+	t.ResponseContainer.PullImageAuthConfiguration = authConfig
+	return nil
+}
+
+func (t *TestDockerClient) PushImage(opts docker.PushImageOptions, auth docker.AuthConfiguration) error {
+	t.ResponseContainer.PushImageOptions = opts
+	t.ResponseContainer.PushImageAuthConfiguration = auth
+	return nil
+}
+
+func (t *TestDockerClient) RemoveImage(name string) error {
+	t.ResponseContainer.RemoveImageName = name
+	return nil
+}
+
+func CreateTestDockerClient(responseContainer *ResponseContainer) *TestDockerClient {
+	return &TestDockerClient{ResponseContainer: responseContainer}
+}
+
+func TestPullImage(t *testing.T) {
+
+	t.Run("tests to ensure that PrivateRegistry creates the proper repo name", func(t *testing.T) {
+		responseContainer := &ResponseContainer{}
+		var client DockerClient
+		client = CreateTestDockerClient(responseContainer)
+		repo := Repository{
+			PrivateRegistry: "private-registry-name",
+			Name:            "elasticsearch",
+		}
+
+		m := mirror{
+			dockerClient: &client,
+		}
+
+		m.setup(repo)
+		m.pullImage("latest")
+
+		got := responseContainer.PullImageOptions.Repository
+		want := "private-registry-name/elasticsearch"
+
+		if got != want {
+			t.Errorf("Expected %q, got %q", want, got)
+		}
+	})
+
+	t.Run("tests to ensure that without PrivateRegistry, a repo name is correct", func(t *testing.T) {
+		responseContainer := &ResponseContainer{}
+		var client DockerClient
+		client = CreateTestDockerClient(responseContainer)
+		repo := Repository{
+			Name: "elasticsearch",
+		}
+
+		m := mirror{
+			dockerClient: &client,
+		}
+
+		m.setup(repo)
+		m.pullImage("latest")
+
+		got := responseContainer.PullImageOptions.Repository
+		want := "elasticsearch"
+
+		if got != want {
+			t.Errorf("Expected %q, got %q", want, got)
+		}
+	})
+}
+
 func getTimeAsString(date time.Time) string {
 	return strconv.FormatInt(date.Unix(), 10)
 }