Add image store

The image store abstracts image handling. It keeps track of the
available images, and makes it possible to delete existing images or
register new ones. The image store holds references to the underlying
layers for each image.

The image/v1 package provides compatibility functions for interoperating
with older (non-content-addressable) image structures.

Signed-off-by: Tonis Tiigi <[email protected]>

Author: tonistiigi

image/fixtures/post1.9/expected_computed_id

@@ -0,0 +1,192 @@
+package image
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+)
+
+// IDWalkFunc is function called by StoreBackend.Walk
+type IDWalkFunc func(id ID) error
+
+// StoreBackend provides interface for image.Store persistence
+type StoreBackend interface {
+	Walk(f IDWalkFunc) error
+	Get(id ID) ([]byte, error)
+	Set(data []byte) (ID, error)
+	Delete(id ID) error
+	SetMetadata(id ID, key string, data []byte) error
+	GetMetadata(id ID, key string) ([]byte, error)
+	DeleteMetadata(id ID, key string) error
+}
+
+// fs implements StoreBackend using the filesystem.
+type fs struct {
+	sync.RWMutex
+	root string
+}
+
+const (
+	contentDirName  = "content"
+	metadataDirName = "metadata"
+)
+
+// NewFSStoreBackend returns new filesystem based backend for image.Store
+func NewFSStoreBackend(root string) (StoreBackend, error) {
+	return newFSStore(root)
+}
+
+func newFSStore(root string) (*fs, error) {
+	s := &fs{
+		root: root,
+	}
+	if err := os.MkdirAll(filepath.Join(root, contentDirName, string(digest.Canonical)), 0700); err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(filepath.Join(root, metadataDirName, string(digest.Canonical)), 0700); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+func (s *fs) contentFile(id ID) string {
+	dgst := digest.Digest(id)
+	return filepath.Join(s.root, contentDirName, string(dgst.Algorithm()), dgst.Hex())
+}
+
+func (s *fs) metadataDir(id ID) string {
+	dgst := digest.Digest(id)
+	return filepath.Join(s.root, metadataDirName, string(dgst.Algorithm()), dgst.Hex())
+}
+
+// Walk calls the supplied callback for each image ID in the storage backend.
+func (s *fs) Walk(f IDWalkFunc) error {
+	// Only Canonical digest (sha256) is currently supported
+	s.RLock()
+	dir, err := ioutil.ReadDir(filepath.Join(s.root, contentDirName, string(digest.Canonical)))
+	s.RUnlock()
+	if err != nil {
+		return err
+	}
+	for _, v := range dir {
+		dgst := digest.NewDigestFromHex(string(digest.Canonical), v.Name())
+		if err := dgst.Validate(); err != nil {
+			logrus.Debugf("Skipping invalid digest %s: %s", dgst, err)
+			continue
+		}
+		if err := f(ID(dgst)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Get returns the content stored under a given ID.
+func (s *fs) Get(id ID) ([]byte, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.get(id)
+}
+
+func (s *fs) get(id ID) ([]byte, error) {
+	content, err := ioutil.ReadFile(s.contentFile(id))
+	if err != nil {
+		return nil, err
+	}
+
+	// todo: maybe optional
+	validated, err := digest.FromBytes(content)
+	if err != nil {
+		return nil, err
+	}
+	if ID(validated) != id {
+		return nil, fmt.Errorf("failed to verify image: %v", id)
+	}
+
+	return content, nil
+}
+
+// Set stores content under a given ID.
+func (s *fs) Set(data []byte) (ID, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	if len(data) == 0 {
+		return "", fmt.Errorf("Invalid empty data")
+	}
+
+	dgst, err := digest.FromBytes(data)
+	if err != nil {
+		return "", err
+	}
+	id := ID(dgst)
+	filePath := s.contentFile(id)
+	tempFilePath := s.contentFile(id) + ".tmp"
+	if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil {
+		return "", err
+	}
+	if err := os.Rename(tempFilePath, filePath); err != nil {
+		return "", err
+	}
+
+	return id, nil
+}
+
+// Delete removes content and metadata files associated with the ID.
+func (s *fs) Delete(id ID) error {
+	s.Lock()
+	defer s.Unlock()
+
+	if err := os.RemoveAll(s.metadataDir(id)); err != nil {
+		return err
+	}
+	if err := os.Remove(s.contentFile(id)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// SetMetadata sets metadata for a given ID. It fails if there's no base file.
+func (s *fs) SetMetadata(id ID, key string, data []byte) error {
+	s.Lock()
+	defer s.Unlock()
+	if _, err := s.get(id); err != nil {
+		return err
+	}
+
+	baseDir := filepath.Join(s.metadataDir(id))
+	if err := os.MkdirAll(baseDir, 0700); err != nil {
+		return err
+	}
+	filePath := filepath.Join(s.metadataDir(id), key)
+	tempFilePath := filePath + ".tmp"
+	if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil {
+		return err
+	}
+	return os.Rename(tempFilePath, filePath)
+}
+
+// GetMetadata returns metadata for a given ID.
+func (s *fs) GetMetadata(id ID, key string) ([]byte, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	if _, err := s.get(id); err != nil {
+		return nil, err
+	}
+	return ioutil.ReadFile(filepath.Join(s.metadataDir(id), key))
+}
+
+// DeleteMetadata removes the metadata associated with an ID.
+func (s *fs) DeleteMetadata(id ID, key string) error {
+	s.Lock()
+	defer s.Unlock()
+
+	return os.RemoveAll(filepath.Join(s.metadataDir(id), key))
+}

image/fixtures/post1.9/expected_config

@@ -0,0 +1,391 @@
+package image
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+)
+
+func TestFSGetSet(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testGetSet(t, fs)
+}
+
+func TestFSGetInvalidData(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id, err := fs.Set([]byte("foobar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dgst := digest.Digest(id)
+
+	if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, string(dgst.Algorithm()), dgst.Hex()), []byte("foobar2"), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = fs.Get(id)
+	if err == nil {
+		t.Fatal("Expected get to fail after data modification.")
+	}
+}
+
+func TestFSInvalidSet(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id, err := digest.FromBytes([]byte("foobar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.Mkdir(filepath.Join(tmpdir, contentDirName, string(id.Algorithm()), id.Hex()), 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = fs.Set([]byte("foobar"))
+	if err == nil {
+		t.Fatal("Expecting error from invalid filesystem data.")
+	}
+}
+
+func TestFSInvalidRoot(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	tcases := []struct {
+		root, invalidFile string
+	}{
+		{"root", "root"},
+		{"root", "root/content"},
+		{"root", "root/metadata"},
+	}
+
+	for _, tc := range tcases {
+		root := filepath.Join(tmpdir, tc.root)
+		filePath := filepath.Join(tmpdir, tc.invalidFile)
+		err := os.MkdirAll(filepath.Dir(filePath), 0700)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f, err := os.Create(filePath)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+
+		_, err = NewFSStoreBackend(root)
+		if err == nil {
+			t.Fatalf("Expected error from root %q and invlid file %q", tc.root, tc.invalidFile)
+		}
+
+		os.RemoveAll(root)
+	}
+
+}
+
+func testMetadataGetSet(t *testing.T, store StoreBackend) {
+	id, err := store.Set([]byte("foo"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	id2, err := store.Set([]byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tcases := []struct {
+		id    ID
+		key   string
+		value []byte
+	}{
+		{id, "tkey", []byte("tval1")},
+		{id, "tkey2", []byte("tval2")},
+		{id2, "tkey", []byte("tval3")},
+	}
+
+	for _, tc := range tcases {
+		err = store.SetMetadata(tc.id, tc.key, tc.value)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		actual, err := store.GetMetadata(tc.id, tc.key)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if bytes.Compare(actual, tc.value) != 0 {
+			t.Fatalf("Metadata expected %q, got %q", tc.value, actual)
+		}
+	}
+
+	_, err = store.GetMetadata(id2, "tkey2")
+	if err == nil {
+		t.Fatal("Expected error for getting metadata for unknown key")
+	}
+
+	id3, err := digest.FromBytes([]byte("baz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = store.SetMetadata(ID(id3), "tkey", []byte("tval"))
+	if err == nil {
+		t.Fatal("Expected error for setting metadata for unknown ID.")
+	}
+
+	_, err = store.GetMetadata(ID(id3), "tkey")
+	if err == nil {
+		t.Fatal("Expected error for getting metadata for unknown ID.")
+	}
+}
+
+func TestFSMetadataGetSet(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testMetadataGetSet(t, fs)
+}
+
+func TestFSDelete(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testDelete(t, fs)
+}
+
+func TestFSWalker(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testWalker(t, fs)
+}
+
+func TestFSInvalidWalker(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fooID, err := fs.Set([]byte("foo"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, "sha256/foobar"), []byte("foobar"), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	n := 0
+	err = fs.Walk(func(id ID) error {
+		if id != fooID {
+			t.Fatalf("Invalid walker ID %q, expected %q", id, fooID)
+		}
+		n++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Invalid data should not have caused walker error, got %v", err)
+	}
+	if n != 1 {
+		t.Fatalf("Expected 1 walk initialization, got %d", n)
+	}
+}
+
+func testGetSet(t *testing.T, store StoreBackend) {
+	type tcase struct {
+		input    []byte
+		expected ID
+	}
+	tcases := []tcase{
+		{[]byte("foobar"), ID("sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")},
+	}
+
+	randomInput := make([]byte, 8*1024)
+	_, err := rand.Read(randomInput)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// skipping use of digest pkg because its used by the imlementation
+	h := sha256.New()
+	_, err = h.Write(randomInput)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tcases = append(tcases, tcase{
+		input:    randomInput,
+		expected: ID("sha256:" + hex.EncodeToString(h.Sum(nil))),
+	})
+
+	for _, tc := range tcases {
+		id, err := store.Set([]byte(tc.input))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if id != tc.expected {
+			t.Fatalf("Expected ID %q, got %q", tc.expected, id)
+		}
+	}
+
+	for _, emptyData := range [][]byte{nil, {}} {
+		_, err := store.Set(emptyData)
+		if err == nil {
+			t.Fatal("Expected error for nil input.")
+		}
+	}
+
+	for _, tc := range tcases {
+		data, err := store.Get(tc.expected)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if bytes.Compare(data, tc.input) != 0 {
+			t.Fatalf("Expected data %q, got %q", tc.input, data)
+		}
+	}
+
+	for _, key := range []ID{"foobar:abc", "sha256:abc", "sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2a"} {
+		_, err := store.Get(key)
+		if err == nil {
+			t.Fatalf("Expected error for ID %q.", key)
+		}
+	}
+
+}
+
+func testDelete(t *testing.T, store StoreBackend) {
+	id, err := store.Set([]byte("foo"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	id2, err := store.Set([]byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = store.Delete(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = store.Get(id)
+	if err == nil {
+		t.Fatalf("Expected getting deleted item %q to fail", id)
+	}
+	_, err = store.Get(id2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = store.Delete(id2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = store.Get(id2)
+	if err == nil {
+		t.Fatalf("Expected getting deleted item %q to fail", id2)
+	}
+}
+
+func testWalker(t *testing.T, store StoreBackend) {
+	id, err := store.Set([]byte("foo"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	id2, err := store.Set([]byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tcases := make(map[ID]struct{})
+	tcases[id] = struct{}{}
+	tcases[id2] = struct{}{}
+	n := 0
+	err = store.Walk(func(id ID) error {
+		delete(tcases, id)
+		n++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if n != 2 {
+		t.Fatalf("Expected 2 walk initializations, got %d", n)
+	}
+	if len(tcases) != 0 {
+		t.Fatalf("Expected empty unwalked set, got %+v", tcases)
+	}
+
+	// stop on error
+	tcases = make(map[ID]struct{})
+	tcases[id] = struct{}{}
+	err = store.Walk(func(id ID) error {
+		return errors.New("")
+	})
+	if err == nil {
+		t.Fatalf("Exected error from walker.")
+	}
+}

image/fixtures/post1.9/layer_id

@@ -2,36 +2,23 @@ package image
 
 import (
 	"encoding/json"
-	"fmt"
-	"regexp"
+	"errors"
+	"io"
 	"time"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/docker/distribution/digest"
-	derr "github.com/docker/docker/errors"
-	"github.com/docker/docker/pkg/version"
 	"github.com/docker/docker/runconfig"
 )
 
-var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
+// ID is the content-addressable ID of an image.
+type ID digest.Digest
 
-// noFallbackMinVersion is the minimum version for which v1compatibility
-// information will not be marshaled through the Image struct to remove
-// blank fields.
-var noFallbackMinVersion = version.Version("1.8.3")
-
-// Descriptor provides the information necessary to register an image in
-// the graph.
-type Descriptor interface {
-	ID() string
-	Parent() string
-	MarshalConfig() ([]byte, error)
+func (id ID) String() string {
+	return digest.Digest(id).String()
 }
 
-// Image stores the image configuration.
-// All fields in this struct must be marked `omitempty` to keep getting
-// predictable hashes from the old `v1Compatibility` configuration.
-type Image struct {
+// V1Image stores the V1 image configuration.
+type V1Image struct {
 	// ID a unique 64 character identifier of the image
 	ID string `json:"id,omitempty"`
 	// Parent id of the image
@@ -55,95 +42,87 @@ type Image struct {
 	// OS is the operating system used to build and run the image
 	OS string `json:"os,omitempty"`
 	// Size is the total size of the image including all layers it is composed of
-	Size int64 `json:",omitempty"` // capitalized for backwards compatibility
-	// ParentID specifies the strong, content address of the parent configuration.
-	ParentID digest.Digest `json:"parent_id,omitempty"`
-	// LayerID provides the content address of the associated layer.
-	LayerID digest.Digest `json:"layer_id,omitempty"`
+	Size int64 `json:",omitempty"`
 }
 
-// NewImgJSON creates an Image configuration from json.
-func NewImgJSON(src []byte) (*Image, error) {
-	ret := &Image{}
+// Image stores the image configuration
+type Image struct {
+	V1Image
+	Parent  ID        `json:"parent,omitempty"`
+	RootFS  *RootFS   `json:"rootfs,omitempty"`
+	History []History `json:"history,omitempty"`
 
-	// FIXME: Is there a cleaner way to "purify" the input json?
-	if err := json.Unmarshal(src, ret); err != nil {
-		return nil, err
-	}
-	return ret, nil
+	// rawJSON caches the immutable JSON associated with this image.
+	rawJSON []byte
+
+	// computedID is the ID computed from the hash of the image config.
+	// Not to be confused with the legacy V1 ID in V1Image.
+	computedID ID
 }
 
-// ValidateID checks whether an ID string is a valid image ID.
-func ValidateID(id string) error {
-	if ok := validHex.MatchString(id); !ok {
-		return derr.ErrorCodeInvalidImageID.WithArgs(id)
-	}
-	return nil
+// RawJSON returns the immutable JSON associated with the image.
+func (img *Image) RawJSON() []byte {
+	return img.rawJSON
+}
+
+// ID returns the image's content-addressable ID.
+func (img *Image) ID() ID {
+	return img.computedID
 }
 
-// MakeImageConfig returns immutable configuration JSON for image based on the
-// v1Compatibility object, layer digest and parent StrongID. SHA256() of this
-// config is the new image ID (strongID).
-func MakeImageConfig(v1Compatibility []byte, layerID, parentID digest.Digest) ([]byte, error) {
+// MarshalJSON serializes the image to JSON. It sorts the top-level keys so
+// that JSON that's been manipulated by a push/pull cycle with a legacy
+// registry won't end up with a different key order.
+func (img *Image) MarshalJSON() ([]byte, error) {
+	type MarshalImage Image
 
-	// Detect images created after 1.8.3
-	img, err := NewImgJSON(v1Compatibility)
+	pass1, err := json.Marshal(MarshalImage(*img))
 	if err != nil {
 		return nil, err
 	}
-	useFallback := version.Version(img.DockerVersion).LessThan(noFallbackMinVersion)
-
-	if useFallback {
-		// Fallback for pre-1.8.3. Calculate base config based on Image struct
-		// so that fields with default values added by Docker will use same ID
-		logrus.Debugf("Using fallback hash for %v", layerID)
-
-		v1Compatibility, err = json.Marshal(img)
-		if err != nil {
-			return nil, err
-		}
-	}
 
 	var c map[string]*json.RawMessage
-	if err := json.Unmarshal(v1Compatibility, &c); err != nil {
+	if err := json.Unmarshal(pass1, &c); err != nil {
 		return nil, err
 	}
-
-	if err := layerID.Validate(); err != nil {
-		return nil, fmt.Errorf("invalid layerID: %v", err)
-	}
-
-	c["layer_id"] = rawJSON(layerID)
-
-	if parentID != "" {
-		if err := parentID.Validate(); err != nil {
-			return nil, fmt.Errorf("invalid parentID %v", err)
-		}
-		c["parent_id"] = rawJSON(parentID)
-	}
-
-	delete(c, "id")
-	delete(c, "parent")
-	delete(c, "Size") // Size is calculated from data on disk and is inconsitent
-
 	return json.Marshal(c)
 }
 
-// StrongID returns image ID for the config JSON.
-func StrongID(configJSON []byte) (digest.Digest, error) {
-	digester := digest.Canonical.New()
-	if _, err := digester.Hash().Write(configJSON); err != nil {
-		return "", err
-	}
-	dgst := digester.Digest()
-	logrus.Debugf("H(%v) = %v", string(configJSON), dgst)
-	return dgst, nil
+// History stores build commands that were used to create an image
+type History struct {
+	// Created timestamp for build point
+	Created time.Time `json:"created"`
+	// Author of the build point
+	Author string `json:"author,omitempty"`
+	// CreatedBy keeps the Dockerfile command used while building image.
+	CreatedBy string `json:"created_by,omitempty"`
+	// Comment is custom mesage set by the user when creating the image.
+	Comment string `json:"comment,omitempty"`
+	// EmptyLayer is set to true if this history item did not generate a
+	// layer. Otherwise, the history item is associated with the next
+	// layer in the RootFS section.
+	EmptyLayer bool `json:"empty_layer,omitempty"`
 }
 
-func rawJSON(value interface{}) *json.RawMessage {
-	jsonval, err := json.Marshal(value)
-	if err != nil {
-		return nil
+// Exporter provides interface for exporting and importing images
+type Exporter interface {
+	Load(io.ReadCloser, io.Writer) error
+	// TODO: Load(net.Context, io.ReadCloser, <- chan StatusMessage) error
+	Save([]string, io.Writer) error
+}
+
+// NewFromJSON creates an Image configuration from json.
+func NewFromJSON(src []byte) (*Image, error) {
+	img := &Image{}
+
+	if err := json.Unmarshal(src, img); err != nil {
+		return nil, err
 	}
-	return (*json.RawMessage)(&jsonval)
+	if img.RootFS == nil {
+		return nil, errors.New("Invalid image JSON, no RootFS key.")
+	}
+
+	img.rawJSON = src
+
+	return img, nil
 }

image/fixtures/post1.9/parent_id

@@ -1,55 +1,59 @@
 package image
 
 import (
-	"bytes"
-	"io/ioutil"
+	"encoding/json"
+	"sort"
+	"strings"
 	"testing"
-
-	"github.com/docker/distribution/digest"
 )
 
-var fixtures = []string{
-	"fixtures/pre1.9",
-	"fixtures/post1.9",
-}
+const sampleImageJSON = `{
+	"architecture": "amd64",
+	"os": "linux",
+	"config": {},
+	"rootfs": {
+		"type": "layers",
+		"diff_ids": []
+	}
+}`
 
-func loadFixtureFile(t *testing.T, path string) []byte {
-	fileData, err := ioutil.ReadFile(path)
+func TestJSON(t *testing.T) {
+	img, err := NewFromJSON([]byte(sampleImageJSON))
 	if err != nil {
-		t.Fatalf("error opening %s: %v", path, err)
+		t.Fatal(err)
+	}
+	rawJSON := img.RawJSON()
+	if string(rawJSON) != sampleImageJSON {
+		t.Fatalf("Raw JSON of config didn't match: expected %+v, got %v", sampleImageJSON, rawJSON)
 	}
-
-	return bytes.TrimSpace(fileData)
 }
 
-// TestMakeImageConfig makes sure that MakeImageConfig returns the expected
-// canonical JSON for a reference Image.
-func TestMakeImageConfig(t *testing.T) {
-	for _, fixture := range fixtures {
-		v1Compatibility := loadFixtureFile(t, fixture+"/v1compatibility")
-		expectedConfig := loadFixtureFile(t, fixture+"/expected_config")
-		layerID := digest.Digest(loadFixtureFile(t, fixture+"/layer_id"))
-		parentID := digest.Digest(loadFixtureFile(t, fixture+"/parent_id"))
-
-		json, err := MakeImageConfig(v1Compatibility, layerID, parentID)
-		if err != nil {
-			t.Fatalf("MakeImageConfig on %s returned error: %v", fixture, err)
-		}
-		if !bytes.Equal(json, expectedConfig) {
-			t.Fatalf("did not get expected JSON for %s\nexpected: %s\ngot: %s", fixture, expectedConfig, json)
-		}
+func TestInvalidJSON(t *testing.T) {
+	_, err := NewFromJSON([]byte("{}"))
+	if err == nil {
+		t.Fatal("Expected JSON parse error")
 	}
 }
 
-// TestGetStrongID makes sure that GetConfigJSON returns the expected
-// hash for a reference Image.
-func TestGetStrongID(t *testing.T) {
-	for _, fixture := range fixtures {
-		expectedConfig := loadFixtureFile(t, fixture+"/expected_config")
-		expectedComputedID := digest.Digest(loadFixtureFile(t, fixture+"/expected_computed_id"))
+func TestMarshalKeyOrder(t *testing.T) {
+	b, err := json.Marshal(&Image{
+		V1Image: V1Image{
+			Comment:      "a",
+			Author:       "b",
+			Architecture: "c",
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedOrder := []string{"architecture", "author", "comment"}
+	var indexes []int
+	for _, k := range expectedOrder {
+		indexes = append(indexes, strings.Index(string(b), k))
+	}
 
-		if id, err := StrongID(expectedConfig); err != nil || id != expectedComputedID {
-			t.Fatalf("did not get expected ID for %s\nexpected: %s\ngot: %s\nerror: %v", fixture, expectedComputedID, id, err)
-		}
+	if !sort.IntsAreSorted(indexes) {
+		t.Fatal("invalid key order in JSON: ", string(b))
 	}
 }

image/fixtures/post1.9/v1compatibility

@@ -0,0 +1,8 @@
+package image
+
+import "github.com/docker/docker/layer"
+
+// Append appends a new diffID to rootfs
+func (r *RootFS) Append(id layer.DiffID) {
+	r.DiffIDs = append(r.DiffIDs, id)
+}

image/fixtures/pre1.9/expected_computed_id

@@ -0,0 +1,23 @@
+// +build !windows
+
+package image
+
+import "github.com/docker/docker/layer"
+
+// RootFS describes images root filesystem
+// This is currently a placeholder that only supports layers. In the future
+// this can be made into a interface that supports different implementaions.
+type RootFS struct {
+	Type    string         `json:"type"`
+	DiffIDs []layer.DiffID `json:"diff_ids,omitempty"`
+}
+
+// ChainID returns the ChainID for the top layer in RootFS.
+func (r *RootFS) ChainID() layer.ChainID {
+	return layer.CreateChainID(r.DiffIDs)
+}
+
+// NewRootFS returns empty RootFS struct
+func NewRootFS() *RootFS {
+	return &RootFS{Type: "layers"}
+}

image/fixtures/pre1.9/expected_config

@@ -0,0 +1,37 @@
+// +build windows
+
+package image
+
+import (
+	"crypto/sha512"
+	"fmt"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/layer"
+)
+
+// RootFS describes images root filesystem
+// This is currently a placeholder that only supports layers. In the future
+// this can be made into a interface that supports different implementaions.
+type RootFS struct {
+	Type      string         `json:"type"`
+	DiffIDs   []layer.DiffID `json:"diff_ids,omitempty"`
+	BaseLayer string         `json:"base_layer,omitempty"`
+}
+
+// BaseLayerID returns the 64 byte hex ID for the baselayer name.
+func (r *RootFS) BaseLayerID() string {
+	baseID := sha512.Sum384([]byte(r.BaseLayer))
+	return fmt.Sprintf("%x", baseID[:32])
+}
+
+// ChainID returns the ChainID for the top layer in RootFS.
+func (r *RootFS) ChainID() layer.ChainID {
+	baseDiffID, _ := digest.FromBytes([]byte(r.BaseLayerID())) // can never error
+	return layer.CreateChainID(append([]layer.DiffID{layer.DiffID(baseDiffID)}, r.DiffIDs...))
+}
+
+// NewRootFS returns empty RootFS struct
+func NewRootFS() *RootFS {
+	return &RootFS{Type: "layers+base"}
+}

image/fixtures/pre1.9/layer_id

@@ -0,0 +1,286 @@
+package image
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/layer"
+)
+
+// Store is an interface for creating and accessing images
+type Store interface {
+	Create(config []byte) (ID, error)
+	Get(id ID) (*Image, error)
+	Delete(id ID) ([]layer.Metadata, error)
+	Search(partialID string) (ID, error)
+	SetParent(id ID, parent ID) error
+	GetParent(id ID) (ID, error)
+	Children(id ID) []ID
+	Map() map[ID]*Image
+	Heads() map[ID]*Image
+}
+
+// LayerGetReleaser is a minimal interface for getting and releasing images.
+type LayerGetReleaser interface {
+	Get(layer.ChainID) (layer.Layer, error)
+	Release(layer.Layer) ([]layer.Metadata, error)
+}
+
+type imageMeta struct {
+	layer    layer.Layer
+	children map[ID]struct{}
+}
+
+type store struct {
+	sync.Mutex
+	ls        LayerGetReleaser
+	images    map[ID]*imageMeta
+	fs        StoreBackend
+	digestSet *digest.Set
+}
+
+// NewImageStore returns new store object for given layer store
+func NewImageStore(fs StoreBackend, ls LayerGetReleaser) (Store, error) {
+	is := &store{
+		ls:        ls,
+		images:    make(map[ID]*imageMeta),
+		fs:        fs,
+		digestSet: digest.NewSet(),
+	}
+
+	// load all current images and retain layers
+	if err := is.restore(); err != nil {
+		return nil, err
+	}
+
+	return is, nil
+}
+
+func (is *store) restore() error {
+	err := is.fs.Walk(func(id ID) error {
+		img, err := is.Get(id)
+		if err != nil {
+			logrus.Errorf("invalid image %v, %v", id, err)
+			return nil
+		}
+		var l layer.Layer
+		if chainID := img.RootFS.ChainID(); chainID != "" {
+			l, err = is.ls.Get(chainID)
+			if err != nil {
+				return err
+			}
+		}
+		if err := is.digestSet.Add(digest.Digest(id)); err != nil {
+			return err
+		}
+
+		imageMeta := &imageMeta{
+			layer:    l,
+			children: make(map[ID]struct{}),
+		}
+
+		is.images[ID(id)] = imageMeta
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// Second pass to fill in children maps
+	for id := range is.images {
+		if parent, err := is.GetParent(id); err == nil {
+			if parentMeta := is.images[parent]; parentMeta != nil {
+				parentMeta.children[id] = struct{}{}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (is *store) Create(config []byte) (ID, error) {
+	var img Image
+	err := json.Unmarshal(config, &img)
+	if err != nil {
+		return "", err
+	}
+
+	// Must reject any config that references diffIDs from the history
+	// which aren't among the rootfs layers.
+	rootFSLayers := make(map[layer.DiffID]struct{})
+	for _, diffID := range img.RootFS.DiffIDs {
+		rootFSLayers[diffID] = struct{}{}
+	}
+
+	layerCounter := 0
+	for _, h := range img.History {
+		if !h.EmptyLayer {
+			layerCounter++
+		}
+	}
+	if layerCounter > len(img.RootFS.DiffIDs) {
+		return "", errors.New("too many non-empty layers in History section")
+	}
+
+	dgst, err := is.fs.Set(config)
+	if err != nil {
+		return "", err
+	}
+	imageID := ID(dgst)
+
+	is.Lock()
+	defer is.Unlock()
+
+	if _, exists := is.images[imageID]; exists {
+		return imageID, nil
+	}
+
+	layerID := img.RootFS.ChainID()
+
+	var l layer.Layer
+	if layerID != "" {
+		l, err = is.ls.Get(layerID)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	imageMeta := &imageMeta{
+		layer:    l,
+		children: make(map[ID]struct{}),
+	}
+
+	is.images[imageID] = imageMeta
+	if err := is.digestSet.Add(digest.Digest(imageID)); err != nil {
+		delete(is.images, imageID)
+		return "", err
+	}
+
+	return imageID, nil
+}
+
+func (is *store) Search(term string) (ID, error) {
+	is.Lock()
+	defer is.Unlock()
+
+	dgst, err := is.digestSet.Lookup(term)
+	if err != nil {
+		return "", err
+	}
+	return ID(dgst), nil
+}
+
+func (is *store) Get(id ID) (*Image, error) {
+	// todo: Check if image is in images
+	// todo: Detect manual insertions and start using them
+	config, err := is.fs.Get(id)
+	if err != nil {
+		return nil, err
+	}
+
+	img, err := NewFromJSON(config)
+	if err != nil {
+		return nil, err
+	}
+	img.computedID = id
+
+	img.Parent, err = is.GetParent(id)
+	if err != nil {
+		img.Parent = ""
+	}
+
+	return img, nil
+}
+
+func (is *store) Delete(id ID) ([]layer.Metadata, error) {
+	is.Lock()
+	defer is.Unlock()
+
+	imageMeta := is.images[id]
+	if imageMeta == nil {
+		return nil, fmt.Errorf("unrecognized image ID %s", id.String())
+	}
+	for id := range imageMeta.children {
+		is.fs.DeleteMetadata(id, "parent")
+	}
+	if parent, err := is.GetParent(id); err == nil && is.images[parent] != nil {
+		delete(is.images[parent].children, id)
+	}
+
+	delete(is.images, id)
+	is.fs.Delete(id)
+
+	if imageMeta.layer != nil {
+		return is.ls.Release(imageMeta.layer)
+	}
+	return nil, nil
+}
+
+func (is *store) SetParent(id, parent ID) error {
+	is.Lock()
+	defer is.Unlock()
+	parentMeta := is.images[parent]
+	if parentMeta == nil {
+		return fmt.Errorf("unknown parent image ID %s", parent.String())
+	}
+	parentMeta.children[id] = struct{}{}
+	return is.fs.SetMetadata(id, "parent", []byte(parent))
+}
+
+func (is *store) GetParent(id ID) (ID, error) {
+	d, err := is.fs.GetMetadata(id, "parent")
+	if err != nil {
+		return "", err
+	}
+	return ID(d), nil // todo: validate?
+}
+
+func (is *store) Children(id ID) []ID {
+	is.Lock()
+	defer is.Unlock()
+
+	return is.children(id)
+}
+
+func (is *store) children(id ID) []ID {
+	var ids []ID
+	if is.images[id] != nil {
+		for id := range is.images[id].children {
+			ids = append(ids, id)
+		}
+	}
+	return ids
+}
+
+func (is *store) Heads() map[ID]*Image {
+	return is.imagesMap(false)
+}
+
+func (is *store) Map() map[ID]*Image {
+	return is.imagesMap(true)
+}
+
+func (is *store) imagesMap(all bool) map[ID]*Image {
+	is.Lock()
+	defer is.Unlock()
+
+	images := make(map[ID]*Image)
+
+	for id := range is.images {
+		if !all && len(is.children(id)) > 0 {
+			continue
+		}
+		img, err := is.Get(id)
+		if err != nil {
+			logrus.Errorf("invalid image access: %q, error: %q", id, err)
+			continue
+		}
+		images[id] = img
+	}
+	return images
+}

image/fixtures/pre1.9/parent_id

@@ -0,0 +1,205 @@
+package image
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/layer"
+)
+
+func TestRestore(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id1, err := fs.Set([]byte(`{"comment": "abc", "rootfs": {"type": "layers"}}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fs.Set([]byte(`invalid`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	id2, err := fs.Set([]byte(`{"comment": "def", "rootfs": {"type": "layers", "diff_ids": ["2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"]}}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = fs.SetMetadata(id2, "parent", []byte(id1))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	is, err := NewImageStore(fs, &mockLayerGetReleaser{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	imgs := is.Map()
+	if actual, expected := len(imgs), 2; actual != expected {
+		t.Fatalf("invalid images length, expected 2, got %q", len(imgs))
+	}
+
+	img1, err := is.Get(ID(id1))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if actual, expected := img1.computedID, ID(id1); actual != expected {
+		t.Fatalf("invalid image ID: expected %q, got %q", expected, actual)
+	}
+
+	if actual, expected := img1.computedID.String(), string(id1); actual != expected {
+		t.Fatalf("invalid image ID string: expected %q, got %q", expected, actual)
+	}
+
+	img2, err := is.Get(ID(id2))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if actual, expected := img1.Comment, "abc"; actual != expected {
+		t.Fatalf("invalid comment for image1: expected %q, got %q", expected, actual)
+	}
+
+	if actual, expected := img2.Comment, "def"; actual != expected {
+		t.Fatalf("invalid comment for image2: expected %q, got %q", expected, actual)
+	}
+
+	p, err := is.GetParent(ID(id1))
+	if err == nil {
+		t.Fatal("expected error for getting parent")
+	}
+
+	p, err = is.GetParent(ID(id2))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if actual, expected := p, ID(id1); actual != expected {
+		t.Fatalf("invalid parent: expected %q, got %q", expected, actual)
+	}
+
+	children := is.Children(ID(id1))
+	if len(children) != 1 {
+		t.Fatalf("invalid children length: %q", len(children))
+	}
+	if actual, expected := children[0], ID(id2); actual != expected {
+		t.Fatalf("invalid child for id1: expected %q, got %q", expected, actual)
+	}
+
+	heads := is.Heads()
+	if actual, expected := len(heads), 1; actual != expected {
+		t.Fatalf("invalid images length: expected %q, got %q", expected, actual)
+	}
+
+	sid1, err := is.Search(string(id1)[:10])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if actual, expected := sid1, ID(id1); actual != expected {
+		t.Fatalf("searched ID mismatch: expected %q, got %q", expected, actual)
+	}
+
+	sid1, err = is.Search(digest.Digest(id1).Hex()[:6])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if actual, expected := sid1, ID(id1); actual != expected {
+		t.Fatalf("searched ID mismatch: expected %q, got %q", expected, actual)
+	}
+
+	invalidPattern := digest.Digest(id1).Hex()[1:6]
+	_, err = is.Search(invalidPattern)
+	if err == nil {
+		t.Fatalf("expected search for %q to fail", invalidPattern)
+	}
+
+}
+
+func TestAddDelete(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "images-fs-store")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	fs, err := NewFSStoreBackend(tmpdir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	is, err := NewImageStore(fs, &mockLayerGetReleaser{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id1, err := is.Create([]byte(`{"comment": "abc", "rootfs": {"type": "layers", "diff_ids": ["2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"]}}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if actual, expected := id1, ID("sha256:8d25a9c45df515f9d0fe8e4a6b1c64dd3b965a84790ddbcc7954bb9bc89eb993"); actual != expected {
+		t.Fatalf("create ID mismatch: expected %q, got %q", expected, actual)
+	}
+
+	img, err := is.Get(id1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if actual, expected := img.Comment, "abc"; actual != expected {
+		t.Fatalf("invalid comment in image: expected %q, got %q", expected, actual)
+	}
+
+	id2, err := is.Create([]byte(`{"comment": "def", "rootfs": {"type": "layers", "diff_ids": ["2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"]}}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = is.SetParent(id2, id1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pid1, err := is.GetParent(id2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if actual, expected := pid1, id1; actual != expected {
+		t.Fatalf("invalid parent for image: expected %q, got %q", expected, actual)
+	}
+
+	_, err = is.Delete(id1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = is.Get(id1)
+	if err == nil {
+		t.Fatalf("expected get for deleted image %q to fail", id1)
+	}
+	_, err = is.Get(id2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid1, err = is.GetParent(id2)
+	if err == nil {
+		t.Fatalf("expected parent check for image %q to fail, got %q", id2, pid1)
+	}
+
+}
+
+type mockLayerGetReleaser struct{}
+
+func (ls *mockLayerGetReleaser) Get(layer.ChainID) (layer.Layer, error) {
+	return nil, nil
+}
+
+func (ls *mockLayerGetReleaser) Release(layer.Layer) ([]layer.Metadata, error) {
+	return nil, nil
+}

image/fixtures/pre1.9/v1compatibility

@@ -0,0 +1,284 @@
+package tarexport
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/image/v1"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/chrootarchive"
+	"github.com/docker/docker/pkg/symlink"
+)
+
+func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer) error {
+	tmpDir, err := ioutil.TempDir("", "docker-import-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil {
+		return err
+	}
+	// read manifest, if no file then load in legacy mode
+	manifestPath, err := safePath(tmpDir, manifestFileName)
+	if err != nil {
+		return err
+	}
+	manifestFile, err := os.Open(manifestPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return l.legacyLoad(tmpDir, outStream)
+		}
+		return manifestFile.Close()
+	}
+	defer manifestFile.Close()
+
+	var manifest []manifestItem
+	if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
+		return err
+	}
+
+	for _, m := range manifest {
+		configPath, err := safePath(tmpDir, m.Config)
+		if err != nil {
+			return err
+		}
+		config, err := ioutil.ReadFile(configPath)
+		if err != nil {
+			return err
+		}
+		img, err := image.NewFromJSON(config)
+		if err != nil {
+			return err
+		}
+		var rootFS image.RootFS
+		rootFS = *img.RootFS
+		rootFS.DiffIDs = nil
+
+		if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
+			return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual)
+		}
+
+		for i, diffID := range img.RootFS.DiffIDs {
+			layerPath, err := safePath(tmpDir, m.Layers[i])
+			if err != nil {
+				return err
+			}
+			newLayer, err := l.loadLayer(layerPath, rootFS)
+			if err != nil {
+				return err
+			}
+			defer layer.ReleaseAndLog(l.ls, newLayer)
+			if expected, actual := diffID, newLayer.DiffID(); expected != actual {
+				return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
+			}
+			rootFS.Append(diffID)
+		}
+
+		imgID, err := l.is.Create(config)
+		if err != nil {
+			return err
+		}
+
+		for _, repoTag := range m.RepoTags {
+			named, err := reference.ParseNamed(repoTag)
+			if err != nil {
+				return err
+			}
+			ref, ok := named.(reference.NamedTagged)
+			if !ok {
+				return fmt.Errorf("invalid tag %q", repoTag)
+			}
+			l.setLoadedTag(ref, imgID, outStream)
+		}
+
+	}
+
+	return nil
+}
+
+func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS) (layer.Layer, error) {
+	rawTar, err := os.Open(filename)
+	if err != nil {
+		logrus.Debugf("Error reading embedded tar: %v", err)
+		return nil, err
+	}
+	inflatedLayerData, err := archive.DecompressStream(rawTar)
+	if err != nil {
+		return nil, err
+	}
+
+	defer rawTar.Close()
+	defer inflatedLayerData.Close()
+
+	return l.ls.Register(inflatedLayerData, rootFS.ChainID())
+}
+
+func (l *tarexporter) setLoadedTag(ref reference.NamedTagged, imgID image.ID, outStream io.Writer) error {
+	if prevID, err := l.ts.Get(ref); err == nil && prevID != imgID {
+		fmt.Fprintf(outStream, "The image %s already exists, renaming the old one with ID %s to empty string\n", ref.String(), string(prevID)) // todo: this message is wrong in case of multiple tags
+	}
+
+	if err := l.ts.Add(ref, imgID, true); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (l *tarexporter) legacyLoad(tmpDir string, outStream io.Writer) error {
+	legacyLoadedMap := make(map[string]image.ID)
+
+	dirs, err := ioutil.ReadDir(tmpDir)
+	if err != nil {
+		return err
+	}
+
+	// every dir represents an image
+	for _, d := range dirs {
+		if d.IsDir() {
+			if err := l.legacyLoadImage(d.Name(), tmpDir, legacyLoadedMap); err != nil {
+				return err
+			}
+		}
+	}
+
+	// load tags from repositories file
+	repositoriesPath, err := safePath(tmpDir, legacyRepositoriesFileName)
+	if err != nil {
+		return err
+	}
+	repositoriesFile, err := os.Open(repositoriesPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+		return repositoriesFile.Close()
+	}
+	defer repositoriesFile.Close()
+
+	repositories := make(map[string]map[string]string)
+	if err := json.NewDecoder(repositoriesFile).Decode(&repositories); err != nil {
+		return err
+	}
+
+	for name, tagMap := range repositories {
+		for tag, oldID := range tagMap {
+			imgID, ok := legacyLoadedMap[oldID]
+			if !ok {
+				return fmt.Errorf("invalid target ID: %v", oldID)
+			}
+			named, err := reference.WithName(name)
+			if err != nil {
+				return err
+			}
+			ref, err := reference.WithTag(named, tag)
+			if err != nil {
+				return err
+			}
+			l.setLoadedTag(ref, imgID, outStream)
+		}
+	}
+
+	return nil
+}
+
+func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[string]image.ID) error {
+	if _, loaded := loadedMap[oldID]; loaded {
+		return nil
+	}
+	configPath, err := safePath(sourceDir, filepath.Join(oldID, legacyConfigFileName))
+	if err != nil {
+		return err
+	}
+	imageJSON, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		logrus.Debugf("Error reading json: %v", err)
+		return err
+	}
+
+	var img struct{ Parent string }
+	if err := json.Unmarshal(imageJSON, &img); err != nil {
+		return err
+	}
+
+	var parentID image.ID
+	if img.Parent != "" {
+		for {
+			var loaded bool
+			if parentID, loaded = loadedMap[img.Parent]; !loaded {
+				if err := l.legacyLoadImage(img.Parent, sourceDir, loadedMap); err != nil {
+					return err
+				}
+			} else {
+				break
+			}
+		}
+	}
+
+	// todo: try to connect with migrate code
+	rootFS := image.NewRootFS()
+	var history []image.History
+
+	if parentID != "" {
+		parentImg, err := l.is.Get(parentID)
+		if err != nil {
+			return err
+		}
+
+		rootFS = parentImg.RootFS
+		history = parentImg.History
+	}
+
+	layerPath, err := safePath(sourceDir, filepath.Join(oldID, legacyLayerFileName))
+	if err != nil {
+		return err
+	}
+	newLayer, err := l.loadLayer(layerPath, *rootFS)
+	if err != nil {
+		return err
+	}
+	rootFS.Append(newLayer.DiffID())
+
+	h, err := v1.HistoryFromConfig(imageJSON, false)
+	if err != nil {
+		return err
+	}
+	history = append(history, h)
+
+	config, err := v1.MakeConfigFromV1Config(imageJSON, rootFS, history)
+	if err != nil {
+		return err
+	}
+	imgID, err := l.is.Create(config)
+	if err != nil {
+		return err
+	}
+
+	metadata, err := l.ls.Release(newLayer)
+	layer.LogReleaseMetadata(metadata)
+	if err != nil {
+		return err
+	}
+
+	if parentID != "" {
+		if err := l.is.SetParent(imgID, parentID); err != nil {
+			return err
+		}
+	}
+
+	loadedMap[oldID] = imgID
+	return nil
+}
+
+func safePath(base, path string) (string, error) {
+	return symlink.FollowSymlinkInScope(filepath.Join(base, path), base)
+}

image/fs.go

@@ -0,0 +1,303 @@
+package tarexport
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/image/v1"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/registry"
+	"github.com/docker/docker/tag"
+)
+
+type imageDescriptor struct {
+	refs   []reference.NamedTagged
+	layers []string
+}
+
+type saveSession struct {
+	*tarexporter
+	outDir      string
+	images      map[image.ID]*imageDescriptor
+	savedLayers map[string]struct{}
+}
+
+func (l *tarexporter) Save(names []string, outStream io.Writer) error {
+	images, err := l.parseNames(names)
+	if err != nil {
+		return err
+	}
+
+	return (&saveSession{tarexporter: l, images: images}).save(outStream)
+}
+
+func (l *tarexporter) parseNames(names []string) (map[image.ID]*imageDescriptor, error) {
+	imgDescr := make(map[image.ID]*imageDescriptor)
+
+	addAssoc := func(id image.ID, ref reference.Named) {
+		if _, ok := imgDescr[id]; !ok {
+			imgDescr[id] = &imageDescriptor{}
+		}
+
+		if ref != nil {
+			var tagged reference.NamedTagged
+			if _, ok := ref.(reference.Digested); ok {
+				return
+			}
+			var ok bool
+			if tagged, ok = ref.(reference.NamedTagged); !ok {
+				var err error
+				if tagged, err = reference.WithTag(ref, tag.DefaultTag); err != nil {
+					return
+				}
+			}
+
+			for _, t := range imgDescr[id].refs {
+				if tagged.String() == t.String() {
+					return
+				}
+			}
+			imgDescr[id].refs = append(imgDescr[id].refs, tagged)
+		}
+	}
+
+	for _, name := range names {
+		ref, err := reference.ParseNamed(name)
+		if err != nil {
+			return nil, err
+		}
+		ref = registry.NormalizeLocalReference(ref)
+		if ref.Name() == string(digest.Canonical) {
+			imgID, err := l.is.Search(name)
+			if err != nil {
+				return nil, err
+			}
+			addAssoc(imgID, nil)
+			continue
+		}
+		if _, ok := ref.(reference.Digested); !ok {
+			if _, ok := ref.(reference.NamedTagged); !ok {
+				assocs := l.ts.ReferencesByName(ref)
+				for _, assoc := range assocs {
+					addAssoc(assoc.ImageID, assoc.Ref)
+				}
+				if len(assocs) == 0 {
+					imgID, err := l.is.Search(name)
+					if err != nil {
+						return nil, err
+					}
+					addAssoc(imgID, nil)
+				}
+				continue
+			}
+		}
+		var imgID image.ID
+		if imgID, err = l.ts.Get(ref); err != nil {
+			return nil, err
+		}
+		addAssoc(imgID, ref)
+
+	}
+	return imgDescr, nil
+}
+
+func (s *saveSession) save(outStream io.Writer) error {
+	s.savedLayers = make(map[string]struct{})
+
+	// get image json
+	tempDir, err := ioutil.TempDir("", "docker-export-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+
+	s.outDir = tempDir
+	reposLegacy := make(map[string]map[string]string)
+
+	var manifest []manifestItem
+
+	for id, imageDescr := range s.images {
+		if err = s.saveImage(id); err != nil {
+			return err
+		}
+
+		var repoTags []string
+		var layers []string
+
+		for _, ref := range imageDescr.refs {
+			if _, ok := reposLegacy[ref.Name()]; !ok {
+				reposLegacy[ref.Name()] = make(map[string]string)
+			}
+			reposLegacy[ref.Name()][ref.Tag()] = imageDescr.layers[len(imageDescr.layers)-1]
+			repoTags = append(repoTags, ref.String())
+		}
+
+		for _, l := range imageDescr.layers {
+			layers = append(layers, filepath.Join(l, legacyLayerFileName))
+		}
+
+		manifest = append(manifest, manifestItem{
+			Config:   digest.Digest(id).Hex() + ".json",
+			RepoTags: repoTags,
+			Layers:   layers,
+		})
+	}
+
+	if len(reposLegacy) > 0 {
+		reposFile := filepath.Join(tempDir, legacyRepositoriesFileName)
+		f, err := os.OpenFile(reposFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		if err != nil {
+			f.Close()
+			return err
+		}
+		if err := json.NewEncoder(f).Encode(reposLegacy); err != nil {
+			return err
+		}
+		if err := f.Close(); err != nil {
+			return err
+		}
+		if err := os.Chtimes(reposFile, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
+			return err
+		}
+	}
+
+	manifestFileName := filepath.Join(tempDir, manifestFileName)
+	f, err := os.OpenFile(manifestFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		f.Close()
+		return err
+	}
+	if err := json.NewEncoder(f).Encode(manifest); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	if err := os.Chtimes(manifestFileName, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
+		return err
+	}
+
+	fs, err := archive.Tar(tempDir, archive.Uncompressed)
+	if err != nil {
+		return err
+	}
+	defer fs.Close()
+
+	if _, err := io.Copy(outStream, fs); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *saveSession) saveImage(id image.ID) error {
+	img, err := s.is.Get(id)
+	if err != nil {
+		return err
+	}
+
+	if len(img.RootFS.DiffIDs) == 0 {
+		return fmt.Errorf("empty export - not implemented")
+	}
+
+	var parent digest.Digest
+	var layers []string
+	for i := range img.RootFS.DiffIDs {
+		v1Img := image.V1Image{}
+		if i == len(img.RootFS.DiffIDs)-1 {
+			v1Img = img.V1Image
+		}
+		rootFS := *img.RootFS
+		rootFS.DiffIDs = rootFS.DiffIDs[:i+1]
+		v1ID, err := v1.CreateID(v1Img, rootFS.ChainID(), parent)
+		if err != nil {
+			return err
+		}
+
+		v1Img.ID = v1ID.Hex()
+		if parent != "" {
+			v1Img.Parent = parent.Hex()
+		}
+
+		if err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created); err != nil {
+			return err
+		}
+		layers = append(layers, v1Img.ID)
+		parent = v1ID
+	}
+
+	configFile := filepath.Join(s.outDir, digest.Digest(id).Hex()+".json")
+	if err := ioutil.WriteFile(configFile, img.RawJSON(), 0644); err != nil {
+		return err
+	}
+	if err := os.Chtimes(configFile, img.Created, img.Created); err != nil {
+		return err
+	}
+
+	s.images[id].layers = layers
+	return nil
+}
+
+func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) error {
+	if _, exists := s.savedLayers[legacyImg.ID]; exists {
+		return nil
+	}
+
+	outDir := filepath.Join(s.outDir, legacyImg.ID)
+	if err := os.Mkdir(outDir, 0755); err != nil {
+		return err
+	}
+
+	// todo: why is this version file here?
+	if err := ioutil.WriteFile(filepath.Join(outDir, legacyVersionFileName), []byte("1.0"), 0644); err != nil {
+		return err
+	}
+
+	imageConfig, err := json.Marshal(legacyImg)
+	if err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(outDir, legacyConfigFileName), imageConfig, 0644); err != nil {
+		return err
+	}
+
+	// serialize filesystem
+	tarFile, err := os.Create(filepath.Join(outDir, legacyLayerFileName))
+	if err != nil {
+		return err
+	}
+	defer tarFile.Close()
+
+	l, err := s.ls.Get(id)
+	if err != nil {
+		return err
+	}
+	defer layer.ReleaseAndLog(s.ls, l)
+
+	arch, err := l.TarStream()
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(tarFile, arch); err != nil {
+		return err
+	}
+
+	for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} {
+		// todo: maybe save layer created timestamp?
+		if err := os.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil {
+			return err
+		}
+	}
+
+	s.savedLayers[legacyImg.ID] = struct{}{}
+	return nil
+}

image/fs_test.go

@@ -0,0 +1,36 @@
+package tarexport
+
+import (
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/tag"
+)
+
+const (
+	manifestFileName           = "manifest.json"
+	legacyLayerFileName        = "layer.tar"
+	legacyConfigFileName       = "json"
+	legacyVersionFileName      = "VERSION"
+	legacyRepositoriesFileName = "repositories"
+)
+
+type manifestItem struct {
+	Config   string
+	RepoTags []string
+	Layers   []string
+}
+
+type tarexporter struct {
+	is image.Store
+	ls layer.Store
+	ts tag.Store
+}
+
+// NewTarExporter returns new ImageExporter for tar packages
+func NewTarExporter(is image.Store, ls layer.Store, ts tag.Store) image.Exporter {
+	return &tarexporter{
+		is: is,
+		ls: ls,
+		ts: ts,
+	}
+}

image/image.go

@@ -0,0 +1,148 @@
+package v1
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/version"
+)
+
+var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
+
+// noFallbackMinVersion is the minimum version for which v1compatibility
+// information will not be marshaled through the Image struct to remove
+// blank fields.
+var noFallbackMinVersion = version.Version("1.8.3")
+
+// HistoryFromConfig creates a History struct from v1 configuration JSON
+func HistoryFromConfig(imageJSON []byte, emptyLayer bool) (image.History, error) {
+	h := image.History{}
+	var v1Image image.V1Image
+	if err := json.Unmarshal(imageJSON, &v1Image); err != nil {
+		return h, err
+	}
+
+	return image.History{
+		Author:     v1Image.Author,
+		Created:    v1Image.Created,
+		CreatedBy:  strings.Join(v1Image.ContainerConfig.Cmd.Slice(), " "),
+		Comment:    v1Image.Comment,
+		EmptyLayer: emptyLayer,
+	}, nil
+}
+
+// CreateID creates an ID from v1 image, layerID and parent ID.
+// Used for backwards compatibility with old clients.
+func CreateID(v1Image image.V1Image, layerID layer.ChainID, parent digest.Digest) (digest.Digest, error) {
+	v1Image.ID = ""
+	v1JSON, err := json.Marshal(v1Image)
+	if err != nil {
+		return "", err
+	}
+
+	var config map[string]*json.RawMessage
+	if err := json.Unmarshal(v1JSON, &config); err != nil {
+		return "", err
+	}
+
+	// FIXME: note that this is slightly incompatible with RootFS logic
+	config["layer_id"] = rawJSON(layerID)
+	if parent != "" {
+		config["parent"] = rawJSON(parent)
+	}
+
+	configJSON, err := json.Marshal(config)
+	if err != nil {
+		return "", err
+	}
+	logrus.Debugf("CreateV1ID %s", configJSON)
+
+	return digest.FromBytes(configJSON)
+}
+
+// MakeConfigFromV1Config creates an image config from the legacy V1 config format.
+func MakeConfigFromV1Config(imageJSON []byte, rootfs *image.RootFS, history []image.History) ([]byte, error) {
+	var dver struct {
+		DockerVersion string `json:"docker_version"`
+	}
+
+	if err := json.Unmarshal(imageJSON, &dver); err != nil {
+		return nil, err
+	}
+
+	useFallback := version.Version(dver.DockerVersion).LessThan(noFallbackMinVersion)
+
+	if useFallback {
+		var v1Image image.V1Image
+		err := json.Unmarshal(imageJSON, &v1Image)
+		if err != nil {
+			return nil, err
+		}
+		imageJSON, err = json.Marshal(v1Image)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var c map[string]*json.RawMessage
+	if err := json.Unmarshal(imageJSON, &c); err != nil {
+		return nil, err
+	}
+
+	delete(c, "id")
+	delete(c, "parent")
+	delete(c, "Size") // Size is calculated from data on disk and is inconsitent
+	delete(c, "parent_id")
+	delete(c, "layer_id")
+	delete(c, "throwaway")
+
+	c["rootfs"] = rawJSON(rootfs)
+	c["history"] = rawJSON(history)
+
+	return json.Marshal(c)
+}
+
+// MakeV1ConfigFromConfig creates an legacy V1 image config from an Image struct
+func MakeV1ConfigFromConfig(img *image.Image, v1ID, parentV1ID string, throwaway bool) ([]byte, error) {
+	// Top-level v1compatibility string should be a modified version of the
+	// image config.
+	var configAsMap map[string]*json.RawMessage
+	if err := json.Unmarshal(img.RawJSON(), &configAsMap); err != nil {
+		return nil, err
+	}
+
+	// Delete fields that didn't exist in old manifest
+	delete(configAsMap, "rootfs")
+	delete(configAsMap, "history")
+	configAsMap["id"] = rawJSON(v1ID)
+	if parentV1ID != "" {
+		configAsMap["parent"] = rawJSON(parentV1ID)
+	}
+	if throwaway {
+		configAsMap["throwaway"] = rawJSON(true)
+	}
+
+	return json.Marshal(configAsMap)
+}
+
+func rawJSON(value interface{}) *json.RawMessage {
+	jsonval, err := json.Marshal(value)
+	if err != nil {
+		return nil
+	}
+	return (*json.RawMessage)(&jsonval)
+}
+
+// ValidateID checks whether an ID string is a valid image ID.
+func ValidateID(id string) error {
+	if ok := validHex.MatchString(id); !ok {
+		return fmt.Errorf("image ID '%s' is invalid ", id)
+	}
+	return nil
+}