Introducing metadata storage interface for isolation and warm fuzzies

Author: meatballhat

metadata/database.go

@@ -110,10 +110,10 @@ func NewDatabase(url string, log *logrus.Logger) (*Database, error) {
 }
 
 // Save saves some metadata
-func (db *Database) Save(m *Metadata) (sql.Result, error) {
+func (db *Database) Save(m *Metadata) error {
 	statement := db.st.InsertMetadata.St
 	if statement == nil {
-		return nil, errNoStatement
+		return errNoStatement
 	}
 
 	res, err := statement.Exec(m.JobID, m.Size, m.Path, m.ContentType)
@@ -131,7 +131,7 @@ func (db *Database) Save(m *Metadata) (sql.Result, error) {
 		db.log.WithFields(logFields).Info("saving metadata")
 	}
 
-	return res, err
+	return err
 }
 
 // Lookup looks up some metadata

metadata/lookup_saver.go

@@ -0,0 +1,8 @@
+package metadata
+
+// LookupSaver is the interface needed for reads and writes of metadata
+type LookupSaver interface {
+	Save(*Metadata) error
+	Lookup(string, string) (*Metadata, error)
+	LookupAll(string) ([]*Metadata, error)
+}

metadata/null_lookup_saver.go

@@ -0,0 +1,57 @@
+package metadata
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+type NullLookupSaver struct {
+	mdMap map[string]*Metadata
+
+	l *sync.Mutex
+}
+
+func NewNullLookupSaver() *NullLookupSaver {
+	return &NullLookupSaver{
+		mdMap: map[string]*Metadata{},
+
+		l: &sync.Mutex{},
+	}
+}
+
+func (nls *NullLookupSaver) Save(m *Metadata) error {
+	nls.l.Lock()
+	defer nls.l.Unlock()
+	nls.mdMap[fmt.Sprintf("%s-%s", m.JobID, m.Path)] = m
+	return nil
+}
+
+func (nls *NullLookupSaver) Lookup(jobID, path string) (*Metadata, error) {
+	nls.l.Lock()
+	defer nls.l.Unlock()
+	m, ok := nls.mdMap[fmt.Sprintf("%s-%s", jobID, path)]
+	if ok {
+		return m, nil
+	}
+	return nil, errNoMetadata
+}
+
+func (nls *NullLookupSaver) LookupAll(jobID string) ([]*Metadata, error) {
+	nls.l.Lock()
+	defer nls.l.Unlock()
+	mds := []*Metadata{}
+
+	for key, m := range nls.mdMap {
+		keyParts := strings.SplitN(key, "-", 1)
+		if len(keyParts) < 2 {
+			return nil, errNoMetadata
+		}
+
+		if keyParts[0] == jobID {
+			mds = append(mds, m)
+		}
+	}
+
+	return mds, nil
+}

server/server.go

@@ -27,7 +27,7 @@ type Server struct {
 	log   *logrus.Logger
 	store store.Storer
 	auth  auth.Auther
-	md    *metadata.Database
+	md    metadata.LookupSaver
 }
 
 // Main is the top of the pile.  Start here.
@@ -41,7 +41,7 @@ func Main(log *logrus.Logger) {
 		opts.FileStorePrefix = "tmp"
 	}
 
-	server, err := NewServer(opts, log)
+	server, err := NewServer(opts, log, nil)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -60,7 +60,7 @@ func Main(log *logrus.Logger) {
 }
 
 // NewServer creates a new *Server with a router and its routes registered
-func NewServer(opts *Options, log *logrus.Logger) (*Server, error) {
+func NewServer(opts *Options, log *logrus.Logger, md metadata.LookupSaver) (*Server, error) {
 	var err error
 
 	log.Debug("creating new server")
@@ -72,7 +72,7 @@ func NewServer(opts *Options, log *logrus.Logger) (*Server, error) {
 	srv.setupRouter()
 	srv.setupNegroni()
 
-	err = srv.getDB()
+	err = srv.getMd(md)
 	if err != nil {
 		return nil, err
 	}
@@ -188,19 +188,25 @@ func (srv *Server) setupAuther() error {
 	panic("fell through to a bad place ¯\\_(ツ)_/¯")
 }
 
-func (srv *Server) getDB() error {
+func (srv *Server) getMd(md metadata.LookupSaver) error {
 	srv.log.Debug("getting database handle")
-	db, err := metadata.NewDatabase(srv.opts.DatabaseURL, srv.log)
-	if err != nil {
-		return err
-	}
 
-	err = db.Init()
-	if err != nil {
-		return err
+	if md == nil {
+		mddb, err := metadata.NewDatabase(srv.opts.DatabaseURL, srv.log)
+
+		if err != nil {
+			return err
+		}
+
+		err = mddb.Init()
+		if err != nil {
+			return err
+		}
+
+		md = mddb
 	}
 
-	srv.md = db
+	srv.md = md
 	return nil
 }
 

server/server_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/hamfist/artifacts-service/metadata"
 )
 
 var (
@@ -29,13 +30,12 @@ func TestServerDefaults(t *testing.T) {
 	opts := NewOptions()
 	log := getPanicLogger()
 
-	srv, err := NewServer(opts, log)
+	srv, err := NewServer(opts, log, metadata.NewNullLookupSaver())
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
 
 	if !reflect.DeepEqual(srv.opts, opts) {
 		t.Fatalf("opts %v != %v", srv.opts, opts)
 	}
-
 }

store/file_store.go

@@ -18,15 +18,15 @@ import (
 type FileStore struct {
 	Prefix string
 	log    *logrus.Logger
-	db     *metadata.Database
+	md     metadata.LookupSaver
 }
 
 // NewFileStore returns a *FileStore.  AMAZE.
-func NewFileStore(prefix string, log *logrus.Logger, db *metadata.Database) *FileStore {
+func NewFileStore(prefix string, log *logrus.Logger, md metadata.LookupSaver) *FileStore {
 	return &FileStore{
 		Prefix: prefix,
 		log:    log,
-		db:     db,
+		md:     md,
 	}
 }
 

store/s3_store.go

@@ -21,13 +21,13 @@ type S3Store struct {
 	secret string
 	bucket string
 	log    *logrus.Logger
-	db     *metadata.Database
+	md     metadata.LookupSaver
 	b      *s3.Bucket
 }
 
 // NewS3Store initializes an *S3Store.  Wow!
 func NewS3Store(key, secret, bucket, regionName string,
-	log *logrus.Logger, db *metadata.Database) (*S3Store, error) {
+	log *logrus.Logger, md metadata.LookupSaver) (*S3Store, error) {
 
 	log.Debug("getting aws auth")
 	auth, err := aws.GetAuth(key, secret)
@@ -62,7 +62,7 @@ func NewS3Store(key, secret, bucket, regionName string,
 		bucket: bucket,
 
 		log: log,
-		db:  db,
+		md:  md,
 		b:   b,
 	}, nil
 }
@@ -89,7 +89,7 @@ func (s3s *S3Store) Store(a *artifact.Artifact) error {
 	}
 
 	md := artifactToMetadata(a)
-	_, err = s3s.db.Save(md)
+	err = s3s.md.Save(md)
 	if err != nil {
 		return err
 	}

store/s3_store_test.go

@@ -1,11 +1,15 @@
 package store
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/hamfist/artifacts-service/metadata"
+)
 
 func TestNewS3StoreDefaults(t *testing.T) {
 	log := getPanicLogger()
-	db := getTestDB()
-	s3s, err := NewS3Store("key", "secret", "bucket", "us-west-2", log, db)
+	md := metadata.NewNullLookupSaver()
+	s3s, err := NewS3Store("key", "secret", "bucket", "us-west-2", log, md)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}

store/store_test.go

@@ -12,6 +12,10 @@ var (
 	dbURL = os.Getenv("DATABASE_URL")
 )
 
+type testSaver struct {
+	mds map[string]*metadata.Metadata
+}
+
 func init() {
 	os.Clearenv()
 	os.Setenv("DATABASE_URL", dbURL)
@@ -25,15 +29,6 @@ func getPanicLogger() *logrus.Logger {
 	return log
 }
 
-func getTestDB() *metadata.Database {
-	db, err := metadata.NewDatabase(dbURL, getPanicLogger())
-	if err != nil {
-		panic(err)
-	}
-
-	return db
-}
-
 func TestStuff(t *testing.T) {
 	if false {
 		t.Fail()