Merge pull request #1557 from openziti/consolidate-db-stores

Consolidate fabric/edge persistence code #1555

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
 release/
-db/
+/db/
 # Binaries for programs and plugins
 *.exe
 *.exe~

CHANGELOG.md

@@ -30,22 +30,28 @@ in the bbolt datastore. There's now basic support for querying in-memory types a
 
 ## Component Updates and Bug Fixes
 
-* github.com/openziti/channel/v2: [v2.0.105 -> v2.0.110](https://github.com/openziti/channel/compare/v2.0.105...v2.0.110)
+* github.com/openziti/channel/v2: [v2.0.105 -> v2.0.111](https://github.com/openziti/channel/compare/v2.0.105...v2.0.111)
+    * [Issue #118](https://github.com/openziti/channel/issues/118) - Allowing checking if reconnecting impl is currently connected
+
 * github.com/openziti/edge-api: [v0.26.0 -> v0.26.1](https://github.com/openziti/edge-api/compare/v0.26.0...v0.26.1)
 * github.com/openziti/foundation/v2: [v2.0.33 -> v2.0.35](https://github.com/openziti/foundation/compare/v2.0.33...v2.0.35)
 * github.com/openziti/identity: [v1.0.66 -> v1.0.68](https://github.com/openziti/identity/compare/v1.0.66...v1.0.68)
 * github.com/openziti/metrics: [v1.2.37 -> v1.2.40](https://github.com/openziti/metrics/compare/v1.2.37...v1.2.40)
 * github.com/openziti/runzmd: [v1.0.33 -> v1.0.36](https://github.com/openziti/runzmd/compare/v1.0.33...v1.0.36)
-* github.com/openziti/sdk-golang: [v0.20.129 -> v0.21.1](https://github.com/openziti/sdk-golang/compare/v0.20.129...v0.21.1)
+* github.com/openziti/sdk-golang: [v0.20.129 -> v0.20.139](https://github.com/openziti/sdk-golang/compare/v0.20.129...v0.20.139)
+    * [Issue #457](https://github.com/openziti/sdk-golang/issues/457) - Add  inspect support
+    * [Issue #450](https://github.com/openziti/sdk-golang/issues/450) - Support idempotent terminator creation
+
 * github.com/openziti/secretstream: [v0.1.13 -> v0.1.14](https://github.com/openziti/secretstream/compare/v0.1.13...v0.1.14)
 * github.com/openziti/storage: [v0.2.23 -> v0.2.26](https://github.com/openziti/storage/compare/v0.2.23...v0.2.26)
     * [Issue #57](https://github.com/openziti/storage/issues/57) - Support querying collections of in memory objects
 
 * github.com/openziti/transport/v2: [v2.0.113 -> v2.0.119](https://github.com/openziti/transport/compare/v2.0.113...v2.0.119)
-* github.com/openziti/dilithium: [v0.3.3 -> v0.3.5](https://github.com/openziti/dilithium/compare/v0.3.3...v0.3.5)
 * github.com/openziti/ziti: [v0.31.0 -> v0.31.1](https://github.com/openziti/ziti/compare/v0.31.0...v0.31.1)
+    * [Issue #1555](https://github.com/openziti/ziti/issues/1555) - Consolidate fabric/edge persistence code
     * [Issue #1547](https://github.com/openziti/ziti/issues/1547) - Support filtering, sorting and paging circuits and links
     * [Issue #1446](https://github.com/openziti/ziti/issues/1446) - Allow for idempotent sdk based terminators 
+    * [Issue #1540](https://github.com/openziti/ziti/issues/1540) - Transit router create fails in HA environment
     * [Issue #1523](https://github.com/openziti/ziti/issues/1523) - Bootstrap members not working
     * [Issue #1525](https://github.com/openziti/ziti/issues/1525) - Improve cluster list output
     * [Issue #1519](https://github.com/openziti/ziti/issues/1519) - Simplify link ack handling

controller/persistence/api_session_certificate_store.go → controller/db/api_session_certificate_store.go

@@ -14,7 +14,7 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"github.com/openziti/storage/ast"

controller/persistence/api_session_store.go → controller/db/api_session_store.go

@@ -14,16 +14,15 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"github.com/kataras/go-events"
 	"github.com/michaelquigley/pfxlog"
-	"github.com/openziti/ziti/common/eid"
-	"github.com/openziti/ziti/controller/change"
-	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/storage/ast"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common/eid"
+	"github.com/openziti/ziti/controller/change"
 	"go.etcd.io/bbolt"
 	"strings"
 	"time"
@@ -132,10 +131,10 @@ func (store *apiSessionStoreImpl) GetEventsEmitter() events.EventEmmiter {
 	return store.eventsEmitter
 }
 
-func (store *apiSessionStoreImpl) onEventualDelete(name string, apiSessionId []byte) {
+func (store *apiSessionStoreImpl) onEventualDelete(db boltz.Db, name string, apiSessionId []byte) {
 	idCollector := &sessionIdCollector{}
-	indexPath := []string{db.RootBucket, boltz.IndexesBucket, EntityTypeApiSessions, EntityTypeSessions}
-	err := store.stores.DbProvider.GetDb().View(func(tx *bbolt.Tx) error {
+	indexPath := []string{RootBucket, boltz.IndexesBucket, EntityTypeApiSessions, EntityTypeSessions}
+	err := db.View(func(tx *bbolt.Tx) error {
 		path := append(indexPath, string(apiSessionId))
 		if bucket := boltz.Path(tx, path...); bucket != nil {
 			boltz.Traverse(bucket.Bucket, "/"+strings.Join(path, "/"), idCollector)
@@ -152,7 +151,7 @@ func (store *apiSessionStoreImpl) onEventualDelete(name string, apiSessionId []b
 
 	for _, id := range idCollector.ids {
 		changeContext := change.New().SetSourceType("events.emitter").SetChangeAuthorType(change.AuthorTypeController)
-		err = store.stores.DbProvider.GetDb().Update(changeContext.NewMutateContext(), func(ctx boltz.MutateContext) error {
+		err = db.Update(changeContext.NewMutateContext(), func(ctx boltz.MutateContext) error {
 			if err := store.stores.session.DeleteById(ctx, id); err != nil {
 				if boltz.IsErrNotFoundErr(err) {
 					return nil
@@ -172,7 +171,7 @@ func (store *apiSessionStoreImpl) onEventualDelete(name string, apiSessionId []b
 	}
 
 	changeContext := change.New().SetSourceType("events.emitter").SetChangeAuthorType(change.AuthorTypeController)
-	err = store.stores.DbProvider.GetDb().Update(changeContext.NewMutateContext(), func(ctx boltz.MutateContext) error {
+	err = db.Update(changeContext.NewMutateContext(), func(ctx boltz.MutateContext) error {
 		if bucket := boltz.Path(ctx.Tx(), indexPath...); bucket != nil {
 			if err := bucket.DeleteBucket(apiSessionId); err != nil {
 				if err != bbolt.ErrBucketNotFound {
@@ -264,7 +263,7 @@ func (store *apiSessionStoreImpl) LoadOneByToken(tx *bbolt.Tx, token string) (*A
 
 func (store *apiSessionStoreImpl) GetCachedSessionId(tx *bbolt.Tx, apiSessionId, sessionType, serviceId string) *string {
 	bucket := boltz.Path(tx,
-		db.RootBucket, boltz.IndexesBucket,
+		RootBucket, boltz.IndexesBucket,
 		EntityTypeApiSessions, EntityTypeSessions,
 		apiSessionId, sessionType,
 	)

controller/persistence/api_session_store_test.go → controller/db/api_session_store_test.go

@@ -14,16 +14,16 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"fmt"
 	"github.com/google/go-cmp/cmp"
-	"github.com/openziti/ziti/common/eid"
-	"github.com/openziti/ziti/controller/change"
 	"github.com/openziti/foundation/v2/stringz"
 	"github.com/openziti/storage/boltz"
 	"github.com/openziti/storage/boltztest"
+	"github.com/openziti/ziti/common/eid"
+	"github.com/openziti/ziti/controller/change"
 	"go.etcd.io/bbolt"
 	"testing"
 	"time"

controller/persistence/auth_policy_store.go → controller/db/auth_policy_store.go

@@ -14,7 +14,7 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"github.com/openziti/storage/ast"

controller/persistence/authenticator_store.go → controller/db/authenticator_store.go

@@ -14,7 +14,7 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"fmt"

controller/persistence/base_entity.go → controller/db/base_entity.go

@@ -14,7 +14,7 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"github.com/openziti/storage/boltz"

controller/db/base_store.go

@@ -17,14 +17,129 @@
 package db
 
 import (
+	"github.com/openziti/foundation/v2/errorz"
+	"github.com/openziti/storage/ast"
 	"github.com/openziti/storage/boltz"
+	"go.etcd.io/bbolt"
+	"strings"
 )
 
-const (
-	FieldName = "name"
-)
+type initializableStore interface {
+	boltz.Store
+	initializeLocal()
+	initializeLinked()
+	initializeIndexes(tx *bbolt.Tx, errorHolder errorz.ErrorHolder)
+}
+
+type Store[E boltz.ExtEntity] interface {
+	boltz.EntityStore[E]
+	initializableStore
+	LoadOneById(tx *bbolt.Tx, id string) (E, error)
+}
 
-type baseStore[T boltz.ExtEntity] struct {
+type baseStore[E boltz.ExtEntity] struct {
 	stores *stores
-	*boltz.BaseStore[T]
+	*boltz.BaseStore[E]
+}
+
+func (store *baseStore[E]) addUniqueNameField() boltz.ReadIndex {
+	symbolName := store.AddSymbol(FieldName, ast.NodeTypeString)
+	return store.AddUniqueIndex(symbolName)
+}
+
+func (store *baseStore[E]) initializeIndexes(tx *bbolt.Tx, errorHolder errorz.ErrorHolder) {
+	store.InitializeIndexes(tx, errorHolder)
+}
+
+func (store *baseStore[E]) LoadOneById(tx *bbolt.Tx, id string) (E, error) {
+	entity := store.NewStoreEntity()
+	if err := store.baseLoadOneById(tx, id, entity); err != nil {
+		return *new(E), err
+	}
+	return entity, nil
+}
+
+func (store *baseStore[E]) baseLoadOneById(tx *bbolt.Tx, id string, entity E) error {
+	found, err := store.LoadEntity(tx, id, entity)
+	if err != nil {
+		return err
+	}
+	if !found {
+		return boltz.NewNotFoundError(store.GetSingularEntityType(), "id", id)
+	}
+	return nil
+}
+
+func (store *baseStore[E]) deleteEntityReferences(tx *bbolt.Tx, entity boltz.NamedExtEntity, rolesSymbol boltz.EntitySetSymbol) error {
+	idRef := entityRef(entity.GetId())
+
+	for _, policyHolderId := range store.GetRelatedEntitiesIdList(tx, entity.GetId(), rolesSymbol.GetStore().GetEntityType()) {
+		err := rolesSymbol.Map(tx, []byte(policyHolderId), func(ctx *boltz.MapContext) {
+			if ctx.ValueS() == idRef {
+				ctx.Delete()
+			}
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (store *baseStore[E]) getParentBucket(entity boltz.Entity, childBucket *boltz.TypedBucket) *boltz.TypedBucket {
+	parentBucket := store.GetParentStore().GetEntityBucket(childBucket.Tx(), []byte(entity.GetId()))
+	parentBucket.ErrorHolderImpl = childBucket.ErrorHolderImpl
+	return parentBucket
+}
+
+type NameIndexed interface {
+	GetNameIndex() boltz.ReadIndex
+}
+
+func (store *baseStore[E]) GetName(tx *bbolt.Tx, id string) *string {
+	symbol := store.GetSymbol(FieldName)
+	if symbol == nil {
+		return nil
+	}
+	_, val := symbol.Eval(tx, []byte(id))
+	if val != nil {
+		result := string(val)
+		return &result
+	}
+	return nil
+}
+
+func (store *baseStore[E]) getRoleAttributesCursorProvider(index boltz.SetReadIndex, values []string, semantic string) (ast.SetCursorProvider, error) {
+	if semantic == "" {
+		semantic = SemanticAllOf
+	}
+
+	if !isSemanticValid(semantic) {
+		return nil, errorz.NewFieldError("invalid semantic", FieldSemantic, semantic)
+	}
+
+	roles, ids, err := splitRolesAndIds(values)
+	if err != nil {
+		return nil, err
+	}
+
+	return func(tx *bbolt.Tx, forward bool) ast.SetCursor {
+		validIds := ast.NewTreeSet(forward)
+		for _, id := range ids {
+			if store.IsEntityPresent(tx, id) {
+				validIds.Add([]byte(id))
+			}
+		}
+
+		var rolesCursor ast.SetCursor
+		if strings.EqualFold(semantic, SemanticAllOf) {
+			rolesCursor = store.IteratorMatchingAllOf(index, roles)(tx, forward)
+		} else {
+			rolesCursor = store.IteratorMatchingAnyOf(index, roles)(tx, forward)
+		}
+		if validIds.Size() == 0 {
+			return rolesCursor
+		}
+		return ast.NewUnionSetCursor(rolesCursor, validIds.ToCursor(), forward)
+	}, nil
 }

controller/persistence/ca_store.go → controller/db/ca_store.go

@@ -14,7 +14,7 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"github.com/openziti/storage/ast"

controller/persistence/config_store.go → controller/db/config_store.go

@@ -14,14 +14,13 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
-	"github.com/openziti/ziti/common/eid"
-	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/storage/ast"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common/eid"
 	"go.etcd.io/bbolt"
 )
 
@@ -88,7 +87,7 @@ func (store *configStoreImpl) initializeLocal() {
 	store.indexName = store.addUniqueNameField()
 	store.symbolType = store.AddFkSymbol(FieldConfigType, store.stores.configType)
 	store.AddMapSymbol(FieldConfigData, ast.NodeTypeAnyType, FieldConfigData)
-	store.symbolServices = store.AddFkSetSymbol(db.EntityTypeServices, store.stores.edgeService)
+	store.symbolServices = store.AddFkSetSymbol(EntityTypeServices, store.stores.edgeService)
 	store.symbolIdentityServices = store.AddSetSymbol(FieldConfigIdentityService, ast.NodeTypeOther)
 	store.identityServicesLinks = &boltz.LinkedSetSymbol{EntitySymbol: store.symbolIdentityServices}
 }

controller/persistence/config_store_test.go → controller/db/config_store_test.go

@@ -14,13 +14,13 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/openziti/ziti/common/eid"
 	"github.com/openziti/storage/boltztest"
+	"github.com/openziti/ziti/common/eid"
 	"go.etcd.io/bbolt"
 	"testing"
 	"time"

controller/persistence/config_type_store.go → controller/db/config_type_store.go

@@ -14,13 +14,13 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"encoding/json"
-	"github.com/openziti/ziti/common/eid"
 	"github.com/openziti/storage/ast"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common/eid"
 	"github.com/pkg/errors"
 	"go.etcd.io/bbolt"
 )

controller/persistence/config_type_store_test.go → controller/db/config_type_store_test.go

@@ -14,12 +14,12 @@
 	limitations under the License.
 */
 
-package persistence
+package db
 
 import (
 	"fmt"
-	"github.com/openziti/ziti/common/eid"
 	"github.com/openziti/storage/boltztest"
+	"github.com/openziti/ziti/common/eid"
 	"go.etcd.io/bbolt"
 	"testing"
 	"time"

controller/persistence/edge_router_policy_store.go → controller/db/edge_router_policy_store.go

@@ -1,13 +1,12 @@
-package persistence
+package db
 
 import (
 	"fmt"
-	"github.com/openziti/ziti/common/eid"
-	"github.com/openziti/ziti/controller/db"
 	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/foundation/v2/stringz"
 	"github.com/openziti/storage/ast"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/common/eid"
 	"sort"
 )
 
@@ -79,7 +78,7 @@ func (store *edgeRouterPolicyStoreImpl) initializeLocal() {
 	store.symbolIdentityRoles = store.AddPublicSetSymbol(FieldIdentityRoles, ast.NodeTypeString)
 	store.symbolEdgeRouterRoles = store.AddPublicSetSymbol(FieldEdgeRouterRoles, ast.NodeTypeString)
 	store.symbolIdentities = store.AddFkSetSymbol(EntityTypeIdentities, store.stores.identity)
-	store.symbolEdgeRouters = store.AddFkSetSymbol(db.EntityTypeRouters, store.stores.edgeRouter)
+	store.symbolEdgeRouters = store.AddFkSetSymbol(EntityTypeRouters, store.stores.edgeRouter)
 
 	store.AddConstraint(boltz.NewSystemEntityEnforcementConstraint(store))
 }