Merge pull request #1548 from openziti/filter-circuits-links

Support filter/sort/page on circuits and links. Fixes #1547

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 * SDK Hosting Improvements
 * Terminator validation utility
+* Circuit/Link query support
 
 ## SDK Hosting Improvments
 
@@ -21,20 +22,29 @@ setup logic is correct. However it may also be used to diagnose and resolve issu
 ziti fabric validate terminators
 ```
 
+## Circuit/Link Query Support
+
+Previously listing circuit and links always showed the full list. This is because these types are in memory only and are not stored
+in the bbolt datastore. There's now basic support for querying in-memory types and circuits and links can now be filtered/paged/sorted
+ the same as other entity types.
+
 ## Component Updates and Bug Fixes
 
 * github.com/openziti/channel/v2: [v2.0.105 -> v2.0.110](https://github.com/openziti/channel/compare/v2.0.105...v2.0.110)
 * github.com/openziti/edge-api: [v0.26.0 -> v0.26.1](https://github.com/openziti/edge-api/compare/v0.26.0...v0.26.1)
 * github.com/openziti/foundation/v2: [v2.0.33 -> v2.0.35](https://github.com/openziti/foundation/compare/v2.0.33...v2.0.35)
 * github.com/openziti/identity: [v1.0.66 -> v1.0.68](https://github.com/openziti/identity/compare/v1.0.66...v1.0.68)
 * github.com/openziti/metrics: [v1.2.37 -> v1.2.40](https://github.com/openziti/metrics/compare/v1.2.37...v1.2.40)
-* github.com/openziti/sdk-golang: [v0.20.129 -> v0.20.139](https://github.com/openziti/sdk-golang/compare/v0.20.129...v0.20.139)
-    * [Issue #457](https://github.com/openziti/sdk-golang/issues/457) - Add  inspect support
-    * [Issue #450](https://github.com/openziti/sdk-golang/issues/450) - Support idempotent terminator creation
-
+* github.com/openziti/runzmd: [v1.0.33 -> v1.0.36](https://github.com/openziti/runzmd/compare/v1.0.33...v1.0.36)
+* github.com/openziti/sdk-golang: [v0.20.129 -> v0.21.1](https://github.com/openziti/sdk-golang/compare/v0.20.129...v0.21.1)
 * github.com/openziti/secretstream: [v0.1.13 -> v0.1.14](https://github.com/openziti/secretstream/compare/v0.1.13...v0.1.14)
+* github.com/openziti/storage: [v0.2.23 -> v0.2.26](https://github.com/openziti/storage/compare/v0.2.23...v0.2.26)
+    * [Issue #57](https://github.com/openziti/storage/issues/57) - Support querying collections of in memory objects
+
 * github.com/openziti/transport/v2: [v2.0.113 -> v2.0.119](https://github.com/openziti/transport/compare/v2.0.113...v2.0.119)
+* github.com/openziti/dilithium: [v0.3.3 -> v0.3.5](https://github.com/openziti/dilithium/compare/v0.3.3...v0.3.5)
 * github.com/openziti/ziti: [v0.31.0 -> v0.31.1](https://github.com/openziti/ziti/compare/v0.31.0...v0.31.1)
+    * [Issue #1547](https://github.com/openziti/ziti/issues/1547) - Support filtering, sorting and paging circuits and links
     * [Issue #1446](https://github.com/openziti/ziti/issues/1446) - Allow for idempotent sdk based terminators 
     * [Issue #1523](https://github.com/openziti/ziti/issues/1523) - Bootstrap members not working
     * [Issue #1525](https://github.com/openziti/ziti/issues/1525) - Improve cluster list output

controller/api_impl/circuit_api_model.go

@@ -17,7 +17,6 @@
 package api_impl
 
 import (
-	"github.com/go-openapi/strfmt"
 	"github.com/openziti/ziti/controller/api"
 	"github.com/openziti/ziti/controller/network"
 
@@ -46,22 +45,20 @@ func (factory *CircuitLinkFactoryIml) Links(entity LinkEntity) rest_model.Links
 }
 
 func MapCircuitToRestModel(_ *network.Network, _ api.RequestContext, circuit *network.Circuit) (*rest_model.CircuitDetail, error) {
-	path := &rest_model.CircuitDetailPath{}
+	path := &rest_model.Path{}
 	for _, node := range circuit.Path.Nodes {
 		path.Nodes = append(path.Nodes, ToEntityRef(node.Name, node, RouterLinkFactory))
 	}
 	for _, link := range circuit.Path.Links {
 		path.Links = append(path.Links, ToEntityRef(link.Id, link, LinkLinkFactory))
 	}
 
-	createdAt := strfmt.DateTime(circuit.CreatedAt)
 	ret := &rest_model.CircuitDetail{
-		ID:         &circuit.Id,
+		BaseEntity: BaseEntityToRestModel(circuit, CircuitLinkFactory),
 		ClientID:   circuit.ClientId,
 		Path:       path,
 		Service:    ToEntityRef(circuit.Service.Name, circuit.Service, ServiceLinkFactory),
 		Terminator: ToEntityRef(circuit.Terminator.GetId(), circuit.Terminator, TerminatorLinkFactory),
-		CreatedAt:  &createdAt,
 	}
 
 	return ret, nil

controller/api_impl/circuit_router.go

@@ -18,14 +18,13 @@ package api_impl
 
 import (
 	"github.com/go-openapi/runtime/middleware"
+	"github.com/openziti/storage/boltz"
 	"github.com/openziti/ziti/controller/api"
 	"github.com/openziti/ziti/controller/change"
 	"github.com/openziti/ziti/controller/network"
 	"github.com/openziti/ziti/controller/rest_model"
 	"github.com/openziti/ziti/controller/rest_server/operations"
 	"github.com/openziti/ziti/controller/rest_server/operations/circuit"
-	"github.com/openziti/storage/boltz"
-	"sort"
 )
 
 func init() {
@@ -59,10 +58,16 @@ func (r *CircuitRouter) Register(fabricApi *operations.ZitiFabricAPI, wrapper Re
 
 func (r *CircuitRouter) ListCircuits(n *network.Network, rc api.RequestContext) {
 	ListWithEnvelopeFactory(rc, defaultToListEnvelope, func(rc api.RequestContext, queryOptions *PublicQueryOptions) (*QueryResult, error) {
-		circuits := n.GetAllCircuits()
-		sort.Slice(circuits, func(i, j int) bool {
-			return circuits[i].Id < circuits[j].Id
-		})
+		query, err := queryOptions.getFullQuery(n.GetCircuitStore())
+		if err != nil {
+			return nil, err
+		}
+
+		circuits, count, err := n.GetCircuitStore().QueryEntitiesC(query)
+		if err != nil {
+			return nil, err
+		}
+
 		apiCircuits := make([]*rest_model.CircuitDetail, 0, len(circuits))
 		for _, modelCircuit := range circuits {
 			apiCircuit, err := MapCircuitToRestModel(n, rc, modelCircuit)
@@ -73,9 +78,9 @@ func (r *CircuitRouter) ListCircuits(n *network.Network, rc api.RequestContext)
 		}
 		result := &QueryResult{
 			Result:           apiCircuits,
-			Count:            int64(len(circuits)),
-			Limit:            -1,
-			Offset:           0,
+			Count:            count,
+			Limit:            *query.GetLimit(),
+			Offset:           *query.GetSkip(),
 			FilterableFields: nil,
 		}
 		return result, nil

controller/api_impl/link_router.go

@@ -18,15 +18,14 @@ package api_impl
 
 import (
 	"github.com/go-openapi/runtime/middleware"
+	"github.com/openziti/storage/boltz"
 	"github.com/openziti/ziti/controller/api"
 	"github.com/openziti/ziti/controller/change"
 	"github.com/openziti/ziti/controller/fields"
 	"github.com/openziti/ziti/controller/network"
 	"github.com/openziti/ziti/controller/rest_model"
 	"github.com/openziti/ziti/controller/rest_server/operations"
 	"github.com/openziti/ziti/controller/rest_server/operations/link"
-	"github.com/openziti/storage/boltz"
-	"sort"
 )
 
 func init() {
@@ -64,10 +63,16 @@ func (r *LinkRouter) Register(fabricApi *operations.ZitiFabricAPI, wrapper Reque
 
 func (r *LinkRouter) ListLinks(n *network.Network, rc api.RequestContext) {
 	ListWithEnvelopeFactory(rc, defaultToListEnvelope, func(rc api.RequestContext, queryOptions *PublicQueryOptions) (*QueryResult, error) {
-		links := n.GetAllLinks()
-		sort.Slice(links, func(i, j int) bool {
-			return links[i].Id < links[j].Id
-		})
+		query, err := queryOptions.getFullQuery(n.GetLinkStore())
+		if err != nil {
+			return nil, err
+		}
+
+		links, count, err := n.GetLinkStore().QueryEntitiesC(query)
+		if err != nil {
+			return nil, err
+		}
+
 		apiLinks := make([]*rest_model.LinkDetail, 0, len(links))
 		for _, modelLink := range links {
 			apiLink, err := MapLinkToRestModel(n, rc, modelLink)
@@ -78,9 +83,9 @@ func (r *LinkRouter) ListLinks(n *network.Network, rc api.RequestContext) {
 		}
 		result := &QueryResult{
 			Result:           apiLinks,
-			Count:            int64(len(links)),
-			Limit:            -1,
-			Offset:           0,
+			Count:            count,
+			Limit:            *query.GetLimit(),
+			Offset:           *query.GetSkip(),
 			FilterableFields: nil,
 		}
 		return result, nil

controller/api_impl/query_options.go

@@ -43,18 +43,21 @@ func (qo *PublicQueryOptions) String() string {
 	return fmt.Sprintf("[QueryOption Predicate: '%v', Sort: '%v', Paging: '%v']", qo.Predicate, qo.Sort, qo.Paging)
 }
 
-func (qo *PublicQueryOptions) getFullQuery(store boltz.Store) (ast.Query, error) {
+func (qo *PublicQueryOptions) getFullQuery(symbolTypes ast.SymbolTypes) (ast.Query, error) {
 	if qo.Predicate == "" {
 		qo.Predicate = "true"
 	}
 
-	query, err := ast.Parse(store, qo.Predicate)
+	query, err := ast.Parse(symbolTypes, qo.Predicate)
 	if err != nil {
 		return nil, errorz.NewInvalidFilter(err)
 	}
 
-	if err = boltz.ValidateSymbolsArePublic(query, store); err != nil {
-		return nil, errorz.NewInvalidFilter(err)
+	store, isStore := symbolTypes.(boltz.Store)
+	if isStore {
+		if err = boltz.ValidateSymbolsArePublic(query, store); err != nil {
+			return nil, errorz.NewInvalidFilter(err)
+		}
 	}
 
 	pfxlog.Logger().Debugf("query: %v", qo)
@@ -78,13 +81,15 @@ func (qo *PublicQueryOptions) getFullQuery(store boltz.Store) (ast.Query, error)
 	if len(sortFields) == 0 && qo.Sort != "" {
 		sortQueryString := "true sort by " + qo.Sort
 
-		sortQuery, err := ast.Parse(store, sortQueryString)
+		sortQuery, err := ast.Parse(symbolTypes, sortQueryString)
 		if err != nil {
 			return nil, errorz.NewInvalidSort(err)
 		}
 
-		if err = boltz.ValidateSymbolsArePublic(sortQuery, store); err != nil {
-			return nil, errorz.NewInvalidSort(err)
+		if isStore {
+			if err = boltz.ValidateSymbolsArePublic(sortQuery, store); err != nil {
+				return nil, errorz.NewInvalidSort(err)
+			}
 		}
 
 		if err = query.AdoptSortFields(sortQuery); err != nil {

controller/network/circuit.go

@@ -17,10 +17,11 @@
 package network
 
 import (
+	"github.com/openziti/identity"
+	"github.com/openziti/storage/objectz"
+	"github.com/openziti/ziti/common/logcontext"
 	"github.com/openziti/ziti/controller/idgen"
 	"github.com/openziti/ziti/controller/xt"
-	"github.com/openziti/ziti/common/logcontext"
-	"github.com/openziti/identity"
 	"github.com/orcaman/concurrent-map/v2"
 	"sync/atomic"
 	"time"
@@ -36,6 +37,35 @@ type Circuit struct {
 	Rerouting  atomic.Bool
 	PeerData   xt.PeerData
 	CreatedAt  time.Time
+	UpdatedAt  time.Time
+}
+
+func (self *Circuit) GetId() string {
+	return self.Id
+}
+
+func (self *Circuit) SetId(string) {
+	// id cannot be updated
+}
+
+func (self *Circuit) GetCreatedAt() time.Time {
+	return self.CreatedAt
+}
+
+func (self *Circuit) GetUpdatedAt() time.Time {
+	return self.UpdatedAt
+}
+
+func (self *Circuit) GetTags() map[string]interface{} {
+	result := map[string]interface{}{}
+	for k, v := range self.Tags {
+		result[k] = v
+	}
+	return result
+}
+
+func (self *Circuit) IsSystemEntity() bool {
+	return false
 }
 
 func (self *Circuit) cost(minRouterCost uint16) int64 {
@@ -57,13 +87,37 @@ func (self *Circuit) HasRouter(routerId string) bool {
 type circuitController struct {
 	circuits    cmap.ConcurrentMap[string, *Circuit]
 	idGenerator idgen.Generator
+	store       *objectz.ObjectStore[*Circuit]
 }
 
 func newCircuitController() *circuitController {
-	return &circuitController{
+	result := &circuitController{
 		circuits:    cmap.New[*Circuit](),
 		idGenerator: idgen.NewGenerator(),
 	}
+	result.store = objectz.NewObjectStore[*Circuit](func() objectz.ObjectIterator[*Circuit] {
+		return IterateCMap(result.circuits)
+	})
+	result.store.AddStringSymbol("id", func(entity *Circuit) *string {
+		return &entity.Id
+	})
+	result.store.AddStringSymbol("clientId", func(entity *Circuit) *string {
+		return &entity.ClientId
+	})
+	result.store.AddStringSymbol("service", func(entity *Circuit) *string {
+		return &entity.Service.Id
+	})
+	result.store.AddStringSymbol("terminator", func(entity *Circuit) *string {
+		val := entity.Terminator.GetId()
+		return &val
+	})
+	result.store.AddDatetimeSymbol("createdAt", func(entity *Circuit) *time.Time {
+		return &entity.CreatedAt
+	})
+	result.store.AddDatetimeSymbol("updatedAt", func(entity *Circuit) *time.Time {
+		return &entity.CreatedAt
+	})
+	return result
 }
 
 func (self *circuitController) nextCircuitId() (string, error) {

controller/network/cmap_iterator.go

@@ -0,0 +1,54 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package network
+
+import (
+	"github.com/openziti/storage/objectz"
+	cmap "github.com/orcaman/concurrent-map/v2"
+)
+
+func IterateCMap[T any](m cmap.ConcurrentMap[string, T]) objectz.ObjectIterator[T] {
+	iterator := &tupleChannelIterator[T]{
+		c:     m.IterBuffered(),
+		valid: true,
+	}
+	iterator.Next()
+	return iterator
+}
+
+type tupleChannelIterator[T any] struct {
+	c       <-chan cmap.Tuple[string, T]
+	current T
+	valid   bool
+}
+
+func (self *tupleChannelIterator[T]) IsValid() bool {
+	return self.valid
+}
+
+func (self *tupleChannelIterator[T]) Next() {
+	next, ok := <-self.c
+	if !ok {
+		self.valid = false
+	} else {
+		self.current = next.Val
+	}
+}
+
+func (self *tupleChannelIterator[T]) Current() T {
+	return self.current
+}