diff --git a/server/compose/dalutils/records.go b/server/compose/dalutils/records.go index 61e74537b7..8ad3599941 100644 --- a/server/compose/dalutils/records.go +++ b/server/compose/dalutils/records.go @@ -3,6 +3,7 @@ package dalutils import ( "context" "fmt" + "github.com/cortezaproject/corteza/server/compose/types" "github.com/cortezaproject/corteza/server/pkg/dal" "github.com/cortezaproject/corteza/server/pkg/filter" @@ -147,10 +148,10 @@ func drainIterator(ctx context.Context, iter dal.Iterator, mod *types.Module, f } var ( - ok bool - fetched uint - filtered uint - r *types.Record + ok bool + fetched uint + filtered uint + lastRecord *types.Record ) // Get the requested number of record @@ -161,11 +162,19 @@ func drainIterator(ctx context.Context, iter dal.Iterator, mod *types.Module, f } for f.Limit == 0 || uint(len(set)) < f.Limit { + var firstRecord *types.Record + // reset counters every drain fetched = 0 filtered = 0 + add := make(types.RecordSet, 0, 12) err = WalkIterator(ctx, iter, mod, func(r *types.Record) error { + lastRecord = r + if firstRecord == nil { + firstRecord = r + } + // check fetched record if f.Check != nil { if ok, err = f.Check(r); err != nil { @@ -177,7 +186,7 @@ func drainIterator(ctx context.Context, iter dal.Iterator, mod *types.Module, f } fetched++ - set = append(set, r) + add = append(add, r) return err }) @@ -187,8 +196,15 @@ func drainIterator(ctx context.Context, iter dal.Iterator, mod *types.Module, f return } + // If it's reverse, we need to add extra fetches to the start + if f.PageCursor != nil && f.PageCursor.ROrder { + set = append(add, set...) + } else { + set = append(set, add...) + } + total := fetched + filtered - if total == 0 || f.Limit == 0 || (0 < f.Limit && total < f.Limit) { + if total == 0 || f.Limit == 0 { // do not re-fetch if: // 1) nothing was fetch in the previous run // 2) there was no limit (everything was fetched) @@ -198,11 +214,18 @@ func drainIterator(ctx context.Context, iter dal.Iterator, mod *types.Module, f // Fetch more records setLen := uint(len(set)) - if setLen > 0 && setLen < f.Limit { + if total > 0 && setLen < f.Limit { fetchMore := f.Limit - setLen + var crsrRec *types.Record // request more items - if err = iter.More(fetchMore, r); err != nil { + if f.PageCursor == nil || !f.PageCursor.ROrder { + crsrRec = lastRecord + } else { + crsrRec = firstRecord + } + + if err = iter.More(fetchMore, crsrRec); err != nil { return } } @@ -295,6 +318,18 @@ func generatePageNavigation(ctx context.Context, iter dal.Iterator, mod *types.M return } + + recordChecker = func(i dal.Iterator) (ok bool, err error) { + rc := &types.Record{} + err = i.Scan(rc) + rc.SetModule(mod) + + if err != nil { + return + } + + return p.Check(rc) + } ) if setLen == 0 { @@ -314,13 +349,15 @@ func generatePageNavigation(ctx context.Context, iter dal.Iterator, mod *types.M // if limit is not defined and set is empty if p.Limit > 0 && len(set) > 0 { // PrevPage - out.PrevPage, err = dal.PreLoadCursor(ctx, iter, 1, true, first) - if err != nil { - return + if p.PageCursor != nil { + out.PrevPage, err = dal.PreLoadCursor(ctx, iter, 100, true, first, recordChecker) + if err != nil { + return + } } // NextPage - out.NextPage, err = dal.PreLoadCursor(ctx, iter, 1, false, last) + out.NextPage, err = dal.PreLoadCursor(ctx, iter, 100, false, last, recordChecker) if err != nil { return } diff --git a/server/pkg/dal/iterator.go b/server/pkg/dal/iterator.go index f1ef722981..00c3f95a38 100644 --- a/server/pkg/dal/iterator.go +++ b/server/pkg/dal/iterator.go @@ -72,13 +72,16 @@ func IteratorEncodeJSON(ctx context.Context, w io.Writer, iter Iterator, initTar // PreLoadCursor into iterator and check it exist then return the cursor // @todo this should be move to the Iterator -func PreLoadCursor(ctx context.Context, iter Iterator, limit uint, reverse bool, r ValueGetter) (out *filter.PagingCursor, err error) { - if reverse { - out, err = iter.BackCursor(r) - } else { - out, err = iter.ForwardCursor(r) +func PreLoadCursor(ctx context.Context, iter Iterator, limit uint, reverse bool, r ValueGetter, fx func(Iterator) (bool, error)) (out *filter.PagingCursor, err error) { + makeCursor := func() (*filter.PagingCursor, error) { + if reverse { + return iter.BackCursor(r) + } else { + return iter.ForwardCursor(r) + } } + out, err = makeCursor() if err != nil { return } @@ -88,11 +91,25 @@ func PreLoadCursor(ctx context.Context, iter Iterator, limit uint, reverse bool, return nil, nil } - if !iter.Next(ctx) { - out = nil - } + for { + if !iter.Next(ctx) { + out = nil + return + } - return + ok, err := fx(iter) + if err != nil { + return nil, err + } + + if ok { + return out, err + + // // @todo Skip the things we don't have access to; could cause some edge cases so probably not + // // It adds some performance since we skip unneeded stuff but could some records change in the mean time? + // // return makeCursor() + } + } } // IteratorSorting return iterator sorting diff --git a/server/store/adapters/rdbms/dal/iterator.go b/server/store/adapters/rdbms/dal/iterator.go index 5ed3a6f158..7dbe040a8a 100644 --- a/server/store/adapters/rdbms/dal/iterator.go +++ b/server/store/adapters/rdbms/dal/iterator.go @@ -69,7 +69,13 @@ func (i *iterator) More(max uint, last dal.ValueGetter) (err error) { i.limit = max if last != nil { - if i.cursor, err = i.collectCursorValues(last); err != nil { + if i.cursor == nil || !i.cursor.ROrder { + i.cursor, err = i.ForwardCursor(last) + } else { + i.cursor, err = i.BackCursor(last) + } + + if err != nil { return fmt.Errorf("could not collect cursor values: %w", err) } }