fix: remove duplicate items with the cursor paginator in date mode

We get duplicate items with the cursor paginator in date mode if the
timestamp of the last item has a non zero fractional part.

The solution is to increment the next cursor by one.

Author: yansal

paginator.go

@@ -146,9 +146,21 @@ func (p *CursorPaginator) MakeNextURI() null.String {
 		return null.NewString("", false)
 	}
 
-	// convert to timestamp if cusror mode is Date
+	// convert to timestamp
 	if p.Options.CursorOptions.Mode == DateModeCursor {
-		nextCursor = nextCursor.(time.Time).Unix()
+		timestamp := nextCursor.(time.Time).Unix()
+		if !p.Options.CursorOptions.Reverse {
+			// The next cursor must be the timestamp of the last item incremented by one.
+			// Otherwise, we would get duplicates as the last item of the current page would be included
+			// in the next page.
+			// One problem with this approach is that we may lose items if two items are within the same
+			// second. If items A and B are within the same second and A is the last item in the page,
+			// the next cursor will be the timestamp of A incremented by one, and the next page won't
+			// contain B.
+			// TODO: The (non-backward-compatible) solution is to increase the precision of timestamps
+			timestamp++
+		}
+		nextCursor = timestamp
 	}
 
 	return null.StringFrom(GenerateCursorURI(p.Limit, nextCursor, p.Options))

paginator_test.go

@@ -25,3 +25,67 @@ func TestNewCursorPaginator(t *testing.T) {
 	is.True(ok)
 	is.True(got.After(now), "%v should be after %v", got, now)
 }
+
+func TestCursorPaginator_Next_DateMode(t *testing.T) {
+	is := assert.New(t)
+
+	var u *User
+	is.NoError(db.DropTableIfExists(u).Error)
+	is.NoError(db.CreateTable(u).Error)
+	ts := time.Unix(0, 1548252003033986000) // non zero fractional part
+	is.NoError(db.Create(&User{DateCreation: ts}).Error)
+	is.NoError(db.Create(&User{DateCreation: ts.Add(time.Second)}).Error)
+
+	var users []User
+	s, err := NewGORMStore(db.Model(u).Order("date_creation"), &users)
+	is.NoError(err)
+
+	v := url.Values{"limit": []string{"1"}, "since": []string{"1"}}
+	opts := NewOptions()
+	opts.CursorOptions.Mode = DateModeCursor
+	opts.CursorOptions.DBName = "date_creation"
+	opts.CursorOptions.StructName = "DateCreation"
+	p, err := NewCursorPaginator(s, &http.Request{URL: &url.URL{RawQuery: v.Encode()}}, opts)
+	is.NoError(err)
+	is.NoError(p.Page())
+	is.Len(users, 1)
+	is.Equal(1, users[0].ID)
+
+	next := p.MakeNextURI()
+	is.True(next.Valid)
+	// the next uri cursor is the timestamp of the last element incremented by one
+	is.Contains(next.String, strconv.FormatInt(users[0].DateCreation.Unix()+1, 10))
+}
+
+func TestCursorPaginator_Next_DateMode_Reverse(t *testing.T) {
+	is := assert.New(t)
+
+	var u *User
+	is.NoError(db.DropTableIfExists(u).Error)
+	is.NoError(db.CreateTable(u).Error)
+	ts := time.Unix(0, 1548252003033986000) // non zero fractional part
+	is.NoError(db.Create(&User{DateCreation: ts}).Error)
+	is.NoError(db.Create(&User{DateCreation: ts.Add(time.Second)}).Error)
+
+	var users []User
+	s, err := NewGORMStore(db.Model(u).Order("date_creation desc"), &users)
+	is.NoError(err)
+
+	since := strconv.FormatInt(time.Now().Unix(), 10)
+	v := url.Values{"limit": []string{"1"}, "since": []string{since}}
+	opts := NewOptions()
+	opts.CursorOptions.Mode = DateModeCursor
+	opts.CursorOptions.DBName = "date_creation"
+	opts.CursorOptions.StructName = "DateCreation"
+	opts.CursorOptions.Reverse = true
+	p, err := NewCursorPaginator(s, &http.Request{URL: &url.URL{RawQuery: v.Encode()}}, opts)
+	is.NoError(err)
+	is.NoError(p.Page())
+	is.Len(users, 1)
+	is.Equal(2, users[0].ID)
+
+	next := p.MakeNextURI()
+	is.True(next.Valid)
+	// the next uri cursor is the timestamp of the last element
+	is.Contains(next.String, strconv.FormatInt(users[0].DateCreation.Unix(), 10))
+}