Cursor based pagination for scim resources.

Cursor pagination changes - phase 1

formatting adjustments before PR

more formatting adjustments and removing unneccesary comments

Formatting the code

Adjustments made to Cursor Pagination for SCIM resources

Changes made as per the code review

Changing the response type from a List to UsersGetResponse type object

Changing the response type from a List to UsersGetResponse type object - changes based on the code review

formatting changes

Made changes to support POST/.Search cursor pagination

Made changes so that the ServiceProviderConfig endpoint shows cursor pagination capability

minor changes

Fix for NullPointerException when running UserResourceManagerTest

Updated wording for serviceProviderConfigEndpoint changes

Formatting changes

Removing unused parameters from processPagination method

Updating tests broken due to changes and writing new tests for cursor pagination

formatting

modules/charon-core/src/main/java/org/wso2/charon3/core/config/CharonConfiguration.java

@@ -36,6 +36,7 @@ public class CharonConfiguration implements Configuration {
     private int maxPayLoadSize;
     private int maxResults;
     private ArrayList<Object[]> authenticationSchemes = new ArrayList<Object[]>();
+    private boolean cursorSupport;
 
     //default count value for pagination
     private int count;
@@ -78,6 +79,15 @@ public void setFilterSupport(boolean supported, int maxResults) {
         this.maxResults = maxResults;
     }
 
+    /**
+     * Set cursor pagination support
+     * @param supported
+     */
+    public void setCursorPaginationSupport(boolean supported) {
+
+        this.cursorSupport = supported;
+    }
+
     /*
      * set Change Password Support
      * @param supported
@@ -145,6 +155,7 @@ public HashMap<String, Object> getConfig() {
         configMap.put(SCIMConfigConstants.PATCH, patchSupport);
         configMap.put(SCIMConfigConstants.AUTHENTICATION_SCHEMES, authenticationSchemes);
         configMap.put(SCIMConfigConstants.PAGINATION_DEFAULT_COUNT, count);
+        configMap.put(SCIMConfigConstants.CURSOR, cursorSupport);
         return  configMap;
     }
 

modules/charon-core/src/main/java/org/wso2/charon3/core/config/SCIMConfigConstants.java

@@ -47,6 +47,7 @@ public class SCIMConfigConstants {
     public static final String AUTHENTICATION_SCHEMES = "authenticationSchemes";
     public static final String SCIM_SCHEMA_EXTENSION_CONFIG = "scim2-schema-extension.config";
     public static final String PAGINATION_DEFAULT_COUNT = "pagination-default-count";
+    public static final String CURSOR = "cursor";
 
 
 }

modules/charon-core/src/main/java/org/wso2/charon3/core/encoder/JSONDecoder.java

@@ -830,7 +830,13 @@ public SearchRequest decodeSearchRequestBody(String scimResourceString,
             searchRequest.setExcludedAttributes(excludedAttributes);
             searchRequest.setSchema((String) schemas.get(0));
             searchRequest.setCountStr(decodedJsonObj.optString(SCIMConstants.OperationalConstants.COUNT));
-            searchRequest.setStartIndexStr(decodedJsonObj.optString(SCIMConstants.OperationalConstants.START_INDEX));
+            if (!decodedJsonObj.optString(SCIMConstants.OperationalConstants.START_INDEX).equals("")) {
+                searchRequest.setStartIndexStr(
+                        decodedJsonObj.optString(SCIMConstants.OperationalConstants.START_INDEX));
+            }
+            if (decodedJsonObj.has(SCIMConstants.OperationalConstants.CURSOR)) {
+                searchRequest.setCursor(decodedJsonObj.optString(SCIMConstants.OperationalConstants.CURSOR));
+            }
             searchRequest.setDomainName(decodedJsonObj.optString(SCIMConstants.OperationalConstants.DOMAIN));
             searchRequest.setFilter(rootNode);
             if (!decodedJsonObj.optString(SCIMConstants.OperationalConstants.SORT_BY).equals("")) {

modules/charon-core/src/main/java/org/wso2/charon3/core/encoder/JSONEncoder.java

@@ -418,6 +418,10 @@ public String buildServiceProviderConfigJsonBody(HashMap<String, Object> config)
         changePasswordObject.put(SCIMConstants.ServiceProviderConfigSchemaConstants.SUPPORTED,
                 config.get(SCIMConfigConstants.CHNAGE_PASSWORD));
 
+        JSONObject paginationObject = new JSONObject();
+        paginationObject.put(SCIMConstants.ServiceProviderConfigSchemaConstants.CURSOR,
+                config.get(SCIMConfigConstants.CURSOR));
+
         JSONArray authenticationSchemesArray = new JSONArray();
         ArrayList<Object[]> values = (ArrayList<Object[]>) config.get(SCIMConfigConstants.AUTHENTICATION_SCHEMES);
 
@@ -457,6 +461,8 @@ public String buildServiceProviderConfigJsonBody(HashMap<String, Object> config)
                 etagObject);
         rootObject.put(SCIMConstants.ServiceProviderConfigSchemaConstants.AUTHENTICATION_SCHEMAS,
                 authenticationSchemesArray);
+        rootObject.put(SCIMConstants.ServiceProviderConfigSchemaConstants.PAGINATION,
+                paginationObject);
 
         return rootObject.toString();
 

modules/charon-core/src/main/java/org/wso2/charon3/core/extensions/UserManager.java

@@ -24,6 +24,7 @@
 import org.wso2.charon3.core.exceptions.NotImplementedException;
 import org.wso2.charon3.core.objects.Group;
 import org.wso2.charon3.core.objects.User;
+import org.wso2.charon3.core.objects.plainobjects.Cursor;
 import org.wso2.charon3.core.objects.plainobjects.GroupsGetResponse;
 import org.wso2.charon3.core.objects.plainobjects.UsersGetResponse;
 import org.wso2.charon3.core.schema.AttributeSchema;
@@ -53,22 +54,44 @@ public void deleteUser(String userId)
             throws NotFoundException, CharonException, NotImplementedException, BadRequestException;
 
     /**
-     * List users with Get.
+     * List users with Get using offset pagination.
      *
-     * @param node               Node
-     * @param startIndex         Start Index
-     * @param count              Count
-     * @param sortBy             Sort by
-     * @param sortOrder          Sort order
-     * @param domainName         Domain name
-     * @param requiredAttributes Required user attributes
-     * @return Users with requested attributes
-     * @throws CharonException         Error while listing users
-     * @throws NotImplementedException Operation note implemented
-     * @throws BadRequestException     Bad request
+     * @param node               Tree node built based on the filtering conditions.
+     * @param startIndex         Start Index.
+     * @param count              Count.
+     * @param sortBy             Sort by.
+     * @param sortOrder          Sort order.
+     * @param domainName         Domain name.
+     * @param requiredAttributes Required user attributes.
+     * @return Users with requested attributes.
+     * @throws CharonException         Error while listing users.
+     * @throws NotImplementedException Operation note implemented.
+     * @throws BadRequestException     Bad request.
      */
     default UsersGetResponse listUsersWithGET(Node node, Integer startIndex, Integer count, String sortBy,
-                        String sortOrder, String domainName, Map<String, Boolean> requiredAttributes)
+                                         String sortOrder, String domainName, Map<String, Boolean> requiredAttributes)
+            throws CharonException, NotImplementedException, BadRequestException {
+
+        return null;
+    }
+
+    /**
+     * List users with Get using cursor pagination.
+     *
+     * @param node               Tree node built based on the filtering conditions.
+     * @param cursor             Cursor value for pagination and the Pagination direction.
+     * @param count              Count.
+     * @param sortBy             Sort by.
+     * @param sortOrder          Sort order.
+     * @param domainName         Domain name.
+     * @param requiredAttributes Required user attributes.
+     * @return Users with requested attributes.
+     * @throws CharonException         Error while listing users.
+     * @throws NotImplementedException Operation note implemented.
+     * @throws BadRequestException     Bad request.
+     */
+    default UsersGetResponse listUsersWithGET(Node node, Cursor cursor, Integer count, String sortBy, String sortOrder,
+                                              String domainName, Map<String, Boolean> requiredAttributes)
             throws CharonException, NotImplementedException, BadRequestException {
 
         return null;
@@ -86,7 +109,7 @@ default UsersGetResponse listUsersWithGET(Node node, Integer startIndex, Integer
      */
     @Deprecated
     default UsersGetResponse listUsersWithGET(Node node, int startIndex, int count, String sortBy, String sortOrder,
-            String domainName, Map<String, Boolean> requiredAttributes)
+                                              String domainName, Map<String, Boolean> requiredAttributes)
             throws CharonException, NotImplementedException, BadRequestException {
 
         return null;
@@ -188,7 +211,7 @@ public Group updateGroup(Group oldGroup, Group newGroup, Map<String, Boolean> re
      *
      * @param oldGroup
      * @param newGroup
-     *
+     * 
      * @throws NotImplementedException
      * @throws BadRequestException
      * @throws CharonException

modules/charon-core/src/main/java/org/wso2/charon3/core/objects/ListedResource.java

@@ -130,6 +130,42 @@ public void setStartIndex(int startIndex) {
         }
     }
 
+    /**
+     * Setting the next cursor as an SCIM attribute.
+     *
+     * @param nextCursor  The base64 encoded JSON string, made up of the cursor value and the pagination direction.
+     */
+    public void setNextCursor(String nextCursor) {
+
+        if (!isAttributeExist(SCIMConstants.ListedResourceSchemaConstants.NEXT_CURSOR)) {
+            SimpleAttribute nextCursorAttribute =
+                    new SimpleAttribute(SCIMConstants.ListedResourceSchemaConstants.NEXT_CURSOR, nextCursor);
+            attributeList.put(SCIMConstants.ListedResourceSchemaConstants.NEXT_CURSOR, nextCursorAttribute);
+        } else {
+            SimpleAttribute nextCursorAttribute = ((SimpleAttribute) attributeList
+                    .get(SCIMConstants.ListedResourceSchemaConstants.NEXT_CURSOR));
+            nextCursorAttribute.setValue(nextCursor);
+        }
+    }
+
+    /**
+     * Setting the previous cursor as an SCIM attribute.
+     *
+     * @param previousCursor  The base64 encoded JSON string, made up of the cursor value and the pagination direction.
+     */
+    public void setPreviousCursor(String previousCursor) {
+
+        if (!isAttributeExist(SCIMConstants.ListedResourceSchemaConstants.PREVIOUS_CURSOR)) {
+            SimpleAttribute prevCursorAttribute =
+                    new SimpleAttribute(SCIMConstants.ListedResourceSchemaConstants.PREVIOUS_CURSOR, previousCursor);
+            attributeList.put(SCIMConstants.ListedResourceSchemaConstants.PREVIOUS_CURSOR, prevCursorAttribute);
+        } else {
+            SimpleAttribute previousCursorAttribute = ((SimpleAttribute) attributeList
+                    .get(SCIMConstants.ListedResourceSchemaConstants.PREVIOUS_CURSOR));
+            previousCursorAttribute.setValue(previousCursor);
+        }
+    }
+
     /**
      * set the listed resources
      *

modules/charon-core/src/main/java/org/wso2/charon3/core/objects/plainobjects/Cursor.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.com).
+ *
+ * WSO2 Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.wso2.charon3.core.objects.plainobjects;
+
+/**
+ * This class representation can be used to create a cursor type object which carries direction and cursor value
+ * to be used in cursor-based pagination.
+ */
+public class Cursor {
+
+    private String cursorValue;
+    private String direction;
+
+    public Cursor (String cursorVal, String direction) {
+        this.cursorValue = cursorVal;
+        this.direction = direction;
+    }
+
+    public String getCursorValue() {
+
+        return cursorValue;
+    }
+
+    public void setCursorValue(String cursorValue) {
+
+        this.cursorValue = cursorValue;
+    }
+
+    public String getDirection() {
+
+        return direction;
+    }
+
+    public void setDirection(String direction) {
+
+        this.direction = direction;
+    }
+}

modules/charon-core/src/main/java/org/wso2/charon3/core/objects/plainobjects/UsersGetResponse.java

@@ -22,12 +22,20 @@
 import java.util.List;
 
 /**
+<<<<<<< HEAD
  * This class representation can be used to create a UsersGetResponse type object which carries the total number of
  * users identified from the filter and the list of users returned for this request.
+=======
+ * This class representation can be used to create a UserGetResponse type object which carries the list of users,
+ * total number of users and optionally the previous and next cursor value.
+ *
+>>>>>>> Cursor based pagination for scim resources.
  */
 public class UsersGetResponse {
 
     private int totalUsers;
+    private String nextCursor;
+    private String previousCursor;
     private List<User> users;
 
     /**
@@ -39,6 +47,17 @@ public UsersGetResponse(int totalUsers, List<User> users) {
         this.users = users;
     }
 
+    /**
+     * Constructor used to build a response object when using cursor pagination.
+     */
+    public UsersGetResponse(int totalUsers, String nextCursor, String previousCursor, List<User> users) {
+
+        this.totalUsers = totalUsers;
+        this.nextCursor = nextCursor;
+        this.previousCursor = previousCursor;
+        this.users = users;
+    }
+
     public int getTotalUsers() {
 
         return totalUsers;
@@ -48,6 +67,21 @@ public void setTotalUsers(int totalUsers) {
 
         this.totalUsers = totalUsers;
     }
+    public String getNextCursor() {
+        return nextCursor;
+    }
+
+    public void setNextCursor(String nextCursor) {
+        this.nextCursor = nextCursor;
+    }
+
+    public String getPrevCursor() {
+        return previousCursor;
+    }
+
+    public void setPrevCursor(String prevCursor) {
+        this.previousCursor = prevCursor;
+    }
 
     public List<User> getUsers() {
 

modules/charon-core/src/main/java/org/wso2/charon3/core/protocol/SCIMResponse.java

@@ -28,6 +28,7 @@ public class SCIMResponse {
     //If there are any HTTP header parameters to be set in response other than response code,
     protected Map<String, String> headerParamMap;
 
+
     /*
      * Constructor with three params
      *

modules/charon-core/src/main/java/org/wso2/charon3/core/protocol/endpoints/ResourceManager.java

@@ -78,24 +78,44 @@ public interface ResourceManager {
      */
     @Deprecated
     SCIMResponse listWithGET(UserManager userManager, String filter, int startIndex, int count, String sortBy,
-            String sortOrder, String domainName, String attributes, String excludeAttributes);
+                             String sortOrder, String domainName, String attributes, String excludeAttributes);
 
     /*
-     * Get resources
-     *
-     * @param userManager       User manager
-     * @param filter            Filter to be executed
-     * @param startIndexInt     Starting index value of the filter
-     * @param countInt          Number of required results
-     * @param sortBy            SortBy
-     * @param sortOrder         Sorting order
-     * @param domainName        Domain name
-     * @param attributes        Attributes in the request
-     * @param excludeAttributes Exclude attributes
-     * @return SCIM response
+     * Get resources.
+     *
+     * @param userManager       User manager.
+     * @param filter            Filter to be executed.
+     * @param startIndexInt     Starting index value of the filter.
+     * @param countInt          Number of required results.
+     * @param sortBy            SortBy.
+     * @param sortOrder         Sorting order.
+     * @param domainName        Domain name.
+     * @param attributes        Attributes in the request.
+     * @param excludeAttributes Exclude attributes.
+     * @return SCIM response.
      */
     default SCIMResponse listWithGET(UserManager userManager, String filter, Integer startIndexInt, Integer countInt,
-            String sortBy, String sortOrder, String domainName, String attributes, String excludeAttributes) {
+                String sortBy, String sortOrder, String domainName, String attributes, String excludeAttributes) {
+
+        return null;
+    }
+
+    /*
+     * Get resources with cursor.
+     *
+     * @param userManager       User manager.
+     * @param filter            Filter to be executed.
+     * @param cursor            Cursor value for pagination.
+     * @param countInt          Number of required results.
+     * @param sortBy            SortBy.
+     * @param sortOrder         Sorting order.
+     * @param domainName        Domain name.
+     * @param attributes        Attributes in the request.
+     * @param excludeAttributes Exclude attributes.
+     * @return SCIM response.
+     */
+    default SCIMResponse listWithGET(UserManager userManager, String filter, String cursor, Integer countInt,
+                  String sortBy, String sortOrder, String domainName, String attributes, String excludeAttributes) {
 
         return null;
     }