chore: Various REST-related fixes & improvements

fix: FoD & SC-DAST paging functionality

feat: `fcli * rest call`: Add `--no-paging` and `--transform` options

fix: `fcli * rest call`: Fix `--no-transform` behavior

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/cli/mixin/OutputHelperMixins.java

@@ -71,6 +71,11 @@ public static class DetailsNoQuery extends Other {
         @Getter private StandardOutputConfig basicOutputConfig = StandardOutputConfig.details(); 
     }
 
+    public static class DetailsWithQuery extends Other {
+        @Getter @Mixin private OutputWriterWithQueryFactoryMixin outputWriterFactory;
+        @Getter private StandardOutputConfig basicOutputConfig = StandardOutputConfig.details(); 
+    }
+
 
     public static class Add extends TableNoQuery {
         public static final String CMD_NAME = "add";
@@ -204,9 +209,7 @@ public static class Logout extends TableNoQuery {
         public static final String CMD_NAME = "logout";
     }
 
-    public static class RestCall extends Other {
+    public static class RestCall extends DetailsWithQuery {
         public static final String CMD_NAME = "call";
-        @Getter @Mixin private OutputWriterWithQueryFactoryMixin outputWriterFactory;
-        @Getter private StandardOutputConfig basicOutputConfig = StandardOutputConfig.table();
     }
 }

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/cli/cmd/AbstractRestCallCommand.java

@@ -16,11 +16,14 @@
 import java.nio.file.Path;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fortify.cli.common.json.JsonHelper;
 import com.fortify.cli.common.output.cli.cmd.AbstractOutputCommand;
 import com.fortify.cli.common.output.cli.cmd.IBaseRequestSupplier;
 import com.fortify.cli.common.output.product.IProductHelperSupplier;
 import com.fortify.cli.common.output.transform.IInputTransformer;
 import com.fortify.cli.common.output.transform.IRecordTransformer;
+import com.fortify.cli.common.rest.paging.INextPageUrlProducer;
+import com.fortify.cli.common.rest.paging.INextPageUrlProducerSupplier;
 import com.fortify.cli.common.rest.unirest.IUnirestInstanceSupplier;
 import com.fortify.cli.common.util.DisableTest;
 import com.fortify.cli.common.util.DisableTest.TestType;
@@ -31,21 +34,43 @@
 import kong.unirest.UnirestInstance;
 import lombok.Getter;
 import lombok.SneakyThrows;
+import picocli.CommandLine.ArgGroup;
 import picocli.CommandLine.Option;
 import picocli.CommandLine.Parameters;
 
-public abstract class AbstractRestCallCommand extends AbstractOutputCommand implements IBaseRequestSupplier, IProductHelperSupplier, IInputTransformer, IRecordTransformer {
+/**
+ * Abstract base class for 'fcli <product> rest call' commands. Concrete implementations must 
+ * implement the various abstract methods. As dictated by the {@link IProductHelperSupplier},
+ * implementations must implement the {@link IProductHelperSupplier#getProductHelper()} method,
+ * but please note that the provided product helper may not implement any of the 
+ * {@link IInputTransformer}, {@link IRecordTransformer}, {@link INextPageUrlProducerSupplier}
+ * or {@link INextPageUrlProducer} as this would break the --no-transform and --no-paging options.
+ * Instead, subclasses should implement the corresponding _* methods defined in this class,
+ * to enable/disable paging and transformations on demand.
+ * 
+ * @author Ruud Senden
+ */
+public abstract class AbstractRestCallCommand extends AbstractOutputCommand implements IBaseRequestSupplier, IProductHelperSupplier, IInputTransformer, IRecordTransformer, INextPageUrlProducerSupplier {
     @Parameters(index = "0", arity = "1..1", descriptionKey = "api.uri") String uri;
 
     @Option(names = {"--request", "-X"}, required = false, defaultValue = "GET")
     @DisableTest(TestType.OPT_SHORT_NAME)
     @Getter private String httpMethod;
 
     @Option(names = {"--data", "-d"}, required = false)
-    @Getter private String data; // TODO Add ability to read data from file
+    @Getter private String data;
 
-    @Option(names="--no-transform", negatable = true, defaultValue = "false") 
-    private boolean noTransform;
+    @Option(names="--no-paging", negatable = false, defaultValue = "false") 
+    private boolean noPaging;
+
+    @ArgGroup(exclusive = true) private TransformArgGroup transform = new TransformArgGroup();
+    private static class TransformArgGroup {
+        @Option(names="--no-transform", negatable = false, defaultValue = "false") 
+        private boolean noTransform;
+
+        @Option(names={"-t", "--transform"}, paramLabel = "<expr>") 
+        private String transformExpression;
+    }
 
     // TODO Add options for content-type, arbitrary headers, ...?
 
@@ -64,21 +89,36 @@ public boolean isSingular() {
     }
 
     @Override
-    public JsonNode transformInput(JsonNode input) {
-        if ( !noTransform && getProductHelper() instanceof IInputTransformer ) {
-            input = ((IInputTransformer)getProductHelper()).transformInput(input);
+    public final JsonNode transformInput(JsonNode input) {
+        if ( StringUtils.isNotBlank(transform.transformExpression) ) {
+            input = JsonHelper.evaluateSpelExpression(input, transform.transformExpression, JsonNode.class);
+        } else if ( !transform.noTransform ) {
+            input = _transformInput(input);
         }
         return input;
     }
-    
+
     @Override
-    public JsonNode transformRecord(JsonNode input) {
-        if ( !noTransform && getProductHelper() instanceof IRecordTransformer ) {
-            input = ((IRecordTransformer)getProductHelper()).transformRecord(input);
+    public final JsonNode transformRecord(JsonNode input) {
+        if ( !transform.noTransform ) {
+            input = _transformRecord(input);
         }
         return input;
     }
-
+
+    @Override
+    public final INextPageUrlProducer getNextPageUrlProducer(HttpRequest<?> originalRequest) {
+        INextPageUrlProducer result = null;
+        if ( !noPaging ) {
+            result = _getNextPageUrlProducer(originalRequest);
+        }
+        return result;
+    }
+
+    protected abstract JsonNode _transformRecord(JsonNode input);
+    protected abstract JsonNode _transformInput(JsonNode input);
+    protected abstract INextPageUrlProducer _getNextPageUrlProducer(HttpRequest<?> originalRequest);
+
     @SneakyThrows
     protected final HttpRequest<?> prepareRequest(UnirestInstance unirest) {
         if ( StringUtils.isBlank(uri) ) {

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java

@@ -43,7 +43,7 @@ public static final URI addOrReplaceParam(URI uri, String param, Object newValue
         var pattern = String.format("([&?])(%s=)([^&]*)", param);
         var query = uri.getQuery().replaceAll(pattern, "");
         var newParamAndValue = String.format("%s=%s", param, URLEncoder.encode(newValue.toString(), StandardCharsets.UTF_8));
-        query = (StringUtils.isBlank(query) ? "" : "&") + newParamAndValue;
+        query = (StringUtils.isBlank(query) ? "" : query+"&") + newParamAndValue;
         return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), 
                 uri.getPath(), query, uri.getFragment());
     }

fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/output/cli/AbstractFoDOutputCommand.java

@@ -15,7 +15,7 @@
 import com.fortify.cli.common.output.cli.cmd.AbstractOutputCommand;
 import com.fortify.cli.common.output.product.IProductHelperSupplier;
 import com.fortify.cli.common.rest.unirest.IUnirestInstanceSupplier;
-import com.fortify.cli.fod._common.output.mixin.FoDProductHelperMixin;
+import com.fortify.cli.fod._common.output.mixin.FoDProductHelperStandardMixin;
 
 import kong.unirest.UnirestInstance;
 import lombok.Getter;
@@ -24,7 +24,7 @@
 public abstract class AbstractFoDOutputCommand extends AbstractOutputCommand 
     implements IProductHelperSupplier, IUnirestInstanceSupplier
 {
-    @Getter @Mixin FoDProductHelperMixin productHelper;
+    @Getter @Mixin FoDProductHelperStandardMixin productHelper;
 
     public final UnirestInstance getUnirestInstance() {
         return productHelper.getUnirestInstance();

fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/output/mixin/FoDProductHelperMixin.java → fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/output/mixin/FoDProductHelperBasicMixin.java

@@ -12,37 +12,20 @@
  *******************************************************************************/
 package com.fortify.cli.fod._common.output.mixin;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fortify.cli.common.http.proxy.helper.ProxyHelper;
 import com.fortify.cli.common.output.product.IProductHelper;
-import com.fortify.cli.common.output.transform.IInputTransformer;
-import com.fortify.cli.common.rest.paging.INextPageUrlProducer;
-import com.fortify.cli.common.rest.paging.INextPageUrlProducerSupplier;
 import com.fortify.cli.common.rest.unirest.config.UnirestJsonHeaderConfigurer;
 import com.fortify.cli.common.rest.unirest.config.UnirestUnexpectedHttpResponseConfigurer;
 import com.fortify.cli.common.rest.unirest.config.UnirestUrlConfigConfigurer;
 import com.fortify.cli.common.session.cli.mixin.AbstractSessionUnirestInstanceSupplierMixin;
-import com.fortify.cli.fod._common.rest.helper.FoDInputTransformer;
-import com.fortify.cli.fod._common.rest.helper.FoDPagingHelper;
 import com.fortify.cli.fod._common.session.helper.FoDSessionDescriptor;
 import com.fortify.cli.fod._common.session.helper.FoDSessionHelper;
 
-import kong.unirest.HttpRequest;
 import kong.unirest.UnirestInstance;
 
-public class FoDProductHelperMixin extends AbstractSessionUnirestInstanceSupplierMixin<FoDSessionDescriptor> 
-    implements IProductHelper, IInputTransformer, INextPageUrlProducerSupplier 
-{
-    @Override
-    public INextPageUrlProducer getNextPageUrlProducer(HttpRequest<?> originalRequest) {
-        return FoDPagingHelper.nextPageUrlProducer(originalRequest);
-    }
-
-    @Override
-    public JsonNode transformInput(JsonNode input) {
-        return FoDInputTransformer.getItems(input);
-    }
-
+public class FoDProductHelperBasicMixin extends AbstractSessionUnirestInstanceSupplierMixin<FoDSessionDescriptor> 
+    implements IProductHelper
+{   
     @Override
     protected final FoDSessionDescriptor getSessionDescriptor(String sessionName) {
         return FoDSessionHelper.instance().get(sessionName, true);

fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/output/mixin/FoDProductHelperStandardMixin.java

@@ -0,0 +1,38 @@
+/*******************************************************************************
+ * Copyright 2021, 2023 Open Text.
+ *
+ * The only warranties for products and services of Open Text 
+ * and its affiliates and licensors ("Open Text") are as may 
+ * be set forth in the express warranty statements accompanying 
+ * such products and services. Nothing herein should be construed 
+ * as constituting an additional warranty. Open Text shall not be 
+ * liable for technical or editorial errors or omissions contained 
+ * herein. The information contained herein is subject to change 
+ * without notice.
+ *******************************************************************************/
+package com.fortify.cli.fod._common.output.mixin;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fortify.cli.common.output.transform.IInputTransformer;
+import com.fortify.cli.common.rest.paging.INextPageUrlProducer;
+import com.fortify.cli.common.rest.paging.INextPageUrlProducerSupplier;
+import com.fortify.cli.fod._common.rest.helper.FoDInputTransformer;
+import com.fortify.cli.fod._common.rest.helper.FoDPagingHelper;
+
+import kong.unirest.HttpRequest;
+
+// IMPORTANT: When updating/adding any methods in this class, FoDRestCallCommand
+// also likely needs to be updated
+public class FoDProductHelperStandardMixin extends FoDProductHelperBasicMixin 
+    implements IInputTransformer, INextPageUrlProducerSupplier 
+{
+    @Override
+    public INextPageUrlProducer getNextPageUrlProducer(HttpRequest<?> originalRequest) {
+        return FoDPagingHelper.nextPageUrlProducer(originalRequest);
+    }
+
+    @Override
+    public JsonNode transformInput(JsonNode input) {
+        return FoDInputTransformer.getItems(input);
+    }
+}

fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/rest/cli/cmd/FoDRestCallCommand.java

@@ -12,12 +12,17 @@
  *******************************************************************************/
 package com.fortify.cli.fod.rest.cli.cmd;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins;
 import com.fortify.cli.common.rest.cli.cmd.AbstractRestCallCommand;
+import com.fortify.cli.common.rest.paging.INextPageUrlProducer;
 import com.fortify.cli.common.util.DisableTest;
 import com.fortify.cli.common.util.DisableTest.TestType;
-import com.fortify.cli.fod._common.output.mixin.FoDProductHelperMixin;
+import com.fortify.cli.fod._common.output.mixin.FoDProductHelperBasicMixin;
+import com.fortify.cli.fod._common.rest.helper.FoDInputTransformer;
+import com.fortify.cli.fod._common.rest.helper.FoDPagingHelper;
 
+import kong.unirest.HttpRequest;
 import lombok.Getter;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Mixin;
@@ -26,5 +31,20 @@
 @DisableTest(TestType.CMD_DEFAULT_TABLE_OPTIONS_PRESENT) // Output columns depend on response contents
 public final class FoDRestCallCommand extends AbstractRestCallCommand {
     @Getter @Mixin private OutputHelperMixins.RestCall outputHelper;
-    @Getter @Mixin private FoDProductHelperMixin productHelper;
+    @Getter @Mixin private FoDProductHelperBasicMixin productHelper;
+
+    @Override
+    protected INextPageUrlProducer _getNextPageUrlProducer(HttpRequest<?> originalRequest) {
+        return FoDPagingHelper.nextPageUrlProducer(originalRequest);
+    }
+
+    @Override
+    protected JsonNode _transformInput(JsonNode input) {
+        return FoDInputTransformer.getItems(input);
+    }
+
+    @Override
+    protected JsonNode _transformRecord(JsonNode input) {
+        return input;
+    }
 }

fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/scan/cli/cmd/FoDScanWaitForCommand.java

@@ -16,7 +16,7 @@
 import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins;
 import com.fortify.cli.common.rest.cli.cmd.AbstractWaitForCommand;
 import com.fortify.cli.common.rest.wait.WaitHelper.WaitHelperBuilder;
-import com.fortify.cli.fod._common.output.mixin.FoDProductHelperMixin;
+import com.fortify.cli.fod._common.output.mixin.FoDProductHelperStandardMixin;
 import com.fortify.cli.fod.scan.cli.mixin.FoDScanResolverMixin;
 import com.fortify.cli.fod.scan.helper.FoDScanHelper;
 import com.fortify.cli.fod.scan.helper.FoDScanStatus;
@@ -27,7 +27,7 @@
 
 @Command(name = OutputHelperMixins.WaitFor.CMD_NAME)
 public class FoDScanWaitForCommand extends AbstractWaitForCommand {
-    @Getter @Mixin FoDProductHelperMixin productHelper;
+    @Getter @Mixin FoDProductHelperStandardMixin productHelper;
     @Mixin private FoDScanResolverMixin.PositionalParameterMulti scansResolver;
 
     @Override

fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties

@@ -104,9 +104,14 @@ fcli.fod.rest.usage.description = These commands allow for direct interaction wi
   somewhat similar to using 'curl' but benefiting from standard fcli functionality like session management \
   (no need to manually specify Authorization header), rich output formatting options, and query functionality.
 fcli.fod.rest.call.usage.header = Call an individual Fortify FoD REST API endpoint.
+fcli.fod.rest.call.no-paging = By default, this command will load all pages of data from FoD (from the \
+  given offset if specified as a request parameter). Use this option to return only a single page. 
 fcli.fod.rest.call.no-transform = By default, this command performs generic transformations on FoD REST \
   responses, like only outputting the actual response data (contents of the 'items' property). Use this \
   option to output the original response contents without transformations.
+fcli.fod.rest.call.transform = This option allows for performing custom transformations on the response \
+  data based on a Spring Expression Language (SpEL) expression. For example, this allows for retrieving \
+  data from sub-properties, or using project selection/projection.
 
 
 ### For the "fod app" command ###

fcli-core/fcli-sc-dast/src/main/java/com/fortify/cli/sc_dast/_common/output/cli/cmd/AbstractSCDastOutputCommand.java

@@ -15,7 +15,7 @@
 import com.fortify.cli.common.output.cli.cmd.AbstractOutputCommand;
 import com.fortify.cli.common.output.product.IProductHelperSupplier;
 import com.fortify.cli.common.rest.unirest.IUnirestInstanceSupplier;
-import com.fortify.cli.sc_dast._common.output.cli.mixin.SCDastProductHelperMixin;
+import com.fortify.cli.sc_dast._common.output.cli.mixin.SCDastProductHelperStandardMixin;
 
 import kong.unirest.UnirestInstance;
 import lombok.Getter;
@@ -24,7 +24,7 @@
 public abstract class AbstractSCDastOutputCommand extends AbstractOutputCommand 
     implements IProductHelperSupplier, IUnirestInstanceSupplier
 {
-    @Getter @Mixin SCDastProductHelperMixin productHelper;
+    @Getter @Mixin SCDastProductHelperStandardMixin productHelper;
 
     public final UnirestInstance getUnirestInstance() {
         return productHelper.getUnirestInstance();

fcli-core/fcli-sc-dast/src/main/java/com/fortify/cli/sc_dast/_common/output/cli/mixin/SCDastProductHelperMixin.java → fcli-core/fcli-sc-dast/src/main/java/com/fortify/cli/sc_dast/_common/output/cli/mixin/SCDastProductHelperBasicMixin.java

@@ -12,46 +12,20 @@
  *******************************************************************************/
 package com.fortify.cli.sc_dast._common.output.cli.mixin;
 
-import java.util.function.UnaryOperator;
-
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fortify.cli.common.http.proxy.helper.ProxyHelper;
-import com.fortify.cli.common.output.cli.mixin.IOutputHelper;
 import com.fortify.cli.common.output.product.IProductHelper;
-import com.fortify.cli.common.output.transform.IInputTransformer;
-import com.fortify.cli.common.rest.paging.INextPageUrlProducer;
-import com.fortify.cli.common.rest.paging.INextPageUrlProducerSupplier;
-import com.fortify.cli.common.rest.unirest.IUnirestInstanceSupplier;
 import com.fortify.cli.common.rest.unirest.config.UnirestJsonHeaderConfigurer;
 import com.fortify.cli.common.rest.unirest.config.UnirestUnexpectedHttpResponseConfigurer;
 import com.fortify.cli.common.rest.unirest.config.UnirestUrlConfigConfigurer;
 import com.fortify.cli.common.session.cli.mixin.AbstractSessionUnirestInstanceSupplierMixin;
-import com.fortify.cli.sc_dast._common.rest.helper.SCDastInputTransformer;
-import com.fortify.cli.sc_dast._common.rest.helper.SCDastPagingHelper;
 import com.fortify.cli.sc_dast._common.session.helper.SCDastSessionDescriptor;
 import com.fortify.cli.sc_dast._common.session.helper.SCDastSessionHelper;
 
-import kong.unirest.HttpRequest;
 import kong.unirest.UnirestInstance;
-import lombok.Getter;
-import lombok.Setter;
 
-public class SCDastProductHelperMixin extends AbstractSessionUnirestInstanceSupplierMixin<SCDastSessionDescriptor> 
-    implements IProductHelper, IInputTransformer, INextPageUrlProducerSupplier, IUnirestInstanceSupplier
+public class SCDastProductHelperBasicMixin extends AbstractSessionUnirestInstanceSupplierMixin<SCDastSessionDescriptor> 
+    implements IProductHelper
 {
-    @Getter @Setter private IOutputHelper outputHelper;
-    @Getter private UnaryOperator<JsonNode> inputTransformer = SCDastInputTransformer::getItems;
-
-    @Override
-    public INextPageUrlProducer getNextPageUrlProducer(HttpRequest<?> originalRequest) {
-        return SCDastPagingHelper.nextPageUrlProducer(originalRequest);
-    }
-
-    @Override
-    public JsonNode transformInput(JsonNode input) {
-        return SCDastInputTransformer.getItems(input);
-    }
-
     @Override
     protected final void configure(UnirestInstance unirest, SCDastSessionDescriptor sessionDescriptor) {
         UnirestUnexpectedHttpResponseConfigurer.configure(unirest);