Increase the memory-based throttling threshold to 95%

Additionally:
- Add all memory info of Runtime to MemoryHealthCheck
- Set throttling reason in Stream-Info header for all TOO_MANY_REQUESTS exceptions

Signed-off-by: Benjamin Rögner <[email protected]>

xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RemoteFunctionClient.java

@@ -21,7 +21,8 @@
 
 import static com.here.xyz.hub.util.AtomicUtils.compareAndDecrement;
 import static com.here.xyz.hub.util.AtomicUtils.compareAndIncrementUpTo;
-import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS;
+import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.QUOTA;
+import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.STORAGE_QUEUE_FULL;
 
 import com.google.common.io.ByteStreams;
 import com.here.xyz.Payload;
@@ -31,7 +32,7 @@
 import com.here.xyz.hub.util.ByteSizeAware;
 import com.here.xyz.hub.util.LimitedQueue;
 import com.here.xyz.util.service.Core;
-import com.here.xyz.util.service.HttpException;
+import com.here.xyz.util.service.rest.TooManyRequestsException;
 import io.vertx.core.AsyncResult;
 import io.vertx.core.Context;
 import io.vertx.core.Future;
@@ -186,8 +187,8 @@ private static boolean checkRequesterThrottling(Marker marker, Handler<AsyncResu
       if (!compareAndIncrementUpTo(context.getConnector().getMaxConnectionsPerRequester(), connectionCount)) {
         logger.warn(marker, "Sending to many concurrent requests for user {}. Number of active connections: {}, Maximum allowed per node: {}",
             context.getRequesterId(), connectionCount.get(), context.getConnector().getMaxConnectionsPerRequester());
-        callback.handle(Future.failedFuture(new HttpException(TOO_MANY_REQUESTS, "Maximum number of concurrent requests. "
-                + "Max concurrent connections: " + Math.round(context.getConnector().connectionSettings.maxConnectionsPerRequester * 0.6))));
+        callback.handle(Future.failedFuture(new TooManyRequestsException("Maximum number of concurrent requests. "
+                + "Max concurrent connections: " + Math.round(context.getConnector().connectionSettings.maxConnectionsPerRequester * 0.6), QUOTA)));
         return true;
       }
     }
@@ -471,7 +472,7 @@ private void enqueue(final FunctionCall fc) {
         //Send timeout for discarded (old) calls
         .forEach(timeoutFc ->
             timeoutFc.callback
-                .handle(Future.failedFuture(new HttpException(TOO_MANY_REQUESTS, "Remote function is busy or cannot be invoked."))));
+                .handle(Future.failedFuture(new TooManyRequestsException("Remote function is busy or cannot be invoked.", STORAGE_QUEUE_FULL))));
   }
 
   public class FunctionCall implements ByteSizeAware {

xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RpcClient.java

@@ -21,14 +21,14 @@
 
 import static com.here.xyz.events.GetFeaturesByTileEvent.ResponseType.MVT;
 import static com.here.xyz.events.GetFeaturesByTileEvent.ResponseType.MVT_FLATTENED;
+import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.CONNECTOR;
 import static io.netty.handler.codec.http.HttpHeaderValues.APPLICATION_JSON;
 import static io.netty.handler.codec.http.HttpResponseStatus.BAD_GATEWAY;
 import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
 import static io.netty.handler.codec.http.HttpResponseStatus.CONFLICT;
 import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
 import static io.netty.handler.codec.http.HttpResponseStatus.GATEWAY_TIMEOUT;
 import static io.netty.handler.codec.http.HttpResponseStatus.NOT_IMPLEMENTED;
-import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS;
 import static io.netty.handler.codec.rtsp.RtspResponseStatuses.REQUEST_ENTITY_TOO_LARGE;
 
 import com.fasterxml.jackson.core.JsonParseException;
@@ -57,6 +57,7 @@
 import com.here.xyz.responses.XyzResponse;
 import com.here.xyz.util.service.Core;
 import com.here.xyz.util.service.HttpException;
+import com.here.xyz.util.service.rest.TooManyRequestsException;
 import io.vertx.core.AsyncResult;
 import io.vertx.core.Future;
 import io.vertx.core.Handler;
@@ -400,8 +401,8 @@ private void validateResponsePayload(Marker marker, final Typed payload) throws
         case FORBIDDEN:
           throw new HttpException(FORBIDDEN, "The user is not authorized.", errorResponse.getErrorDetails());
         case TOO_MANY_REQUESTS:
-          throw new HttpException(TOO_MANY_REQUESTS,
-              "The connector cannot process the message due to a limitation in an upstream service or a database.", errorResponse.getErrorDetails());
+          throw new TooManyRequestsException("The connector cannot process the message due to a limitation in an upstream service or a database.",
+              CONNECTOR, errorResponse.getErrorDetails());
         case ILLEGAL_ARGUMENT:
           throw new HttpException(BAD_REQUEST, errorResponse.getErrorMessage(), errorResponse.getErrorDetails());
         case TIMEOUT:

xyz-hub-service/src/main/java/com/here/xyz/hub/rest/Api.java

@@ -34,6 +34,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.here.xyz.XyzSerializable.Public;
 import com.here.xyz.hub.Service;
+import com.here.xyz.hub.XYZHubRESTVerticle;
 import com.here.xyz.hub.connectors.models.Space.CacheProfile;
 import com.here.xyz.hub.rest.ApiParam.Query;
 import com.here.xyz.hub.task.FeatureTask;
@@ -52,6 +53,7 @@
 import com.here.xyz.responses.changesets.ChangesetCollection;
 import com.here.xyz.util.service.HttpException;
 import com.here.xyz.util.service.logging.LogUtil;
+import com.here.xyz.util.service.rest.TooManyRequestsException;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.vertx.core.MultiMap;
 import io.vertx.core.buffer.Buffer;
@@ -305,6 +307,9 @@ void sendResponse(final SpaceTask<?> task) throws JsonProcessingException {
    * @param e the exception that should be used to generate an {@link ErrorResponse}, if null an internal server error is returned.
    */
   void sendErrorResponse(final Task task, final Throwable e) {
+    if (e instanceof TooManyRequestsException throttleException)
+      XYZHubRESTVerticle.addStreamInfo(task.context, "THR", throttleException.reason); //Set the throttling reason at the stream-info header
+
     sendErrorResponse(task.context, e);
   }
 

xyz-hub-service/src/main/java/com/here/xyz/hub/task/FeatureTaskHandler.java

@@ -29,6 +29,7 @@
 import static com.here.xyz.hub.task.FeatureTask.FeatureKey.TYPE;
 import static com.here.xyz.util.service.BaseHttpServerVerticle.HeaderValues.APPLICATION_VND_HERE_FEATURE_MODIFICATION_LIST;
 import static com.here.xyz.util.service.BaseHttpServerVerticle.HeaderValues.APPLICATION_VND_MAPBOX_VECTOR_TILE;
+import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.MEMORY;
 import static io.netty.handler.codec.http.HttpResponseStatus.BAD_GATEWAY;
 import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
 import static io.netty.handler.codec.http.HttpResponseStatus.CONFLICT;
@@ -37,7 +38,6 @@
 import static io.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED;
 import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
 import static io.netty.handler.codec.http.HttpResponseStatus.PRECONDITION_REQUIRED;
-import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.here.xyz.Payload;
@@ -105,6 +105,7 @@
 import com.here.xyz.util.service.Core;
 import com.here.xyz.util.service.HttpException;
 import com.here.xyz.util.service.logging.LogUtil;
+import com.here.xyz.util.service.rest.TooManyRequestsException;
 import io.vertx.core.AsyncResult;
 import io.vertx.core.Future;
 import io.vertx.core.Handler;
@@ -1061,7 +1062,7 @@ static <X extends FeatureTask> void throttle(final X task, final Callback<X> cal
       if (Service.IS_USING_ZGC) {
         if (usedMemoryPercent > Service.configuration.SERVICE_MEMORY_HIGH_UTILIZATION_THRESHOLD) {
           XYZHubRESTVerticle.addStreamInfo(task.context, "THR", "M"); //Reason for throttling is memory
-          throw new HttpException(TOO_MANY_REQUESTS, "Too many requests for the service node.");
+          throw new TooManyRequestsException("Too many requests for the service node.", MEMORY);
         }
       }
       //For other GCs, only throttle requests if the request memory filled up over the specified request memory threshold
@@ -1077,7 +1078,7 @@ else if (globalInflightRequestMemory.sum() >
         RpcClient rpcClient = getRpcClient(storage);
         if (storageInflightRequestMemorySum > rpcClient.getFunctionClient().getPriority() * GLOBAL_INFLIGHT_REQUEST_MEMORY_SIZE) {
           XYZHubRESTVerticle.addStreamInfo(task.context, "THR", "M"); //Reason for throttling is memory
-          throw new HttpException(TOO_MANY_REQUESTS, "Too many requests for the storage.");
+          throw new TooManyRequestsException("Too many requests for the storage.", MEMORY);
         }
       }
     }
@@ -1559,8 +1560,8 @@ public static <X extends FeatureTask<?, X>> void validate(X task, Callback<X> ca
     }
 
     if (task.getEvent() instanceof GetFeaturesByGeometryEvent ev && ev.getGeometry() != null ) {
-    // DS-641 - /spatial - restrict post/ref geom to max. 12000 coords  
-      final int MAX_NR_COORDINATES = 12000; 
+    // DS-641 - /spatial - restrict post/ref geom to max. 12000 coords
+      final int MAX_NR_COORDINATES = 12000;
       int nrCoordinates = ev.getGeometry().getJTSGeometry().getNumPoints();
       if( MAX_NR_COORDINATES < nrCoordinates )
       {

xyz-hub-service/src/main/java/com/here/xyz/hub/util/health/checks/MemoryHealthCheck.java

@@ -24,6 +24,9 @@ public Status execute() {
   private void attachMemoryInfo(Response r) {
     try {
       r.setAdditionalProperty("zgc", Service.IS_USING_ZGC);
+      r.setAdditionalProperty("totalMemoryBytes", Runtime.getRuntime().totalMemory());
+      r.setAdditionalProperty("freeMemoryBytes", Runtime.getRuntime().freeMemory());
+      r.setAdditionalProperty("maxMemoryBytes", Runtime.getRuntime().maxMemory());
       r.setAdditionalProperty("usedMemoryBytes", Service.getUsedMemoryBytes());
       r.setAdditionalProperty("usedMemoryPercent", Service.getUsedMemoryPercent());
     }

xyz-hub-service/src/main/resources/config.json

@@ -28,7 +28,7 @@
   "GLOBAL_MAX_QUEUE_SIZE": 1024,
   "REMOTE_FUNCTION_REQUEST_TIMEOUT": 26,
   "REMOTE_FUNCTION_MAX_CONNECTIONS": 256,
-  "REMOTE_FUNCTION_CONNECTION_HIGH_UTILIZATION_THRESHOLD": 0.9,
+  "REMOTE_FUNCTION_CONNECTION_HIGH_UTILIZATION_THRESHOLD": 0.95,
 
   "GLOBAL_INFLIGHT_REQUEST_MEMORY_SIZE_MB": 666,
   "GLOBAL_INFLIGHT_REQUEST_MEMORY_HIGH_UTILIZATION_THRESHOLD": 0.8,

xyz-util/src/main/java/com/here/xyz/util/service/rest/TooManyRequestsException.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2017-2024 HERE Europe B.V.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package com.here.xyz.util.service.rest;
+
+import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS;
+
+import com.here.xyz.util.service.HttpException;
+import java.util.Map;
+
+public class TooManyRequestsException extends HttpException {
+
+  public final ThrottlingReason reason;
+
+  public TooManyRequestsException(String errorText, ThrottlingReason reason) {
+    super(TOO_MANY_REQUESTS, errorText);
+    this.reason = reason;
+  }
+
+  public TooManyRequestsException(String errorText, ThrottlingReason reason, Map<String, Object> errorDetails) {
+    super(TOO_MANY_REQUESTS, errorText, errorDetails);
+    this.reason = reason;
+  }
+
+  public TooManyRequestsException(String errorText, ThrottlingReason reason, Throwable cause) {
+    super(TOO_MANY_REQUESTS, errorText, cause);
+    this.reason = reason;
+  }
+
+  public enum ThrottlingReason {
+    MEMORY("M"),
+    QUOTA("Q"),
+    STORAGE_QUEUE_FULL("S"),
+    CONNECTOR("C");
+
+    public final String shortCut;
+
+    ThrottlingReason(String shortCut) {
+      this.shortCut = shortCut;
+    }
+  }
+}