From 0b64bf415381c6174e5630e3681e0f039d535a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6gner?= Date: Fri, 11 Oct 2024 16:04:54 +0200 Subject: [PATCH] Increase the memory-based throttling threshold to 95% MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additionally: - Add all memory info of Runtime to MemoryHealthCheck - Set throttling reason in Stream-Info header for all TOO_MANY_REQUESTS exceptions Signed-off-by: Benjamin Rögner --- .../hub/connectors/RemoteFunctionClient.java | 11 ++-- .../here/xyz/hub/connectors/RpcClient.java | 7 ++- .../main/java/com/here/xyz/hub/rest/Api.java | 5 ++ .../here/xyz/hub/task/FeatureTaskHandler.java | 11 ++-- .../util/health/checks/MemoryHealthCheck.java | 3 + .../src/main/resources/config.json | 2 +- .../rest/TooManyRequestsException.java | 58 +++++++++++++++++++ 7 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 xyz-util/src/main/java/com/here/xyz/util/service/rest/TooManyRequestsException.java diff --git a/xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RemoteFunctionClient.java b/xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RemoteFunctionClient.java index 269fbd89b1..588c2234f4 100644 --- a/xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RemoteFunctionClient.java +++ b/xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RemoteFunctionClient.java @@ -21,7 +21,8 @@ import static com.here.xyz.hub.util.AtomicUtils.compareAndDecrement; import static com.here.xyz.hub.util.AtomicUtils.compareAndIncrementUpTo; -import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS; +import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.QUOTA; +import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.STORAGE_QUEUE_FULL; import com.google.common.io.ByteStreams; import com.here.xyz.Payload; @@ -31,7 +32,7 @@ import com.here.xyz.hub.util.ByteSizeAware; import com.here.xyz.hub.util.LimitedQueue; import com.here.xyz.util.service.Core; -import com.here.xyz.util.service.HttpException; +import com.here.xyz.util.service.rest.TooManyRequestsException; import io.vertx.core.AsyncResult; import io.vertx.core.Context; import io.vertx.core.Future; @@ -186,8 +187,8 @@ private static boolean checkRequesterThrottling(Marker marker, Handler timeoutFc.callback - .handle(Future.failedFuture(new HttpException(TOO_MANY_REQUESTS, "Remote function is busy or cannot be invoked.")))); + .handle(Future.failedFuture(new TooManyRequestsException("Remote function is busy or cannot be invoked.", STORAGE_QUEUE_FULL)))); } public class FunctionCall implements ByteSizeAware { diff --git a/xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RpcClient.java b/xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RpcClient.java index c1a46678a8..ab2b0ea116 100644 --- a/xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RpcClient.java +++ b/xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RpcClient.java @@ -21,6 +21,7 @@ import static com.here.xyz.events.GetFeaturesByTileEvent.ResponseType.MVT; import static com.here.xyz.events.GetFeaturesByTileEvent.ResponseType.MVT_FLATTENED; +import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.CONNECTOR; import static io.netty.handler.codec.http.HttpHeaderValues.APPLICATION_JSON; import static io.netty.handler.codec.http.HttpResponseStatus.BAD_GATEWAY; import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST; @@ -28,7 +29,6 @@ import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN; import static io.netty.handler.codec.http.HttpResponseStatus.GATEWAY_TIMEOUT; import static io.netty.handler.codec.http.HttpResponseStatus.NOT_IMPLEMENTED; -import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS; import static io.netty.handler.codec.rtsp.RtspResponseStatuses.REQUEST_ENTITY_TOO_LARGE; import com.fasterxml.jackson.core.JsonParseException; @@ -57,6 +57,7 @@ import com.here.xyz.responses.XyzResponse; import com.here.xyz.util.service.Core; import com.here.xyz.util.service.HttpException; +import com.here.xyz.util.service.rest.TooManyRequestsException; import io.vertx.core.AsyncResult; import io.vertx.core.Future; import io.vertx.core.Handler; @@ -400,8 +401,8 @@ private void validateResponsePayload(Marker marker, final Typed payload) throws case FORBIDDEN: throw new HttpException(FORBIDDEN, "The user is not authorized.", errorResponse.getErrorDetails()); case TOO_MANY_REQUESTS: - throw new HttpException(TOO_MANY_REQUESTS, - "The connector cannot process the message due to a limitation in an upstream service or a database.", errorResponse.getErrorDetails()); + throw new TooManyRequestsException("The connector cannot process the message due to a limitation in an upstream service or a database.", + CONNECTOR, errorResponse.getErrorDetails()); case ILLEGAL_ARGUMENT: throw new HttpException(BAD_REQUEST, errorResponse.getErrorMessage(), errorResponse.getErrorDetails()); case TIMEOUT: diff --git a/xyz-hub-service/src/main/java/com/here/xyz/hub/rest/Api.java b/xyz-hub-service/src/main/java/com/here/xyz/hub/rest/Api.java index 2426be79bc..579c5b5b01 100644 --- a/xyz-hub-service/src/main/java/com/here/xyz/hub/rest/Api.java +++ b/xyz-hub-service/src/main/java/com/here/xyz/hub/rest/Api.java @@ -34,6 +34,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.here.xyz.XyzSerializable.Public; import com.here.xyz.hub.Service; +import com.here.xyz.hub.XYZHubRESTVerticle; import com.here.xyz.hub.connectors.models.Space.CacheProfile; import com.here.xyz.hub.rest.ApiParam.Query; import com.here.xyz.hub.task.FeatureTask; @@ -52,6 +53,7 @@ import com.here.xyz.responses.changesets.ChangesetCollection; import com.here.xyz.util.service.HttpException; import com.here.xyz.util.service.logging.LogUtil; +import com.here.xyz.util.service.rest.TooManyRequestsException; import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.MultiMap; import io.vertx.core.buffer.Buffer; @@ -305,6 +307,9 @@ void sendResponse(final SpaceTask task) throws JsonProcessingException { * @param e the exception that should be used to generate an {@link ErrorResponse}, if null an internal server error is returned. */ void sendErrorResponse(final Task task, final Throwable e) { + if (e instanceof TooManyRequestsException throttleException) + XYZHubRESTVerticle.addStreamInfo(task.context, "THR", throttleException.reason); //Set the throttling reason at the stream-info header + sendErrorResponse(task.context, e); } diff --git a/xyz-hub-service/src/main/java/com/here/xyz/hub/task/FeatureTaskHandler.java b/xyz-hub-service/src/main/java/com/here/xyz/hub/task/FeatureTaskHandler.java index ae3e150e7d..0363900bcc 100644 --- a/xyz-hub-service/src/main/java/com/here/xyz/hub/task/FeatureTaskHandler.java +++ b/xyz-hub-service/src/main/java/com/here/xyz/hub/task/FeatureTaskHandler.java @@ -29,6 +29,7 @@ import static com.here.xyz.hub.task.FeatureTask.FeatureKey.TYPE; import static com.here.xyz.util.service.BaseHttpServerVerticle.HeaderValues.APPLICATION_VND_HERE_FEATURE_MODIFICATION_LIST; import static com.here.xyz.util.service.BaseHttpServerVerticle.HeaderValues.APPLICATION_VND_MAPBOX_VECTOR_TILE; +import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.MEMORY; import static io.netty.handler.codec.http.HttpResponseStatus.BAD_GATEWAY; import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST; import static io.netty.handler.codec.http.HttpResponseStatus.CONFLICT; @@ -37,7 +38,6 @@ import static io.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED; import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND; import static io.netty.handler.codec.http.HttpResponseStatus.PRECONDITION_REQUIRED; -import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS; import com.fasterxml.jackson.core.JsonProcessingException; import com.here.xyz.Payload; @@ -105,6 +105,7 @@ import com.here.xyz.util.service.Core; import com.here.xyz.util.service.HttpException; import com.here.xyz.util.service.logging.LogUtil; +import com.here.xyz.util.service.rest.TooManyRequestsException; import io.vertx.core.AsyncResult; import io.vertx.core.Future; import io.vertx.core.Handler; @@ -1061,7 +1062,7 @@ static void throttle(final X task, final Callback cal if (Service.IS_USING_ZGC) { if (usedMemoryPercent > Service.configuration.SERVICE_MEMORY_HIGH_UTILIZATION_THRESHOLD) { XYZHubRESTVerticle.addStreamInfo(task.context, "THR", "M"); //Reason for throttling is memory - throw new HttpException(TOO_MANY_REQUESTS, "Too many requests for the service node."); + throw new TooManyRequestsException("Too many requests for the service node.", MEMORY); } } //For other GCs, only throttle requests if the request memory filled up over the specified request memory threshold @@ -1077,7 +1078,7 @@ else if (globalInflightRequestMemory.sum() > RpcClient rpcClient = getRpcClient(storage); if (storageInflightRequestMemorySum > rpcClient.getFunctionClient().getPriority() * GLOBAL_INFLIGHT_REQUEST_MEMORY_SIZE) { XYZHubRESTVerticle.addStreamInfo(task.context, "THR", "M"); //Reason for throttling is memory - throw new HttpException(TOO_MANY_REQUESTS, "Too many requests for the storage."); + throw new TooManyRequestsException("Too many requests for the storage.", MEMORY); } } } @@ -1559,8 +1560,8 @@ public static > void validate(X task, Callback ca } if (task.getEvent() instanceof GetFeaturesByGeometryEvent ev && ev.getGeometry() != null ) { - // DS-641 - /spatial - restrict post/ref geom to max. 12000 coords - final int MAX_NR_COORDINATES = 12000; + // DS-641 - /spatial - restrict post/ref geom to max. 12000 coords + final int MAX_NR_COORDINATES = 12000; int nrCoordinates = ev.getGeometry().getJTSGeometry().getNumPoints(); if( MAX_NR_COORDINATES < nrCoordinates ) { diff --git a/xyz-hub-service/src/main/java/com/here/xyz/hub/util/health/checks/MemoryHealthCheck.java b/xyz-hub-service/src/main/java/com/here/xyz/hub/util/health/checks/MemoryHealthCheck.java index c1beb953ab..88e69a5e35 100644 --- a/xyz-hub-service/src/main/java/com/here/xyz/hub/util/health/checks/MemoryHealthCheck.java +++ b/xyz-hub-service/src/main/java/com/here/xyz/hub/util/health/checks/MemoryHealthCheck.java @@ -24,6 +24,9 @@ public Status execute() { private void attachMemoryInfo(Response r) { try { r.setAdditionalProperty("zgc", Service.IS_USING_ZGC); + r.setAdditionalProperty("totalMemoryBytes", Runtime.getRuntime().totalMemory()); + r.setAdditionalProperty("freeMemoryBytes", Runtime.getRuntime().freeMemory()); + r.setAdditionalProperty("maxMemoryBytes", Runtime.getRuntime().maxMemory()); r.setAdditionalProperty("usedMemoryBytes", Service.getUsedMemoryBytes()); r.setAdditionalProperty("usedMemoryPercent", Service.getUsedMemoryPercent()); } diff --git a/xyz-hub-service/src/main/resources/config.json b/xyz-hub-service/src/main/resources/config.json index fdabc45c3f..ce6f8152e8 100644 --- a/xyz-hub-service/src/main/resources/config.json +++ b/xyz-hub-service/src/main/resources/config.json @@ -28,7 +28,7 @@ "GLOBAL_MAX_QUEUE_SIZE": 1024, "REMOTE_FUNCTION_REQUEST_TIMEOUT": 26, "REMOTE_FUNCTION_MAX_CONNECTIONS": 256, - "REMOTE_FUNCTION_CONNECTION_HIGH_UTILIZATION_THRESHOLD": 0.9, + "REMOTE_FUNCTION_CONNECTION_HIGH_UTILIZATION_THRESHOLD": 0.95, "GLOBAL_INFLIGHT_REQUEST_MEMORY_SIZE_MB": 666, "GLOBAL_INFLIGHT_REQUEST_MEMORY_HIGH_UTILIZATION_THRESHOLD": 0.8, diff --git a/xyz-util/src/main/java/com/here/xyz/util/service/rest/TooManyRequestsException.java b/xyz-util/src/main/java/com/here/xyz/util/service/rest/TooManyRequestsException.java new file mode 100644 index 0000000000..cc6d99b0d7 --- /dev/null +++ b/xyz-util/src/main/java/com/here/xyz/util/service/rest/TooManyRequestsException.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2017-2024 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.xyz.util.service.rest; + +import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS; + +import com.here.xyz.util.service.HttpException; +import java.util.Map; + +public class TooManyRequestsException extends HttpException { + + public final ThrottlingReason reason; + + public TooManyRequestsException(String errorText, ThrottlingReason reason) { + super(TOO_MANY_REQUESTS, errorText); + this.reason = reason; + } + + public TooManyRequestsException(String errorText, ThrottlingReason reason, Map errorDetails) { + super(TOO_MANY_REQUESTS, errorText, errorDetails); + this.reason = reason; + } + + public TooManyRequestsException(String errorText, ThrottlingReason reason, Throwable cause) { + super(TOO_MANY_REQUESTS, errorText, cause); + this.reason = reason; + } + + public enum ThrottlingReason { + MEMORY("M"), + QUOTA("Q"), + STORAGE_QUEUE_FULL("S"), + CONNECTOR("C"); + + public final String shortCut; + + ThrottlingReason(String shortCut) { + this.shortCut = shortCut; + } + } +}