From 0382ba8d447d59e5721ea15fa20e2c4faeb851ea Mon Sep 17 00:00:00 2001 From: Dilip Devaraj Date: Wed, 5 Jun 2019 16:44:34 -0700 Subject: [PATCH] Throw exception when annotation size equals 10000. Add more tests (#473) Throw exception when annotation size equals 10000. Add more tests --- .../ElasticSearchAnnotationService.java | 15 ++++-- .../service/schema/ElasticSearchUtils.java | 2 +- .../ElasticSearchAnnotationServiceTest.java | 46 +++++++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/ArgusCore/src/main/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationService.java b/ArgusCore/src/main/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationService.java index f5d168f1c..c3e094dcb 100644 --- a/ArgusCore/src/main/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationService.java +++ b/ArgusCore/src/main/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationService.java @@ -7,6 +7,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.Charset; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -77,6 +78,9 @@ public class ElasticSearchAnnotationService extends DefaultService implements An private final ObjectMapper annotationMapper; public static String ANNOTATION_INDEX_TEMPLATE_PATTERN_START; + private static final String EXCEPTION_MESSAGE = "Your query returns {0} or more annotations." + + " Please modify your query by reducing the time window."; + @Inject ElasticSearchAnnotationService(SystemConfiguration config, MonitorService monitorService, ElasticSearchUtils esUtils) { super(config); @@ -167,7 +171,7 @@ public enum Property { /** Connection count for ES REST client. */ ANNOTATION_ES_CONNECTION_COUNT("service.property.annotation.elasticsearch.connection.count", "10"), /** The hashing algorithm to use for generating document id. */ - ANNOTATION_ES_IDGEN_HASH_ALGO("service.property.annotation.elasticsearch.idgen.hash.algo", "XXHASH"), + ANNOTATION_ES_IDGEN_HASH_ALGO("service.property.annotation.elasticsearch.idgen.hash.algo", "MD5"), /** Replication factor */ ANNOTATION_ES_NUM_REPLICAS("service.property.annotation.elasticsearch.num.replicas", "1"), /** Shard count */ @@ -536,7 +540,7 @@ public List getAnnotations(List queries) { requireNotDisposed(); requireArgument(queries != null, "Annotation queries cannot be null."); List annotations = new ArrayList<>(); - int from = 0, scrollSize = 10000; + int from = 0, scrollSize = ElasticSearchUtils.ANNOTATION_INDEX_MAX_RESULT_WINDOW; String requestUrl = String.format("/%s-*/_search", ANNOTATION_INDEX_TEMPLATE_PATTERN_START); try{ @@ -551,6 +555,11 @@ public List getAnnotations(List queries) { String str = extractResponse(response); AnnotationRecordList list = toEntity(str, new TypeReference() {}); annotations.addAll(list.getRecords()); + + if(annotations.size() == scrollSize) { + logger.error("Maximum annotations limit execeeded for query- " + query.toString()); + throw new RuntimeException(MessageFormat.format(EXCEPTION_MESSAGE, scrollSize)); + } } } catch(IOException ex) { throw new SystemException(ex); @@ -570,7 +579,7 @@ private String constructTermQuery(AnnotationQuery query, int from, int size) thr return rootNode.toString(); } - private void convertTimestampToMillis(AnnotationQuery query) { + protected void convertTimestampToMillis(AnnotationQuery query) { long queryStart = query.getStartTimestamp(); long queryEnd = query.getEndTimestamp(); if (queryStart < 100000000000L) query.setStartTimestamp(queryStart * 1000); diff --git a/ArgusCore/src/main/java/com/salesforce/dva/argus/service/schema/ElasticSearchUtils.java b/ArgusCore/src/main/java/com/salesforce/dva/argus/service/schema/ElasticSearchUtils.java index 4c3eb2a9f..84d887b65 100644 --- a/ArgusCore/src/main/java/com/salesforce/dva/argus/service/schema/ElasticSearchUtils.java +++ b/ArgusCore/src/main/java/com/salesforce/dva/argus/service/schema/ElasticSearchUtils.java @@ -36,7 +36,7 @@ public class ElasticSearchUtils { private static ObjectMapper mapper = new ObjectMapper(); private static final int INDEX_MAX_RESULT_WINDOW = 10000; - private static final int ANNOTATION_INDEX_MAX_RESULT_WINDOW = 10000; + public static int ANNOTATION_INDEX_MAX_RESULT_WINDOW = 10000; public ElasticSearchUtils() { diff --git a/ArgusCore/src/test/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationServiceTest.java b/ArgusCore/src/test/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationServiceTest.java index c55a65f42..87d9feb76 100644 --- a/ArgusCore/src/test/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationServiceTest.java +++ b/ArgusCore/src/test/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationServiceTest.java @@ -53,6 +53,7 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; @@ -182,6 +183,34 @@ public static void setUpClass() { esAnnotationService = new ElasticSearchAnnotationService(systemConfig, mockedMonitor, mockedElasticSearchUtils); } + @Before + public void setUp() { + ElasticSearchUtils.ANNOTATION_INDEX_MAX_RESULT_WINDOW = 10000; + } + + @Test + public void testConvertTimestampToMillis(){ + AnnotationQuery annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359073L, 1557809599073L); + esAnnotationService.convertTimestampToMillis(annotationQuery); + assertEquals(1557809359073L, annotationQuery.getStartTimestamp().longValue()); + assertEquals(1557809599073L, annotationQuery.getEndTimestamp().longValue()); + + annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359L, 1557809599L); + esAnnotationService.convertTimestampToMillis(annotationQuery); + assertEquals(1557809359000L, annotationQuery.getStartTimestamp().longValue()); + assertEquals(1557809599000L, annotationQuery.getEndTimestamp().longValue()); + + annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359123L, 1557809599L); + esAnnotationService.convertTimestampToMillis(annotationQuery); + assertEquals(1557809359123L, annotationQuery.getStartTimestamp().longValue()); + assertEquals(1557809599000L, annotationQuery.getEndTimestamp().longValue()); + + annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359L, 1557809599456L); + esAnnotationService.convertTimestampToMillis(annotationQuery); + assertEquals(1557809359000L, annotationQuery.getStartTimestamp().longValue()); + assertEquals(1557809599456L, annotationQuery.getEndTimestamp().longValue()); + } + @Test public void testAnnotationRecordListMapper() throws IOException { mapper = ElasticSearchAnnotationService.getAnnotationObjectMapper(new AnnotationRecordList.IndexSerializer()); @@ -253,6 +282,16 @@ public void testGetAnnotations() throws IOException { assertEquals(expectedURL, capturedRequest.getEndpoint()); } + @Test (expected = RuntimeException.class) + public void testGetAnnotationsExceedingLimit(){ + ElasticSearchUtils.ANNOTATION_INDEX_MAX_RESULT_WINDOW = 1; + AnnotationQuery annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359073L, 1557809599073L); + List queries = new ArrayList<>(); + queries.add(annotationQuery); + ElasticSearchAnnotationService spyService = _initializeSpyService(esAnnotationService, getReply, getReply); + spyService.getAnnotations(queries); + } + @Test public void testDoExtractResponse() throws Exception { final String message = "this is a test"; @@ -295,6 +334,13 @@ public void testHashedSearchIdentifierAnnotationAndAnnotationQueryMatch(){ Annotation annotation = new Annotation("source", "16ab4b56311", "transactionId", "scope1", "metric1", 1557809559073L); annotation.setTags(annotationQueryTags); assertEquals(AnnotationRecordList.getHashedSearchIdentifier(annotation), ElasticSearchAnnotationService.getHashedSearchIdentifier(annotationQuery)); + + // change order of tags in annotation query + Map annotationQueryTags2 = new HashMap<>(); + annotationQueryTags2.put("podName","pod1"); + annotationQueryTags2.put("device","device1"); + AnnotationQuery annotationQuery2 = new AnnotationQuery("scope1", "metric1", annotationQueryTags2, "transactionId", 1557809359073L, 1557809599073L); + assertEquals(AnnotationRecordList.getHashedSearchIdentifier(annotation), ElasticSearchAnnotationService.getHashedSearchIdentifier(annotationQuery2)); } private ElasticSearchAnnotationService _initializeSpyService(ElasticSearchAnnotationService service,