Throw exception when annotation size equals 10000. Add more tests (#473

)

Throw exception when annotation size equals 10000. Add more tests

ArgusCore/src/main/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationService.java

@@ -7,6 +7,7 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.charset.Charset;
+import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -77,6 +78,9 @@ public class ElasticSearchAnnotationService extends DefaultService implements An
     private final ObjectMapper annotationMapper;
     public static String ANNOTATION_INDEX_TEMPLATE_PATTERN_START;
 
+    private static final String EXCEPTION_MESSAGE = "Your query returns {0} or more annotations."
+            + " Please modify your query by reducing the time window.";
+
     @Inject
     ElasticSearchAnnotationService(SystemConfiguration config, MonitorService monitorService, ElasticSearchUtils esUtils) {
         super(config);
@@ -167,7 +171,7 @@ public enum Property {
         /** Connection count for ES REST client. */
         ANNOTATION_ES_CONNECTION_COUNT("service.property.annotation.elasticsearch.connection.count", "10"),
         /** The hashing algorithm to use for generating document id. */
-        ANNOTATION_ES_IDGEN_HASH_ALGO("service.property.annotation.elasticsearch.idgen.hash.algo", "XXHASH"),
+        ANNOTATION_ES_IDGEN_HASH_ALGO("service.property.annotation.elasticsearch.idgen.hash.algo", "MD5"),
         /** Replication factor */
         ANNOTATION_ES_NUM_REPLICAS("service.property.annotation.elasticsearch.num.replicas", "1"),
         /** Shard count */
@@ -536,7 +540,7 @@ public List<Annotation> getAnnotations(List<AnnotationQuery> queries) {
         requireNotDisposed();
         requireArgument(queries != null, "Annotation queries cannot be null.");
         List<Annotation> annotations = new ArrayList<>();
-        int from = 0, scrollSize = 10000;
+        int from = 0, scrollSize = ElasticSearchUtils.ANNOTATION_INDEX_MAX_RESULT_WINDOW;
 
         String requestUrl = String.format("/%s-*/_search", ANNOTATION_INDEX_TEMPLATE_PATTERN_START);
         try{
@@ -551,6 +555,11 @@ public List<Annotation> getAnnotations(List<AnnotationQuery> queries) {
                 String str = extractResponse(response);
                 AnnotationRecordList list = toEntity(str, new TypeReference<AnnotationRecordList>() {});
                 annotations.addAll(list.getRecords());
+
+                if(annotations.size() == scrollSize) {
+                    logger.error("Maximum annotations limit execeeded for query- " + query.toString());
+                    throw new RuntimeException(MessageFormat.format(EXCEPTION_MESSAGE, scrollSize));
+                }
             }
         } catch(IOException ex) {
             throw new SystemException(ex);
@@ -570,7 +579,7 @@ private String constructTermQuery(AnnotationQuery query, int from, int size) thr
         return rootNode.toString();
     }
 
-    private void convertTimestampToMillis(AnnotationQuery query) {
+    protected void convertTimestampToMillis(AnnotationQuery query) {
         long queryStart = query.getStartTimestamp();
         long queryEnd = query.getEndTimestamp();
         if (queryStart < 100000000000L) query.setStartTimestamp(queryStart * 1000);

ArgusCore/src/main/java/com/salesforce/dva/argus/service/schema/ElasticSearchUtils.java

@@ -36,7 +36,7 @@ public class ElasticSearchUtils {
     private static ObjectMapper mapper = new ObjectMapper();
 
     private static final int INDEX_MAX_RESULT_WINDOW = 10000;
-    private static final int  ANNOTATION_INDEX_MAX_RESULT_WINDOW = 10000;
+    public static int  ANNOTATION_INDEX_MAX_RESULT_WINDOW = 10000;
 
 
     public ElasticSearchUtils() {

ArgusCore/src/test/java/com/salesforce/dva/argus/service/annotation/ElasticSearchAnnotationServiceTest.java

@@ -53,6 +53,7 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RestClient;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
@@ -182,6 +183,34 @@ public static void setUpClass() {
         esAnnotationService = new ElasticSearchAnnotationService(systemConfig, mockedMonitor, mockedElasticSearchUtils);
     }
 
+    @Before
+    public void setUp() {
+        ElasticSearchUtils.ANNOTATION_INDEX_MAX_RESULT_WINDOW = 10000;
+    }
+
+    @Test
+    public void testConvertTimestampToMillis(){
+        AnnotationQuery annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359073L, 1557809599073L);
+        esAnnotationService.convertTimestampToMillis(annotationQuery);
+        assertEquals(1557809359073L, annotationQuery.getStartTimestamp().longValue());
+        assertEquals(1557809599073L, annotationQuery.getEndTimestamp().longValue());
+
+        annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359L, 1557809599L);
+        esAnnotationService.convertTimestampToMillis(annotationQuery);
+        assertEquals(1557809359000L, annotationQuery.getStartTimestamp().longValue());
+        assertEquals(1557809599000L, annotationQuery.getEndTimestamp().longValue());
+
+        annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359123L, 1557809599L);
+        esAnnotationService.convertTimestampToMillis(annotationQuery);
+        assertEquals(1557809359123L, annotationQuery.getStartTimestamp().longValue());
+        assertEquals(1557809599000L, annotationQuery.getEndTimestamp().longValue());
+
+        annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359L, 1557809599456L);
+        esAnnotationService.convertTimestampToMillis(annotationQuery);
+        assertEquals(1557809359000L, annotationQuery.getStartTimestamp().longValue());
+        assertEquals(1557809599456L, annotationQuery.getEndTimestamp().longValue());
+    }
+
     @Test
     public void testAnnotationRecordListMapper() throws IOException {
         mapper = ElasticSearchAnnotationService.getAnnotationObjectMapper(new AnnotationRecordList.IndexSerializer());
@@ -253,6 +282,16 @@ public void testGetAnnotations() throws IOException {
         assertEquals(expectedURL, capturedRequest.getEndpoint());
     }
 
+    @Test (expected = RuntimeException.class)
+    public void testGetAnnotationsExceedingLimit(){
+        ElasticSearchUtils.ANNOTATION_INDEX_MAX_RESULT_WINDOW = 1;
+        AnnotationQuery annotationQuery = new AnnotationQuery("scope1", "metric1", null, "unittest", 1557809359073L, 1557809599073L);
+        List<AnnotationQuery> queries = new ArrayList<>();
+        queries.add(annotationQuery);
+        ElasticSearchAnnotationService spyService = _initializeSpyService(esAnnotationService, getReply, getReply);
+        spyService.getAnnotations(queries);
+    }
+
     @Test
     public void testDoExtractResponse() throws Exception {
         final String message = "this is a test";
@@ -295,6 +334,13 @@ public void testHashedSearchIdentifierAnnotationAndAnnotationQueryMatch(){
         Annotation annotation = new Annotation("source", "16ab4b56311", "transactionId", "scope1", "metric1", 1557809559073L);
         annotation.setTags(annotationQueryTags);
         assertEquals(AnnotationRecordList.getHashedSearchIdentifier(annotation), ElasticSearchAnnotationService.getHashedSearchIdentifier(annotationQuery));
+
+        // change order of tags in annotation query
+        Map<String, String> annotationQueryTags2 = new HashMap<>();
+        annotationQueryTags2.put("podName","pod1");
+        annotationQueryTags2.put("device","device1");
+        AnnotationQuery annotationQuery2 = new AnnotationQuery("scope1", "metric1", annotationQueryTags2, "transactionId", 1557809359073L, 1557809599073L);
+        assertEquals(AnnotationRecordList.getHashedSearchIdentifier(annotation), ElasticSearchAnnotationService.getHashedSearchIdentifier(annotationQuery2));
     }
 
     private ElasticSearchAnnotationService _initializeSpyService(ElasticSearchAnnotationService service,