From 2a1cbb89c43892b91175874bb9e56a4535a3f844 Mon Sep 17 00:00:00 2001 From: Babacar Diasse Date: Tue, 11 Jun 2024 23:29:16 +0200 Subject: [PATCH] feat: Adding support for date_nanos to Anomaly Detection Signed-off-by: Babacar Diasse --- .../timeseries/constant/CommonName.java | 1 + .../AbstractTimeSeriesActionHandler.java | 2 +- .../org/opensearch/ad/ADIntegTestCase.java | 55 +++++++++++++++---- .../opensearch/ad/e2e/RuleModelPerfIT.java | 32 ++++++++++- .../ad/rest/AnomalyDetectorRestApiIT.java | 36 +++++++++++- ...teAnomalyDetectorTransportActionTests.java | 20 +++++++ .../opensearch/timeseries/TestHelpers.java | 11 +++- 7 files changed, 141 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/opensearch/timeseries/constant/CommonName.java b/src/main/java/org/opensearch/timeseries/constant/CommonName.java index 7a6b451f0..447c983a3 100644 --- a/src/main/java/org/opensearch/timeseries/constant/CommonName.java +++ b/src/main/java/org/opensearch/timeseries/constant/CommonName.java @@ -28,6 +28,7 @@ public class CommonName { public static final String KEYWORD_TYPE = "keyword"; public static final String IP_TYPE = "ip"; public static final String DATE_TYPE = "date"; + public static final String DATE_NANOS_TYPE = "date_nanos"; // ====================================== // Index name diff --git a/src/main/java/org/opensearch/timeseries/rest/handler/AbstractTimeSeriesActionHandler.java b/src/main/java/org/opensearch/timeseries/rest/handler/AbstractTimeSeriesActionHandler.java index 1adafb16a..d37c60480 100644 --- a/src/main/java/org/opensearch/timeseries/rest/handler/AbstractTimeSeriesActionHandler.java +++ b/src/main/java/org/opensearch/timeseries/rest/handler/AbstractTimeSeriesActionHandler.java @@ -326,7 +326,7 @@ protected void validateTimeField(boolean indexingDryRun, ActionListener liste foundField = true; Map metadataMap = (Map) type; String typeName = (String) metadataMap.get(CommonName.TYPE); - if (!typeName.equals(CommonName.DATE_TYPE)) { + if (!typeName.equals(CommonName.DATE_TYPE) && !typeName.equals(CommonName.DATE_NANOS_TYPE)) { listener .onFailure( new ValidationException( diff --git a/src/test/java/org/opensearch/ad/ADIntegTestCase.java b/src/test/java/org/opensearch/ad/ADIntegTestCase.java index 2bc69f27e..5bc7134d6 100644 --- a/src/test/java/org/opensearch/ad/ADIntegTestCase.java +++ b/src/test/java/org/opensearch/ad/ADIntegTestCase.java @@ -146,19 +146,29 @@ public void createDetectionStateIndex() throws IOException { createIndex(ADCommonName.DETECTION_STATE_INDEX, ADIndexManagement.getStateMappings()); } - public void createTestDataIndex(String indexName) { - String mappings = "{\"properties\":{\"" - + timeField - + "\":{\"type\":\"date\",\"format\":\"strict_date_time||epoch_millis\"}," - + "\"value\":{\"type\":\"double\"}, \"" - + categoryField - + "\":{\"type\":\"keyword\"},\"" - + ipField - + "\":{\"type\":\"ip\"}," - + "\"is_error\":{\"type\":\"boolean\"}, \"message\":{\"type\":\"text\"}}}"; + public void createTestDataIndex(String indexName, boolean useDateNanos) { + StringBuilder mappingsBuilder = new StringBuilder("{\"properties\":{\"").append(timeField); + if (useDateNanos) { + mappingsBuilder.append("\":{\"type\":\"date_nanos\",\"format\":\"strict_date_time||epoch_millis\"},"); + } else { + mappingsBuilder.append("\":{\"type\":\"date\",\"format\":\"strict_date_time||epoch_millis\"},"); + } + mappingsBuilder + .append("\"value\":{\"type\":\"double\"}, \"") + .append(categoryField) + .append("\":{\"type\":\"keyword\"},\"") + .append(ipField) + .append("\":{\"type\":\"ip\"},") + .append("\"is_error\":{\"type\":\"boolean\"}, \"message\":{\"type\":\"text\"}}}"); + + String mappings = mappingsBuilder.toString(); createIndex(indexName, mappings); } + public void createTestDataIndex(String indexName) { + createTestDataIndex(indexName, false); + } + public void createIndex(String indexName, String mappings) { CreateIndexResponse createIndexResponse = TestHelpers.createIndex(admin(), indexName, mappings); assertEquals(true, createIndexResponse.isAcknowledged()); @@ -283,8 +293,25 @@ public void ingestTestDataValidate(String testIndex, Instant startTime, int dete ingestTestDataValidate(testIndex, startTime, detectionIntervalInMinutes, type, DEFAULT_TEST_DATA_DOCS); } - public void ingestTestDataValidate(String testIndex, Instant startTime, int detectionIntervalInMinutes, String type, int totalDocs) { - createTestDataIndex(testIndex); + public void ingestTestDataValidate( + String testIndex, + Instant startTime, + int detectionIntervalInMinutes, + String type, + boolean useDateNanos + ) { + ingestTestDataValidate(testIndex, startTime, detectionIntervalInMinutes, type, DEFAULT_TEST_DATA_DOCS, useDateNanos); + } + + public void ingestTestDataValidate( + String testIndex, + Instant startTime, + int detectionIntervalInMinutes, + String type, + int totalDocs, + boolean useDateNanos + ) { + createTestDataIndex(testIndex, useDateNanos); List> docs = new ArrayList<>(); Instant currentInterval = Instant.from(startTime); @@ -315,6 +342,10 @@ public void ingestTestDataValidate(String testIndex, Instant startTime, int dete assertEquals(totalDocs, count); } + public void ingestTestDataValidate(String testIndex, Instant startTime, int detectionIntervalInMinutes, String type, int totalDocs) { + ingestTestDataValidate(testIndex, startTime, detectionIntervalInMinutes, type, totalDocs, false); + } + public Feature maxValueFeature() throws IOException { return maxValueFeature(nameField, valueField, nameField); } diff --git a/src/test/java/org/opensearch/ad/e2e/RuleModelPerfIT.java b/src/test/java/org/opensearch/ad/e2e/RuleModelPerfIT.java index f9cb1d018..b8e8cff90 100644 --- a/src/test/java/org/opensearch/ad/e2e/RuleModelPerfIT.java +++ b/src/test/java/org/opensearch/ad/e2e/RuleModelPerfIT.java @@ -50,6 +50,21 @@ public void testRule() throws Exception { } } + public void testRuleWithDateNanos() throws Exception { + // TODO: this test case will run for a much longer time and timeout with security enabled + if (!isHttps()) { + disableResourceNotFoundFaultTolerence(); + // there are 8 entities in the data set. Each one needs 1500 rows as training data. + Map minPrecision = new HashMap<>(); + minPrecision.put("Phoenix", 0.5); + minPrecision.put("Scottsdale", 0.5); + Map minRecall = new HashMap<>(); + minRecall.put("Phoenix", 0.9); + minRecall.put("Scottsdale", 0.6); + verifyRule("rule", 10, minPrecision.size(), 1500, minPrecision, minRecall, 20, true); + } + } + private void verifyTestResults( Triple, Integer, Map>> testResults, Map>> anomalies, @@ -115,6 +130,19 @@ public void verifyRule( Map minPrecision, Map minRecall, int maxError + ) throws Exception { + verifyRule(datasetName, intervalMinutes, numberOfEntities, trainTestSplit, minPrecision, minRecall, maxError, false); + } + + public void verifyRule( + String datasetName, + int intervalMinutes, + int numberOfEntities, + int trainTestSplit, + Map minPrecision, + Map minRecall, + int maxError, + boolean useDateNanos ) throws Exception { String dataFileName = String.format(Locale.ROOT, "data/%s.data", datasetName); String labelFileName = String.format(Locale.ROOT, "data/%s.label", datasetName); @@ -127,7 +155,9 @@ public void verifyRule( String mapping = String .format( Locale.ROOT, - "{ \"mappings\": { \"properties\": { \"timestamp\": { \"type\": \"date\"}," + "{ \"mappings\": { \"properties\": { \"timestamp\": { \"type\":" + + (useDateNanos ? "\"date_nanos\"" : "\"date\"") + + "}," + " \"transform._doc_count\": { \"type\": \"integer\" }," + "\"%s\": { \"type\": \"keyword\"} } } }", categoricalField diff --git a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java index 9acd49fd4..ea3d448b0 100644 --- a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java @@ -117,7 +117,12 @@ private AnomalyDetector createIndexAndGetAnomalyDetector(String indexName) throw } private AnomalyDetector createIndexAndGetAnomalyDetector(String indexName, List features) throws IOException { - TestHelpers.createIndexWithTimeField(client(), indexName, TIME_FIELD); + return createIndexAndGetAnomalyDetector(indexName, features, false); + } + + private AnomalyDetector createIndexAndGetAnomalyDetector(String indexName, List features, boolean useDateNanos) + throws IOException { + TestHelpers.createIndexWithTimeField(client(), indexName, TIME_FIELD, useDateNanos); String testIndexData = "{\"keyword-field\": \"field-1\", \"ip-field\": \"1.2.3.4\", \"timestamp\": 1}"; TestHelpers.ingestDataToIndex(client(), indexName, TestHelpers.toHttpEntity(testIndexData)); AnomalyDetector detector = TestHelpers.randomAnomalyDetector(TIME_FIELD, indexName, features); @@ -200,6 +205,35 @@ public void testCreateAnomalyDetector() throws Exception { assertTrue("incorrect version", version > 0); } + public void testCreateAnomalyDetectorWithDateNanos() throws Exception { + AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME, ImmutableList.of(TestHelpers.randomFeature(true)), true); + updateClusterSettings(ADEnabledSetting.AD_ENABLED, false); + + Exception ex = expectThrows( + ResponseException.class, + () -> TestHelpers + .makeRequest( + client(), + "POST", + TestHelpers.AD_BASE_DETECTORS_URI, + ImmutableMap.of(), + TestHelpers.toHttpEntity(detector), + null + ) + ); + assertThat(ex.getMessage(), containsString(ADCommonMessages.DISABLED_ERR_MSG)); + + updateClusterSettings(ADEnabledSetting.AD_ENABLED, true); + Response response = TestHelpers + .makeRequest(client(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), TestHelpers.toHttpEntity(detector), null); + assertEquals("Create anomaly detector failed", RestStatus.CREATED, TestHelpers.restStatus(response)); + Map responseMap = entityAsMap(response); + String id = (String) responseMap.get("_id"); + int version = (int) responseMap.get("_version"); + assertNotEquals("response is missing Id", AnomalyDetector.NO_ID, id); + assertTrue("incorrect version", version > 0); + } + public void testUpdateAnomalyDetectorCategoryField() throws Exception { AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME); Response response = TestHelpers diff --git a/src/test/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportActionTests.java b/src/test/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportActionTests.java index bd399b433..d31190b9c 100644 --- a/src/test/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportActionTests.java +++ b/src/test/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportActionTests.java @@ -514,4 +514,24 @@ public void testValidateAnomalyDetectorWithNonDateTimeField() throws IOException response.getIssue().getMessage() ); } + + @Test + public void testValidateAnomalyDetectorWithDateNanosWithoutIssue() throws IOException { + AnomalyDetector anomalyDetector = TestHelpers + .randomAnomalyDetector(timeField, "test-index", ImmutableList.of(sumValueFeature(nameField, ipField + ".is_error", "test-2"))); + ingestTestDataValidate(anomalyDetector.getIndices().get(0), Instant.now().minus(1, ChronoUnit.DAYS), 1, "error", true); + ValidateConfigRequest request = new ValidateConfigRequest( + AnalysisType.AD, + anomalyDetector, + ValidationAspect.DETECTOR.getName(), + 5, + 5, + 5, + new TimeValue(5_000L), + 10 + ); + ValidateConfigResponse response = client().execute(ValidateAnomalyDetectorAction.INSTANCE, request).actionGet(5_000); + assertNull(response.getIssue()); + } + } diff --git a/src/test/java/org/opensearch/timeseries/TestHelpers.java b/src/test/java/org/opensearch/timeseries/TestHelpers.java index 7ce60a313..0745c3cd4 100644 --- a/src/test/java/org/opensearch/timeseries/TestHelpers.java +++ b/src/test/java/org/opensearch/timeseries/TestHelpers.java @@ -1167,9 +1167,18 @@ public static void createIndex(RestClient client, String indexName, HttpEntity d } public static void createIndexWithTimeField(RestClient client, String indexName, String timeField) throws IOException { + createIndexWithTimeField(client, indexName, timeField, false); + } + + public static void createIndexWithTimeField(RestClient client, String indexName, String timeField, boolean useDateNanos) + throws IOException { StringBuilder indexMappings = new StringBuilder(); indexMappings.append("{\"properties\":{"); - indexMappings.append("\"" + timeField + "\":{\"type\":\"date\"}"); + if (useDateNanos) { + indexMappings.append("\"" + timeField + "\":{\"type\":\"date_nanos\"}"); + } else { + indexMappings.append("\"" + timeField + "\":{\"type\":\"date\"}"); + } indexMappings.append("}}"); createIndex(client, indexName.toLowerCase(Locale.ROOT), TestHelpers.toHttpEntity("{\"name\": \"test\"}")); createIndexMapping(client, indexName.toLowerCase(Locale.ROOT), TestHelpers.toHttpEntity(indexMappings.toString()));