diff --git a/src/main/java/org/opensearch/ad/model/ADTask.java b/src/main/java/org/opensearch/ad/model/ADTask.java index 96061219a..3070fd8e1 100644 --- a/src/main/java/org/opensearch/ad/model/ADTask.java +++ b/src/main/java/org/opensearch/ad/model/ADTask.java @@ -341,8 +341,10 @@ public static ADTask parse(XContentParser parser, String taskId) throws IOExcept detector.getRecencyEmphasis(), detector.getSeasonIntervals(), detector.getHistoryIntervals(), - detector.getRules() - + detector.getRules(), + detector.getCustomResultIndexMinSize(), + detector.getCustomResultIndexMinAge(), + detector.getCustomResultIndexTTL() ); return new Builder() .taskId(parsedTaskId) diff --git a/src/main/java/org/opensearch/ad/model/AnomalyDetector.java b/src/main/java/org/opensearch/ad/model/AnomalyDetector.java index 1f3094e08..cab54f30a 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyDetector.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyDetector.java @@ -147,6 +147,9 @@ public Integer getShingleSize(Integer customShingleSize) { * @param seasonIntervals seasonality in terms of intervals * @param historyIntervals history intervals we look back during cold start * @param rules custom rules to filter out AD results + * @param customResultIndexMinSize custom result index lifecycle management min size condition + * @param customResultIndexMinAge custom result index lifecycle management min age condition + * @param customResultIndexTTL custom result index lifecycle management ttl */ public AnomalyDetector( String detectorId, @@ -170,7 +173,10 @@ public AnomalyDetector( Integer recencyEmphasis, Integer seasonIntervals, Integer historyIntervals, - List rules + List rules, + Integer customResultIndexMinSize, + Integer customResultIndexMinAge, + Integer customResultIndexTTL ) { super( detectorId, @@ -194,7 +200,10 @@ public AnomalyDetector( recencyEmphasis, seasonIntervals, new ADShingleGetter(seasonIntervals), - historyIntervals + historyIntervals, + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); checkAndThrowValidationErrors(ValidationAspect.DETECTOR); @@ -268,6 +277,9 @@ public AnomalyDetector(StreamInput input) throws IOException { if (input.readBoolean()) { this.rules = input.readList(Rule::new); } + this.customResultIndexMinSize = input.readOptionalInt(); + this.customResultIndexMinAge = input.readOptionalInt(); + this.customResultIndexTTL = input.readOptionalInt(); } public XContentBuilder toXContent(XContentBuilder builder) throws IOException { @@ -275,7 +287,7 @@ public XContentBuilder toXContent(XContentBuilder builder) throws IOException { } /* - * For backward compatiblity reason, we cannot use super class + * For backward compatibility reason, we cannot use super class * Config's writeTo as we have detectionDateRange and * detectorType that Config does not have. */ @@ -330,6 +342,9 @@ public void writeTo(StreamOutput output) throws IOException { } else { output.writeBoolean(false); } + output.writeOptionalInt(customResultIndexMinSize); + output.writeOptionalInt(customResultIndexMinAge); + output.writeOptionalInt(customResultIndexTTL); } @Override @@ -422,8 +437,10 @@ public static AnomalyDetector parse( Integer recencyEmphasis = null; Integer seasonality = null; Integer historyIntervals = null; - List rules = new ArrayList<>(); + Integer customResultIndexMinSize = null; + Integer customResultIndexMinAge = null; + Integer customResultIndexTTL = null; ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { @@ -549,6 +566,15 @@ public static AnomalyDetector parse( rules.add(Rule.parse(parser)); } break; + case RESULT_INDEX_FIELD_MIN_SIZE: + customResultIndexMinSize = parser.intValue(); + break; + case RESULT_INDEX_FIELD_MIN_AGE: + customResultIndexMinAge = parser.intValue(); + break; + case RESULT_INDEX_FIELD_TTL: + customResultIndexTTL = parser.intValue(); + break; default: parser.skipChildren(); break; @@ -576,7 +602,10 @@ public static AnomalyDetector parse( recencyEmphasis, seasonality, historyIntervals, - rules + rules, + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); detector.setDetectionDateRange(detectionDateRange); return detector; diff --git a/src/main/java/org/opensearch/ad/rest/handler/AbstractAnomalyDetectorActionHandler.java b/src/main/java/org/opensearch/ad/rest/handler/AbstractAnomalyDetectorActionHandler.java index 41b69276e..e605b1d8b 100644 --- a/src/main/java/org/opensearch/ad/rest/handler/AbstractAnomalyDetectorActionHandler.java +++ b/src/main/java/org/opensearch/ad/rest/handler/AbstractAnomalyDetectorActionHandler.java @@ -236,7 +236,10 @@ protected AnomalyDetector copyConfig(User user, Config config) { config.getRecencyEmphasis(), config.getSeasonIntervals(), config.getHistoryIntervals(), - detector.getRules() + detector.getRules(), + config.getCustomResultIndexMinSize(), + config.getCustomResultIndexMinAge(), + config.getCustomResultIndexTTL() ); } diff --git a/src/main/java/org/opensearch/forecast/model/ForecastTask.java b/src/main/java/org/opensearch/forecast/model/ForecastTask.java index 4325e56b5..61bf826c5 100644 --- a/src/main/java/org/opensearch/forecast/model/ForecastTask.java +++ b/src/main/java/org/opensearch/forecast/model/ForecastTask.java @@ -339,7 +339,10 @@ public static ForecastTask parse(XContentParser parser, String taskId) throws IO forecaster.getImputationOption(), forecaster.getRecencyEmphasis(), forecaster.getSeasonIntervals(), - forecaster.getHistoryIntervals() + forecaster.getHistoryIntervals(), + forecaster.getCustomResultIndexMinSize(), + forecaster.getCustomResultIndexMinAge(), + forecaster.getCustomResultIndexTTL() ); return new Builder() .taskId(parsedTaskId) diff --git a/src/main/java/org/opensearch/forecast/model/Forecaster.java b/src/main/java/org/opensearch/forecast/model/Forecaster.java index 7ffeb2e4f..8feb24795 100644 --- a/src/main/java/org/opensearch/forecast/model/Forecaster.java +++ b/src/main/java/org/opensearch/forecast/model/Forecaster.java @@ -131,7 +131,10 @@ public Forecaster( ImputationOption imputationOption, Integer recencyEmphasis, Integer seasonIntervals, - Integer historyIntervals + Integer historyIntervals, + Integer customResultIndexMinSize, + Integer customResultIndexMinAge, + Integer customResultIndexTTL ) { super( forecasterId, @@ -155,7 +158,10 @@ public Forecaster( recencyEmphasis, seasonIntervals, new ForecastShingleGetter(seasonIntervals, horizon), - historyIntervals + historyIntervals, + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); checkAndThrowValidationErrors(ValidationAspect.FORECASTER); @@ -294,6 +300,9 @@ public static Forecaster parse( Integer recencyEmphasis = null; Integer seasonality = null; Integer historyIntervals = null; + Integer customResultIndexMinSize = null; + Integer customResultIndexMinAge = null; + Integer customResultIndexTTL = null; ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { @@ -413,6 +422,15 @@ public static Forecaster parse( case HISTORY_INTERVAL_FIELD: historyIntervals = parser.intValue(); break; + case RESULT_INDEX_FIELD_MIN_SIZE: + customResultIndexMinSize = parser.intValue(); + break; + case RESULT_INDEX_FIELD_MIN_AGE: + customResultIndexMinAge = parser.intValue(); + break; + case RESULT_INDEX_FIELD_TTL: + customResultIndexTTL = parser.intValue(); + break; default: parser.skipChildren(); break; @@ -440,7 +458,10 @@ public static Forecaster parse( imputationOption, recencyEmphasis, seasonality, - historyIntervals + historyIntervals, + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); return forecaster; } diff --git a/src/main/java/org/opensearch/forecast/rest/handler/AbstractForecasterActionHandler.java b/src/main/java/org/opensearch/forecast/rest/handler/AbstractForecasterActionHandler.java index b18602222..dafc7123b 100644 --- a/src/main/java/org/opensearch/forecast/rest/handler/AbstractForecasterActionHandler.java +++ b/src/main/java/org/opensearch/forecast/rest/handler/AbstractForecasterActionHandler.java @@ -254,7 +254,10 @@ protected Config copyConfig(User user, Config config) { config.getImputationOption(), config.getRecencyEmphasis(), config.getSeasonIntervals(), - config.getHistoryIntervals() + config.getHistoryIntervals(), + config.getCustomResultIndexMinSize(), + config.getCustomResultIndexMinAge(), + config.getCustomResultIndexTTL() ); } diff --git a/src/main/java/org/opensearch/timeseries/model/Config.java b/src/main/java/org/opensearch/timeseries/model/Config.java index 46ac13d01..8e6009c60 100644 --- a/src/main/java/org/opensearch/timeseries/model/Config.java +++ b/src/main/java/org/opensearch/timeseries/model/Config.java @@ -75,6 +75,9 @@ public abstract class Config implements Writeable, ToXContentObject { public static final String SEASONALITY_FIELD = "suggested_seasonality"; public static final String RECENCY_EMPHASIS_FIELD = "recency_emphasis"; public static final String HISTORY_INTERVAL_FIELD = "history"; + public static final String RESULT_INDEX_FIELD_MIN_SIZE = "result_index_min_size"; + public static final String RESULT_INDEX_FIELD_MIN_AGE = "result_index_min_age"; + public static final String RESULT_INDEX_FIELD_TTL = "result_index_ttl"; protected String id; protected Long version; @@ -111,6 +114,9 @@ public abstract class Config implements Writeable, ToXContentObject { protected Integer seasonIntervals; protected Integer historyIntervals; + protected Integer customResultIndexMinSize; + protected Integer customResultIndexMinAge; + protected Integer customResultIndexTTL; public static String INVALID_RESULT_INDEX_NAME_SIZE = "Result index name size must contains less than " + MAX_RESULT_INDEX_NAME_SIZE @@ -138,7 +144,10 @@ protected Config( Integer recencyEmphasis, Integer seasonIntervals, ShingleGetter shingleGetter, - Integer historyIntervals + Integer historyIntervals, + Integer customResultIndexMinSize, + Integer customResultIndexMinAge, + Integer customResultIndexTTL ) { if (Strings.isBlank(name)) { errorMessage = CommonMessages.EMPTY_NAME; @@ -254,6 +263,9 @@ protected Config( this.recencyEmphasis = Optional.ofNullable(recencyEmphasis).orElse(TimeSeriesSettings.DEFAULT_RECENCY_EMPHASIS); this.seasonIntervals = seasonIntervals; this.historyIntervals = historyIntervals == null ? suggestHistory() : historyIntervals; + this.customResultIndexMinSize = Strings.trimToNull(resultIndex) == null ? null : customResultIndexMinSize; + this.customResultIndexMinAge = Strings.trimToNull(resultIndex) == null ? null : customResultIndexMinAge; + this.customResultIndexTTL = Strings.trimToNull(resultIndex) == null ? null : customResultIndexTTL; } public int suggestHistory() { @@ -294,6 +306,9 @@ public Config(StreamInput input) throws IOException { this.recencyEmphasis = input.readInt(); this.seasonIntervals = input.readInt(); this.historyIntervals = input.readInt(); + this.customResultIndexMinSize = input.readOptionalInt(); + this.customResultIndexMinAge = input.readOptionalInt(); + this.customResultIndexTTL = input.readOptionalInt(); } /* @@ -343,6 +358,9 @@ public void writeTo(StreamOutput output) throws IOException { output.writeInt(recencyEmphasis); output.writeInt(seasonIntervals); output.writeInt(historyIntervals); + output.writeOptionalInt(customResultIndexMinSize); + output.writeOptionalInt(customResultIndexMinAge); + output.writeOptionalInt(customResultIndexTTL); } public boolean invalidShingleSizeRange(Integer shingleSizeToTest) { @@ -396,7 +414,10 @@ public boolean equals(Object o) { && Objects.equal(imputationOption, config.imputationOption) && Objects.equal(recencyEmphasis, config.recencyEmphasis) && Objects.equal(seasonIntervals, config.seasonIntervals) - && Objects.equal(historyIntervals, config.historyIntervals); + && Objects.equal(historyIntervals, config.historyIntervals) + && Objects.equal(customResultIndexMinSize, config.customResultIndexMinSize) + && Objects.equal(customResultIndexMinAge, config.customResultIndexMinAge) + && Objects.equal(customResultIndexTTL, config.customResultIndexTTL); } @Generated @@ -420,7 +441,10 @@ public int hashCode() { imputationOption, recencyEmphasis, seasonIntervals, - historyIntervals + historyIntervals, + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); } @@ -460,6 +484,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (seasonIntervals != null) { builder.field(SEASONALITY_FIELD, seasonIntervals); } + if (customResultIndexMinSize != null) { + builder.field(RESULT_INDEX_FIELD_MIN_SIZE, customResultIndexMinSize); + } + if (customResultIndexMinAge != null) { + builder.field(RESULT_INDEX_FIELD_MIN_AGE, customResultIndexMinAge); + } + if (customResultIndexTTL != null) { + builder.field(RESULT_INDEX_FIELD_TTL, customResultIndexTTL); + } return builder; } @@ -648,6 +681,18 @@ public Integer getHistoryIntervals() { return historyIntervals; } + public Integer getCustomResultIndexMinSize() { + return customResultIndexMinSize; + } + + public Integer getCustomResultIndexMinAge() { + return customResultIndexMinAge; + } + + public Integer getCustomResultIndexTTL() { + return customResultIndexTTL; + } + /** * Identifies redundant feature names. * @@ -699,6 +744,9 @@ public String toString() { .append("recencyEmphasis", recencyEmphasis) .append("seasonIntervals", seasonIntervals) .append("historyIntervals", historyIntervals) + .append("customResultIndexMinSize", customResultIndexMinSize) + .append("customResultIndexMinAge", customResultIndexMinAge) + .append("customResultIndexTTL", customResultIndexTTL) .toString(); } } diff --git a/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java b/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java index 98528cbae..3a45f8097 100644 --- a/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java +++ b/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java @@ -310,7 +310,10 @@ public ToXContentObject[] getConfig(String detectorId, BasicHeader header, boole detector.getRecencyEmphasis(), detector.getSeasonIntervals(), detector.getHistoryIntervals(), - null + null, + detector.getCustomResultIndexMinSize(), + detector.getCustomResultIndexMinAge(), + detector.getCustomResultIndexTTL() ), detectorJob, historicalAdTask, @@ -647,7 +650,10 @@ protected AnomalyDetector cloneDetector(AnomalyDetector anomalyDetector, String anomalyDetector.getRecencyEmphasis(), anomalyDetector.getSeasonIntervals(), anomalyDetector.getHistoryIntervals(), - null + null, + anomalyDetector.getCustomResultIndexMinSize(), + anomalyDetector.getCustomResultIndexMinAge(), + anomalyDetector.getCustomResultIndexTTL() ); return detector; } diff --git a/src/test/java/org/opensearch/ad/model/AnomalyDetectorTests.java b/src/test/java/org/opensearch/ad/model/AnomalyDetectorTests.java index a01d2f21b..ebd3fecaf 100644 --- a/src/test/java/org/opensearch/ad/model/AnomalyDetectorTests.java +++ b/src/test/java/org/opensearch/ad/model/AnomalyDetectorTests.java @@ -322,6 +322,9 @@ public void testInvalidShingleSize() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -354,6 +357,9 @@ public void testNullDetectorName() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -386,6 +392,9 @@ public void testBlankDetectorName() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -418,6 +427,9 @@ public void testNullTimeField() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -450,6 +462,9 @@ public void testNullIndices() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -482,6 +497,9 @@ public void testEmptyIndices() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -514,6 +532,9 @@ public void testNullDetectionInterval() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -545,6 +566,9 @@ public void testInvalidRecency() { -1, randomIntBetween(1, 256), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -577,6 +601,9 @@ public void testInvalidDetectionInterval() { null, // emphasis is not customized randomIntBetween(1, 256), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -609,6 +636,9 @@ public void testInvalidWindowDelay() { null, // emphasis is not customized randomIntBetween(1, 256), randomIntBetween(1, 1000), + null, + null, + null, null ) ); @@ -654,6 +684,9 @@ public void testGetShingleSize() throws IOException { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); assertEquals((int) anomalyDetector.getShingleSize(), 5); @@ -684,6 +717,9 @@ public void testGetShingleSizeReturnsDefaultValue() throws IOException { randomIntBetween(1, 10000), seasonalityIntervals, randomIntBetween(1, 1000), + null, + null, + null, null ); // seasonalityIntervals is not null and custom shingle size is null, use seasonalityIntervals to deterine shingle size @@ -711,6 +747,9 @@ public void testGetShingleSizeReturnsDefaultValue() throws IOException { null, null, randomIntBetween(1, 1000), + null, + null, + null, null ); // seasonalityIntervals is null and custom shingle size is null, use default shingle size @@ -740,6 +779,9 @@ public void testNullFeatureAttributes() throws IOException { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); assertNotNull(anomalyDetector.getFeatureAttributes()); @@ -769,6 +811,9 @@ public void testValidateResultIndex() throws IOException { randomIntBetween(1, 10000), randomIntBetween(1, TimeSeriesSettings.MAX_SHINGLE_SIZE * TimeSeriesSettings.SEASONALITY_TO_SHINGLE_RATIO), randomIntBetween(1, 1000), + null, + null, + null, null ); String errorMessage = anomalyDetector.validateCustomResultIndex("abc"); @@ -799,4 +844,61 @@ public void testParseAnomalyDetectorWithNoDescription() throws IOException { AnomalyDetector parsedDetector = AnomalyDetector.parse(TestHelpers.parser(detectorString), "id", 1L, null, null); assertEquals(parsedDetector.getDescription(), ""); } + + public void testParseAnomalyDetector_withCustomIndex_withCustomResultIndexMinSize() throws IOException { + String detectorString = "{\"name\":\"AhtYYGWTgqkzairTchcs\",\"description\":\"iIiAVPMyFgnFlEniLbMyfJxyoGvJAl\"," + + "\"time_field\":\"HmdFH\",\"indices\":[\"ffsBF\"],\"filter_query\":{\"bool\":{\"filter\":[{\"exists\":" + + "{\"field\":\"value\",\"boost\":1}}],\"adjust_pure_negative\":true,\"boost\":1}},\"window_delay\":" + + "{\"period\":{\"interval\":2,\"unit\":\"Minutes\"}},\"shingle_size\":8,\"schema_version\":-512063255," + + "\"feature_attributes\":[{\"feature_id\":\"OTYJs\",\"feature_name\":\"eYYCM\",\"feature_enabled\":false," + + "\"aggregation_query\":{\"XzewX\":{\"value_count\":{\"field\":\"ok\"}}}}],\"recency_emphasis\":3342," + + "\"history\":62,\"last_update_time\":1717192049845,\"category_field\":[\"Tcqcb\"],\"result_index\":" + + "\"opensearch-ad-plugin-result-test\",\"imputation_option\":{\"method\":\"FIXED_VALUES\",\"defaultFill\"" + + ":[],\"integerSensitive\":false},\"suggested_seasonality\":64,\"detection_interval\":{\"period\":" + + "{\"interval\":5,\"unit\":\"Minutes\"}},\"detector_type\":\"MULTI_ENTITY\",\"rules\":[],\"result_index_min_size\":1500}"; + AnomalyDetector parsedDetector = AnomalyDetector.parse(TestHelpers.parser(detectorString), "id", 1L, null, null); + assertEquals(1500, (int) parsedDetector.getCustomResultIndexMinSize()); + } + + public void testParseAnomalyDetector_withCustomIndex_withNullCustomResultIndexMinSize() throws IOException { + String detectorString = "{\"name\":\"todagtCMkwpcaedpyYUM\",\"time_field\":\"dJRwh\",\"indices\":[\"eIrgWMqAED\"]," + + "\"feature_attributes\":[{\"feature_id\":\"lxYRN\",\"feature_name\":\"eqSeU\",\"feature_enabled\"" + + ":true,\"aggregation_query\":{\"aa\":{\"value_count\":{\"field\":\"ok\"}}}}],\"detection_interval\":" + + "{\"period\":{\"interval\":425,\"unit\":\"Minutes\"}},\"window_delay\":{\"period\":{\"interval\":973," + + "\"unit\":\"Minutes\"}},\"shingle_size\":4,\"schema_version\":-1203962153,\"ui_metadata\":{\"JbAaV\":{\"feature_id\":" + + "\"rIFjS\",\"feature_name\":\"QXCmS\",\"feature_enabled\":false,\"aggregation_query\":{\"aa\":" + + "{\"value_count\":{\"field\":\"ok\"}}}}},\"last_update_time\":1568396089028}"; + AnomalyDetector parsedDetector = AnomalyDetector.parse(TestHelpers.parser(detectorString), "id", 1L, null, null); + assertEquals(null, parsedDetector.getCustomResultIndexMinSize()); + } + + public void testParseAnomalyDetector_withCustomIndex_withCustomResultIndexMinAge() throws IOException { + String detectorString = "{\"name\":\"AhtYYGWTgqkzairTchcs\",\"description\":\"iIiAVPMyFgnFlEniLbMyfJxyoGvJAl\"," + + "\"time_field\":\"HmdFH\",\"indices\":[\"ffsBF\"],\"filter_query\":{\"bool\":{\"filter\":[{\"exists\":" + + "{\"field\":\"value\",\"boost\":1}}],\"adjust_pure_negative\":true,\"boost\":1}},\"window_delay\":" + + "{\"period\":{\"interval\":2,\"unit\":\"Minutes\"}},\"shingle_size\":8,\"schema_version\":-512063255," + + "\"feature_attributes\":[{\"feature_id\":\"OTYJs\",\"feature_name\":\"eYYCM\",\"feature_enabled\":false," + + "\"aggregation_query\":{\"XzewX\":{\"value_count\":{\"field\":\"ok\"}}}}],\"recency_emphasis\":3342," + + "\"history\":62,\"last_update_time\":1717192049845,\"category_field\":[\"Tcqcb\"],\"result_index\":" + + "\"opensearch-ad-plugin-result-test\",\"imputation_option\":{\"method\":\"FIXED_VALUES\",\"defaultFill\"" + + ":[],\"integerSensitive\":false},\"suggested_seasonality\":64,\"detection_interval\":{\"period\":" + + "{\"interval\":5,\"unit\":\"Minutes\"}},\"detector_type\":\"MULTI_ENTITY\",\"rules\":[],\"result_index_min_age\":7}"; + AnomalyDetector parsedDetector = AnomalyDetector.parse(TestHelpers.parser(detectorString), "id", 1L, null, null); + assertEquals(7, (int) parsedDetector.getCustomResultIndexMinAge()); + } + + public void testParseAnomalyDetector_withCustomIndex_withCustomResultIndexTTL() throws IOException { + String detectorString = "{\"name\":\"AhtYYGWTgqkzairTchcs\",\"description\":\"iIiAVPMyFgnFlEniLbMyfJxyoGvJAl\"," + + "\"time_field\":\"HmdFH\",\"indices\":[\"ffsBF\"],\"filter_query\":{\"bool\":{\"filter\":[{\"exists\":" + + "{\"field\":\"value\",\"boost\":1}}],\"adjust_pure_negative\":true,\"boost\":1}},\"window_delay\":" + + "{\"period\":{\"interval\":2,\"unit\":\"Minutes\"}},\"shingle_size\":8,\"schema_version\":-512063255," + + "\"feature_attributes\":[{\"feature_id\":\"OTYJs\",\"feature_name\":\"eYYCM\",\"feature_enabled\":false," + + "\"aggregation_query\":{\"XzewX\":{\"value_count\":{\"field\":\"ok\"}}}}],\"recency_emphasis\":3342," + + "\"history\":62,\"last_update_time\":1717192049845,\"category_field\":[\"Tcqcb\"],\"result_index\":" + + "\"opensearch-ad-plugin-result-test\",\"imputation_option\":{\"method\":\"FIXED_VALUES\",\"defaultFill\"" + + ":[],\"integerSensitive\":false},\"suggested_seasonality\":64,\"detection_interval\":{\"period\":" + + "{\"interval\":5,\"unit\":\"Minutes\"}},\"detector_type\":\"MULTI_ENTITY\",\"rules\":[],\"result_index_ttl\":30}"; + AnomalyDetector parsedDetector = AnomalyDetector.parse(TestHelpers.parser(detectorString), "id", 1L, null, null); + assertEquals(30, (int) parsedDetector.getCustomResultIndexTTL()); + } } diff --git a/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java b/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java index d7148fb2e..933436368 100644 --- a/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java +++ b/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java @@ -219,6 +219,9 @@ public static Response createAnomalyDetector( randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); diff --git a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java index b8e052f26..9acd49fd4 100644 --- a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java @@ -149,6 +149,9 @@ public void testCreateAnomalyDetectorWithDuplicateName() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); @@ -228,6 +231,9 @@ public void testUpdateAnomalyDetectorCategoryField() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); Exception ex = expectThrows( @@ -291,6 +297,9 @@ public void testUpdateAnomalyDetector() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); @@ -359,6 +368,9 @@ public void testUpdateAnomalyDetectorNameToExisting() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); @@ -404,6 +416,9 @@ public void testUpdateAnomalyDetectorNameToNew() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); @@ -455,6 +470,9 @@ public void testUpdateAnomalyDetectorWithNotExistingIndex() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); @@ -823,6 +841,9 @@ public void testUpdateAnomalyDetectorWithRunningAdJob() throws Exception { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); diff --git a/src/test/java/org/opensearch/ad/rest/HistoricalAnalysisRestApiIT.java b/src/test/java/org/opensearch/ad/rest/HistoricalAnalysisRestApiIT.java index ae8cc5b05..0cfeaa833 100644 --- a/src/test/java/org/opensearch/ad/rest/HistoricalAnalysisRestApiIT.java +++ b/src/test/java/org/opensearch/ad/rest/HistoricalAnalysisRestApiIT.java @@ -323,7 +323,10 @@ private AnomalyDetector randomAnomalyDetector(AnomalyDetector detector) { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), - null + null, + detector.getCustomResultIndexMinSize(), + detector.getCustomResultIndexMinAge(), + detector.getCustomResultIndexTTL() ); } diff --git a/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java b/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java index cf62c63c8..bc814565a 100644 --- a/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java +++ b/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java @@ -272,6 +272,9 @@ public void testUpdateApiFilterByEnabledForAdmin() throws IOException { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); // User client has admin all access, and has "opensearch" backend role so client should be able to update detector @@ -323,6 +326,9 @@ public void testUpdateApiFilterByEnabled() throws IOException { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); enableFilterBy(); diff --git a/src/test/java/org/opensearch/ad/transport/AnomalyResultTransportActionTests.java b/src/test/java/org/opensearch/ad/transport/AnomalyResultTransportActionTests.java index a5fb25c92..f8208215f 100644 --- a/src/test/java/org/opensearch/ad/transport/AnomalyResultTransportActionTests.java +++ b/src/test/java/org/opensearch/ad/transport/AnomalyResultTransportActionTests.java @@ -223,6 +223,9 @@ private AnomalyDetector randomDetector(List indices, List featu randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); } @@ -250,6 +253,9 @@ private AnomalyDetector randomHCDetector(List indices, List fea randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); } diff --git a/src/test/java/org/opensearch/ad/transport/ForwardADTaskRequestTests.java b/src/test/java/org/opensearch/ad/transport/ForwardADTaskRequestTests.java index 7a28bfa7c..6e93ef01a 100644 --- a/src/test/java/org/opensearch/ad/transport/ForwardADTaskRequestTests.java +++ b/src/test/java/org/opensearch/ad/transport/ForwardADTaskRequestTests.java @@ -82,6 +82,9 @@ public void testNullDetectorIdAndTaskAction() throws IOException { randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); ForwardADTaskRequest request = new ForwardADTaskRequest(detector, null, null, null, null, Version.V_2_1_0); diff --git a/src/test/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportActionTests.java b/src/test/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportActionTests.java index bfe7a192e..bd399b433 100644 --- a/src/test/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportActionTests.java +++ b/src/test/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportActionTests.java @@ -400,6 +400,9 @@ public void testValidateAnomalyDetectorWithInvalidDetectorName() throws IOExcept randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); ingestTestDataValidate(anomalyDetector.getIndices().get(0), Instant.now().minus(1, ChronoUnit.DAYS), 1, "error"); @@ -444,6 +447,9 @@ public void testValidateAnomalyDetectorWithDetectorNameTooLong() throws IOExcept randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); ingestTestDataValidate(anomalyDetector.getIndices().get(0), Instant.now().minus(1, ChronoUnit.DAYS), 1, "error"); diff --git a/src/test/java/org/opensearch/forecast/model/ForecasterTests.java b/src/test/java/org/opensearch/forecast/model/ForecasterTests.java index ea61305d6..5dd5927d5 100644 --- a/src/test/java/org/opensearch/forecast/model/ForecasterTests.java +++ b/src/test/java/org/opensearch/forecast/model/ForecasterTests.java @@ -56,6 +56,9 @@ public class ForecasterTests extends AbstractTimeSeriesTest { Integer horizon = 1; int recencyEmphasis = 20; int seasonality = 20; + Integer customResultIndexMinSize = null; + Integer customResultIndexMinAge = null; + Integer customResultIndexTTL = null; public void testForecasterConstructor() { ImputationOption imputationOption = TestHelpers.randomImputationOption(0); @@ -82,7 +85,10 @@ public void testForecasterConstructor() { imputationOption, recencyEmphasis, seasonality, - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); assertEquals(forecasterId, forecaster.getId()); @@ -132,7 +138,10 @@ public void testForecasterConstructorWithNullForecastInterval() { TestHelpers.randomImputationOption(0), recencyEmphasis, seasonality, - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); }); @@ -167,7 +176,10 @@ public void testNegativeInterval() { TestHelpers.randomImputationOption(0), recencyEmphasis, seasonality, - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); }); @@ -202,7 +214,10 @@ public void testMaxCategoryFieldsLimits() { TestHelpers.randomImputationOption(0), recencyEmphasis, seasonality, - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); }); @@ -237,7 +252,10 @@ public void testBlankName() { TestHelpers.randomImputationOption(0), recencyEmphasis, seasonality, - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); }); @@ -272,7 +290,10 @@ public void testInvalidCustomResultIndex() { TestHelpers.randomImputationOption(0), recencyEmphasis, seasonality, - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); }); @@ -306,7 +327,10 @@ public void testValidCustomResultIndex() { TestHelpers.randomImputationOption(0), recencyEmphasis, seasonality, - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); assertEquals(resultIndex, forecaster.getCustomResultIndex()); @@ -338,7 +362,10 @@ public void testInvalidHorizon() { TestHelpers.randomImputationOption(0), recencyEmphasis, seasonality, - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); }); @@ -409,4 +436,34 @@ public void testParseNullImpute() throws IOException { Forecaster parsedForecaster = Forecaster.parse(TestHelpers.parser(forecasterString)); assertEquals("Parsing forecaster doesn't work", forecaster, parsedForecaster); } + + public void testParseNullCustomResultIndex_nullCustomResultIndexConditions() throws IOException { + Forecaster forecaster = TestHelpers.ForecasterBuilder + .newInstance() + .setCustomResultIndex(null) + .setCustomResultIndexMinSize(null) + .setCustomResultIndexMinAge(null) + .setCustomResultIndexTTL(null) + .build(); + String forecasterString = TestHelpers + .xContentBuilderToString(forecaster.toXContent(TestHelpers.builder(), ToXContent.EMPTY_PARAMS)); + LOG.info(forecasterString); + Forecaster parsedForecaster = Forecaster.parse(TestHelpers.parser(forecasterString)); + assertEquals(forecaster, parsedForecaster); + } + + public void testValidCustomResultIndex_withIndexConditions() throws IOException { + Forecaster forecaster = TestHelpers.ForecasterBuilder + .newInstance() + .setCustomResultIndex(ForecastCommonName.CUSTOM_RESULT_INDEX_PREFIX + "test") + .setCustomResultIndexMinSize(2000) + .setCustomResultIndexMinAge(7) + .setCustomResultIndexTTL(30) + .build(); + String forecasterString = TestHelpers + .xContentBuilderToString(forecaster.toXContent(TestHelpers.builder(), ToXContent.EMPTY_PARAMS)); + LOG.info(forecasterString); + Forecaster parsedForecaster = Forecaster.parse(TestHelpers.parser(forecasterString)); + assertEquals(forecaster, parsedForecaster); + } } diff --git a/src/test/java/org/opensearch/timeseries/TestHelpers.java b/src/test/java/org/opensearch/timeseries/TestHelpers.java index e71d95b7e..7106ad3fb 100644 --- a/src/test/java/org/opensearch/timeseries/TestHelpers.java +++ b/src/test/java/org/opensearch/timeseries/TestHelpers.java @@ -333,7 +333,10 @@ public static AnomalyDetector randomAnomalyDetector( randomIntBetween(1, 10000), randomIntBetween(1, TimeSeriesSettings.MAX_SHINGLE_SIZE * 2), randomIntBetween(1, 1000), - new ArrayList() + new ArrayList(), + null, + null, + null ); } @@ -382,6 +385,9 @@ public static AnomalyDetector randomDetector( randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); } @@ -441,6 +447,9 @@ public static AnomalyDetector randomAnomalyDetectorUsingCategoryFields( randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); } @@ -476,6 +485,9 @@ public static AnomalyDetector randomAnomalyDetector(String timefield, String ind randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); } @@ -503,6 +515,9 @@ public static AnomalyDetector randomAnomalyDetectorWithEmptyFeature() throws IOE randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); } @@ -536,6 +551,9 @@ public static AnomalyDetector randomAnomalyDetectorWithInterval(TimeConfiguratio randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); } @@ -698,6 +716,9 @@ public AnomalyDetector build() { randomIntBetween(1, 10000), randomIntBetween(1, TimeSeriesSettings.MAX_SHINGLE_SIZE * 2), randomIntBetween(1, 1000), + null, + null, + null, null ); } @@ -728,6 +749,9 @@ public static AnomalyDetector randomAnomalyDetectorWithInterval(TimeConfiguratio randomIntBetween(1, 10000), randomInt(TimeSeriesSettings.MAX_SHINGLE_SIZE / 2), randomIntBetween(1, 1000), + null, + null, + null, null ); } @@ -1681,6 +1705,9 @@ public static class ForecasterBuilder { String resultIndex; Integer horizon; ImputationOption imputationOption; + Integer customResultIndexMinSize; + Integer customResultIndexMinAge; + Integer customResultIndexTTL; ForecasterBuilder() throws IOException { forecasterId = randomAlphaOfLength(10); @@ -1702,6 +1729,9 @@ public static class ForecasterBuilder { resultIndex = null; horizon = randomIntBetween(1, 20); imputationOption = randomImputationOption((int) features.stream().filter(Feature::getEnabled).count()); + customResultIndexMinSize = null; + customResultIndexMinAge = null; + customResultIndexTTL = null; } public static ForecasterBuilder newInstance() throws IOException { @@ -1793,6 +1823,21 @@ public ForecasterBuilder setCustomResultIndex(String resultIndex) { return this; } + public ForecasterBuilder setCustomResultIndexMinSize(Integer customResultIndexMinSize) { + this.customResultIndexMinSize = customResultIndexMinSize; + return this; + } + + public ForecasterBuilder setCustomResultIndexMinAge(Integer customResultIndexMinAge) { + this.customResultIndexMinAge = customResultIndexMinAge; + return this; + } + + public ForecasterBuilder setCustomResultIndexTTL(Integer customResultIndexTTL) { + this.customResultIndexTTL = customResultIndexTTL; + return this; + } + public ForecasterBuilder setNullImputationOption() { this.imputationOption = null; return this; @@ -1821,7 +1866,10 @@ public Forecaster build() { imputationOption, randomIntBetween(1, 1000), randomIntBetween(1, 128), - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + customResultIndexMinSize, + customResultIndexMinAge, + customResultIndexTTL ); } } @@ -1850,7 +1898,10 @@ public static Forecaster randomForecaster() throws IOException { randomImputationOption(feature.getEnabled() ? 1 : 0), randomIntBetween(1, 1000), randomIntBetween(1, 128), - randomIntBetween(1, 1000) + randomIntBetween(1, 1000), + null, + null, + null ); }