diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index 204b81914816a..951336e115431 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -46,6 +46,7 @@ public final class TransformField { public static final ParseField DEDUCE_MAPPINGS = new ParseField("deduce_mappings"); public static final ParseField NUM_FAILURE_RETRIES = new ParseField("num_failure_retries"); public static final ParseField UNATTENDED = new ParseField("unattended"); + public static final ParseField SKIP_DEST_INDEX_CREATION = new ParseField("skip_dest_index_creation"); public static final ParseField FIELD = new ParseField("field"); public static final ParseField SYNC = new ParseField("sync"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index 9b0fa3876819b..18ec02ff27452 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -31,6 +31,19 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; public class SettingsConfig implements Writeable, ToXContentObject { + + public static final SettingsConfig EMPTY = new SettingsConfig( + null, + null, + (Integer) null, + (Integer) null, + (Integer) null, + (Integer) null, + (Integer) null, + (Integer) null, + (Integer) null + ); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); @@ -44,6 +57,7 @@ public class SettingsConfig implements Writeable, ToXContentObject { private static final int DEFAULT_DEDUCE_MAPPINGS = -1; private static final int DEFAULT_NUM_FAILURE_RETRIES = -2; private static final int DEFAULT_UNATTENDED = -1; + private static final int DEFAULT_SKIP_DEST_INDEX_CREATION = -1; private static ConstructingObjectParser createParser(boolean lenient) { ConstructingObjectParser parser = new ConstructingObjectParser<>( @@ -57,7 +71,8 @@ private static ConstructingObjectParser createParser(boole (Integer) args[4], (Integer) args[5], (Integer) args[6], - (Integer) args[7] + (Integer) args[7], + (Integer) args[8] ) ); parser.declareIntOrNull(optionalConstructorArg(), DEFAULT_MAX_PAGE_SEARCH_SIZE, TransformField.MAX_PAGE_SEARCH_SIZE); @@ -98,6 +113,13 @@ private static ConstructingObjectParser createParser(boole TransformField.UNATTENDED, ValueType.BOOLEAN_OR_NULL ); + // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser + parser.declareField( + optionalConstructorArg(), + p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_SKIP_DEST_INDEX_CREATION : p.booleanValue() ? 1 : 0, + TransformField.SKIP_DEST_INDEX_CREATION, + ValueType.BOOLEAN_OR_NULL + ); return parser; } @@ -109,10 +131,7 @@ private static ConstructingObjectParser createParser(boole private final Integer deduceMappings; private final Integer numFailureRetries; private final Integer unattended; - - public SettingsConfig() { - this(null, null, (Integer) null, (Integer) null, (Integer) null, (Integer) null, (Integer) null, (Integer) null); - } + private final Integer skipDestIndexCreation; public SettingsConfig( Integer maxPageSearchSize, @@ -122,7 +141,8 @@ public SettingsConfig( Boolean usePit, Boolean deduceMappings, Integer numFailureRetries, - Boolean unattended + Boolean unattended, + Boolean skipDestIndexCreation ) { this( maxPageSearchSize, @@ -132,11 +152,12 @@ public SettingsConfig( usePit == null ? null : usePit ? 1 : 0, deduceMappings == null ? null : deduceMappings ? 1 : 0, numFailureRetries, - unattended == null ? null : unattended ? 1 : 0 + unattended == null ? null : unattended ? 1 : 0, + skipDestIndexCreation == null ? null : skipDestIndexCreation ? 1 : 0 ); } - SettingsConfig( + private SettingsConfig( Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis, @@ -144,7 +165,8 @@ public SettingsConfig( Integer usePit, Integer deduceMappings, Integer numFailureRetries, - Integer unattended + Integer unattended, + Integer skipDestIndexCreation ) { this.maxPageSearchSize = maxPageSearchSize; this.docsPerSecond = docsPerSecond; @@ -154,6 +176,7 @@ public SettingsConfig( this.deduceMappings = deduceMappings; this.numFailureRetries = numFailureRetries; this.unattended = unattended; + this.skipDestIndexCreation = skipDestIndexCreation; } public SettingsConfig(final StreamInput in) throws IOException { @@ -178,6 +201,11 @@ public SettingsConfig(final StreamInput in) throws IOException { } else { unattended = DEFAULT_UNATTENDED; } + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { // TODO + skipDestIndexCreation = in.readOptionalInt(); + } else { + skipDestIndexCreation = DEFAULT_SKIP_DEST_INDEX_CREATION; + } } public Integer getMaxPageSearchSize() { @@ -192,7 +220,7 @@ public Boolean getDatesAsEpochMillis() { return datesAsEpochMillis != null ? datesAsEpochMillis > 0 : null; } - public Integer getDatesAsEpochMillisForUpdate() { + Integer getDatesAsEpochMillisForUpdate() { return datesAsEpochMillis; } @@ -200,7 +228,7 @@ public Boolean getAlignCheckpoints() { return alignCheckpoints != null ? (alignCheckpoints > 0) || (alignCheckpoints == DEFAULT_ALIGN_CHECKPOINTS) : null; } - public Integer getAlignCheckpointsForUpdate() { + Integer getAlignCheckpointsForUpdate() { return alignCheckpoints; } @@ -208,7 +236,7 @@ public Boolean getUsePit() { return usePit != null ? (usePit > 0) || (usePit == DEFAULT_USE_PIT) : null; } - public Integer getUsePitForUpdate() { + Integer getUsePitForUpdate() { return usePit; } @@ -216,7 +244,7 @@ public Boolean getDeduceMappings() { return deduceMappings != null ? (deduceMappings > 0) || (deduceMappings == DEFAULT_DEDUCE_MAPPINGS) : null; } - public Integer getDeduceMappingsForUpdate() { + Integer getDeduceMappingsForUpdate() { return deduceMappings; } @@ -224,7 +252,7 @@ public Integer getNumFailureRetries() { return numFailureRetries != null ? (numFailureRetries == DEFAULT_NUM_FAILURE_RETRIES ? null : numFailureRetries) : null; } - public Integer getNumFailureRetriesForUpdate() { + Integer getNumFailureRetriesForUpdate() { return numFailureRetries; } @@ -232,10 +260,20 @@ public Boolean getUnattended() { return unattended != null ? (unattended == DEFAULT_UNATTENDED) ? null : (unattended > 0) : null; } - public Integer getUnattendedForUpdate() { + Integer getUnattendedForUpdate() { return unattended; } + public Boolean getSkipDestIndexCreation() { + return skipDestIndexCreation != null + ? (skipDestIndexCreation == DEFAULT_SKIP_DEST_INDEX_CREATION) ? null : (skipDestIndexCreation > 0) + : null; + } + + Integer getSkipDestIndexCreationForUpdate() { + return skipDestIndexCreation; + } + public ActionRequestValidationException validate(ActionRequestValidationException validationException) { if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) { validationException = addValidationError( @@ -288,6 +326,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_5_0)) { out.writeOptionalInt(unattended); } + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { + out.writeOptionalInt(skipDestIndexCreation); + } } @Override @@ -318,6 +359,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (unattended != null && (unattended.equals(DEFAULT_UNATTENDED) == false)) { builder.field(TransformField.UNATTENDED.getPreferredName(), unattended > 0 ? true : false); } + if (skipDestIndexCreation != null && (skipDestIndexCreation.equals(DEFAULT_SKIP_DEST_INDEX_CREATION) == false)) { + builder.field(TransformField.SKIP_DEST_INDEX_CREATION.getPreferredName(), skipDestIndexCreation > 0 ? true : false); + } builder.endObject(); return builder; } @@ -339,7 +383,8 @@ public boolean equals(Object other) { && Objects.equals(usePit, that.usePit) && Objects.equals(deduceMappings, that.deduceMappings) && Objects.equals(numFailureRetries, that.numFailureRetries) - && Objects.equals(unattended, that.unattended); + && Objects.equals(unattended, that.unattended) + && Objects.equals(skipDestIndexCreation, that.skipDestIndexCreation); } @Override @@ -352,7 +397,8 @@ public int hashCode() { usePit, deduceMappings, numFailureRetries, - unattended + unattended, + skipDestIndexCreation ); } @@ -374,6 +420,7 @@ public static class Builder { private Integer deduceMappings; private Integer numFailureRetries; private Integer unattended; + private Integer skipDestIndexCreation; /** * Default builder @@ -394,6 +441,7 @@ public Builder(SettingsConfig base) { this.deduceMappings = base.deduceMappings; this.numFailureRetries = base.numFailureRetries; this.unattended = base.unattended; + this.skipDestIndexCreation = base.skipDestIndexCreation; } /** @@ -495,13 +543,26 @@ public Builder setNumFailureRetries(Integer numFailureRetries) { * An explicit `null` resets to default. * * @param unattended true if this is a unattended transform. - * @return the {@link Builder} with usePit set. + * @return the {@link Builder} with unattended set. */ public Builder setUnattended(Boolean unattended) { this.unattended = unattended == null ? DEFAULT_UNATTENDED : unattended ? 1 : 0; return this; } + /** + * Whether to skip destination index creation for the transform. + * + * An explicit `null` resets to default. + * + * @param skipDestIndexCreation true if destination index creation should be skipped. + * @return the {@link Builder} with skipDestIndexCreation set. + */ + public Builder setSkipDestIndexCreation(Boolean skipDestIndexCreation) { + this.skipDestIndexCreation = skipDestIndexCreation == null ? DEFAULT_SKIP_DEST_INDEX_CREATION : skipDestIndexCreation ? 1 : 0; + return this; + } + /** * Update settings according to given settings config. * @@ -545,6 +606,11 @@ public Builder update(SettingsConfig update) { if (update.getUnattendedForUpdate() != null) { this.unattended = update.getUnattendedForUpdate().equals(DEFAULT_UNATTENDED) ? null : update.getUnattendedForUpdate(); } + if (update.getSkipDestIndexCreationForUpdate() != null) { + this.skipDestIndexCreation = update.getSkipDestIndexCreationForUpdate().equals(DEFAULT_SKIP_DEST_INDEX_CREATION) + ? null + : update.getSkipDestIndexCreationForUpdate(); + } return this; } @@ -558,7 +624,8 @@ public SettingsConfig build() { usePit, deduceMappings, numFailureRetries, - unattended + unattended, + skipDestIndexCreation ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index d89eb9b397180..09bbd307d5996 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -234,7 +234,7 @@ public TransformConfig( this.pivotConfig = pivotConfig; this.latestConfig = latestConfig; this.description = description; - this.settings = settings == null ? new SettingsConfig() : settings; + this.settings = settings == null ? SettingsConfig.EMPTY : settings; this.metadata = metadata; this.retentionPolicyConfig = retentionPolicyConfig; if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) { @@ -600,7 +600,8 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) { builder.getSettings().getUsePit(), builder.getSettings().getDeduceMappings(), builder.getSettings().getNumFailureRetries(), - builder.getSettings().getUnattended() + builder.getSettings().getUnattended(), + builder.getSettings().getSkipDestIndexCreation() ) ); } @@ -616,7 +617,8 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) { builder.getSettings().getUsePit(), builder.getSettings().getDeduceMappings(), builder.getSettings().getNumFailureRetries(), - builder.getSettings().getUnattended() + builder.getSettings().getUnattended(), + builder.getSettings().getSkipDestIndexCreation() ) ); } @@ -632,7 +634,8 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) { builder.getSettings().getUsePit(), builder.getSettings().getDeduceMappings(), builder.getSettings().getNumFailureRetries(), - builder.getSettings().getUnattended() + builder.getSettings().getUnattended(), + builder.getSettings().getSkipDestIndexCreation() ) ); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java index 62b9e2e48a907..f88fd2857704c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java @@ -33,33 +33,35 @@ public class SettingsConfigTests extends AbstractSerializingTransformTestCase newMetadata = randomMetadata(); RetentionPolicyConfig retentionPolicyConfig = new TimeRetentionPolicyConfig("time_field", new TimeValue(60_000)); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDestIndexIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDestIndexIT.java index 29576231d848c..353e9d31a8133 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDestIndexIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDestIndexIT.java @@ -114,32 +114,103 @@ public void testTransformDestIndexAliases() throws Exception { assertAliases(destIndex2, destAliasAll, destAliasLatest); } - public void testTransformDestIndexCreatedDuringUpdate_NoDeferValidation() throws Exception { - testTransformDestIndexCreatedDuringUpdate(false); + public void testTransformDestIndexCreatedDuringUpdate_SkipDestIndexCreation_NoDeferValidation() throws Exception { + testTransformDestIndexCreatedDuringUpdate( + "test_dest_index_on_update-skip", + new SettingsConfig.Builder().setSkipDestIndexCreation(true).build(), + false + ); + } + + public void testTransformDestIndexCreatedDuringUpdate_SkipDestIndexCreation_DeferValidation() throws Exception { + testTransformDestIndexCreatedDuringUpdate( + "test_dest_index_on_update-skip-defer", + new SettingsConfig.Builder().setSkipDestIndexCreation(true).build(), + true + ); + } + + public void testTransformDestIndexCreatedDuringUpdate_Unattended_NoDeferValidation() throws Exception { + testTransformDestIndexCreatedDuringUpdate( + "test_dest_index_on_update-unattended", + new SettingsConfig.Builder().setUnattended(true).build(), + false + ); } - public void testTransformDestIndexCreatedDuringUpdate_DeferValidation() throws Exception { - testTransformDestIndexCreatedDuringUpdate(true); + public void testTransformDestIndexCreatedDuringUpdate_Unattended_DeferValidation() throws Exception { + testTransformDestIndexCreatedDuringUpdate( + "test_dest_index_on_update-unattended-defer", + new SettingsConfig.Builder().setUnattended(true).build(), + true + ); } - private void testTransformDestIndexCreatedDuringUpdate(boolean deferValidation) throws Exception { - String transformId = "test_dest_index_on_update" + (deferValidation ? "-defer" : ""); + private void testTransformDestIndexCreatedDuringUpdate(String transformId, SettingsConfig settingsConfig, boolean deferValidation) + throws Exception { + String sourceIndexIndex = transformId + "-src"; String destIndex = transformId + "-dest"; + // We want to use an empty source index to make sure transform will not write to the destination index + putReviewsIndex(sourceIndexIndex, "date", false); assertFalse(indexExists(destIndex)); - // Create and start the unattended transform - createPivotReviewsTransform( - transformId, - destIndex, - null, - null, - null, + // Create and start the transform that skips destination index creation + createPivotReviewsTransform(transformId, destIndex, null, null, null, settingsConfig, null, null, sourceIndexIndex); + startTransform(transformId); + + // Verify that the destination index creation got skipped + assertFalse(indexExists(destIndex)); + + // Update the unattended transform. This will trigger destination index creation. + // The update has to change something in the config (here, max_page_search_size). Otherwise it would have been optimized away. + updateTransform(transformId, """ + { "settings": { "max_page_search_size": 123 } }""", deferValidation); + + // Verify that the destination index now exists + assertTrue(indexExists(destIndex)); + } + + public void testTransformDestIndexCreated2DuringUpdate_SkipDestIndexCreation_NoDeferValidation() throws Exception { + testTransformDestIndexCreated2DuringUpdate( + "test_dest_index_on_update2-skip", + new SettingsConfig.Builder().setSkipDestIndexCreation(true).build(), + false + ); + } + + public void testTransformDestIndexCreated2DuringUpdate_SkipDestIndexCreation_DeferValidation() throws Exception { + testTransformDestIndexCreated2DuringUpdate( + "test_dest_index_on_update2-skip-defer", + new SettingsConfig.Builder().setSkipDestIndexCreation(true).build(), + true + ); + } + + public void testTransformDestIndexCreated2DuringUpdate_Unattended_NoDeferValidation() throws Exception { + testTransformDestIndexCreated2DuringUpdate( + "test_dest_index_on_update2-unattended", new SettingsConfig.Builder().setUnattended(true).build(), - null, - null, - REVIEWS_INDEX_NAME + false ); + } + + public void testTransformDestIndexCreated2DuringUpdate_Unattended_DeferValidation() throws Exception { + testTransformDestIndexCreated2DuringUpdate( + "test_dest_index_on_update2-unattended-defer", + new SettingsConfig.Builder().setUnattended(true).build(), + true + ); + } + + private void testTransformDestIndexCreated2DuringUpdate(String transformId, SettingsConfig settingsConfig, boolean deferValidation) + throws Exception { + String destIndex = transformId + "-dest"; + + // Create and start the transform that skips destination index creation + createPivotReviewsTransform(transformId, destIndex, null, null, null, settingsConfig, null, null, REVIEWS_INDEX_NAME); + assertFalse(indexExists(destIndex)); + startTransform(transformId); // Update the unattended transform. This will trigger destination index creation. diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java index c62ff49ae6865..dbe09663abc20 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java @@ -160,7 +160,7 @@ public void assertGetProgress(int userWithMissingBuckets) throws Exception { null ); - Pivot pivot = new Pivot(pivotConfig, new SettingsConfig(), TransformConfigVersion.CURRENT, Collections.emptySet()); + Pivot pivot = new Pivot(pivotConfig, SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet()); TransformProgress progress = getProgress(pivot, getProgressQuery(pivot, config.getSource().getIndex(), null)); @@ -188,7 +188,7 @@ public void assertGetProgress(int userWithMissingBuckets) throws Exception { Collections.singletonMap("every_50", new HistogramGroupSource("missing_field", null, missingBucket, 50.0)) ); pivotConfig = new PivotConfig(histgramGroupConfig, aggregationConfig, null); - pivot = new Pivot(pivotConfig, new SettingsConfig(), TransformConfigVersion.CURRENT, Collections.emptySet()); + pivot = new Pivot(pivotConfig, SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet()); progress = getProgress( pivot, diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java index db24470433003..21723d894e8e0 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.xpack.core.transform.action.StartTransformAction; import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction; import org.elasticsearch.xpack.core.transform.transforms.AuthorizationState; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; @@ -186,10 +187,8 @@ protected void masterOperation( // <3> If the destination index exists, start the task, otherwise deduce our mappings for the destination index and create it ActionListener validationListener = ActionListener.wrap(validationResponse -> { - if (Boolean.TRUE.equals(transformConfigHolder.get().getSettings().getUnattended())) { - logger.debug( - () -> format("[%s] Skip dest index creation as this is an unattended transform", transformConfigHolder.get().getId()) - ); + if (shouldSkipDestIndexCreation(transformConfigHolder.get().getSettings())) { + logger.debug(() -> format("[%s] Skip dest index creation for this transform", transformConfigHolder.get().getId())); createOrGetIndexListener.onResponse(true); return; } @@ -204,10 +203,8 @@ protected void masterOperation( createOrGetIndexListener ); }, e -> { - if (Boolean.TRUE.equals(transformConfigHolder.get().getSettings().getUnattended())) { - logger.debug( - () -> format("[%s] Skip dest index creation as this is an unattended transform", transformConfigHolder.get().getId()) - ); + if (shouldSkipDestIndexCreation(transformConfigHolder.get().getSettings())) { + logger.debug(() -> format("[%s] Skip dest index creation for this transform", transformConfigHolder.get().getId())); createOrGetIndexListener.onResponse(true); return; } @@ -389,4 +386,14 @@ private static boolean isNotStopped(PersistentTasksCustomMetadata.PersistentTask return state != null && state.getTaskState().equals(TransformTaskState.STOPPED) == false; } } + + // Visible for testing + static boolean shouldSkipDestIndexCreation(SettingsConfig settingsConfig) { + // "skip_dest_index_creation" setting takes precedence if explicitly set + if (settingsConfig.getSkipDestIndexCreation() != null) { + return settingsConfig.getSkipDestIndexCreation(); + } + // in other case, we fall back to the "unattended" setting + return settingsConfig.getUnattended() != null && settingsConfig.getUnattended(); + } } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformConfigLinterTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformConfigLinterTests.java index 30b86c71f473b..689978b64d7a4 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformConfigLinterTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformConfigLinterTests.java @@ -44,7 +44,7 @@ public void testGetWarnings_Pivot_WithScriptBasedRuntimeFields() { AggregationConfigTests.randomAggregationConfig(), null ); - Function function = new Pivot(pivotConfig, new SettingsConfig(), TransformConfigVersion.CURRENT, Collections.emptySet()); + Function function = new Pivot(pivotConfig, SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet()); SourceConfig sourceConfig = SourceConfigTests.randomSourceConfig(); assertThat(TransformConfigLinter.getWarnings(function, sourceConfig, null), is(empty())); @@ -117,7 +117,7 @@ public void testGetWarnings_Pivot_CouldNotFindAnyOptimization() { AggregationConfigTests.randomAggregationConfig(), null ); - Function function = new Pivot(pivotConfig, new SettingsConfig(), TransformConfigVersion.CURRENT, Collections.emptySet()); + Function function = new Pivot(pivotConfig, SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet()); SourceConfig sourceConfig = SourceConfigTests.randomSourceConfig(); SyncConfig syncConfig = TimeSyncConfigTests.randomTimeSyncConfig(); assertThat( diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStartTransformActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStartTransformActionTests.java new file mode 100644 index 0000000000000..11ec31b34b62a --- /dev/null +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStartTransformActionTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.transform.action; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; + +public class TransportStartTransformActionTests extends ESTestCase { + + // There are 9 combinations of the 2 settings ("skip_dest_index_creation" and "unattended") to test + public void testShouldSkipDestIndexCreation() { + SettingsConfig settingsConfig = new SettingsConfig.Builder().build(); + assertFalse(TransportStartTransformAction.shouldSkipDestIndexCreation(settingsConfig)); + + settingsConfig = new SettingsConfig.Builder().setUnattended(false).build(); + assertFalse(TransportStartTransformAction.shouldSkipDestIndexCreation(settingsConfig)); + + settingsConfig = new SettingsConfig.Builder().setUnattended(true).build(); + assertTrue(TransportStartTransformAction.shouldSkipDestIndexCreation(settingsConfig)); + + settingsConfig = new SettingsConfig.Builder().setSkipDestIndexCreation(false).build(); + assertFalse(TransportStartTransformAction.shouldSkipDestIndexCreation(settingsConfig)); + + settingsConfig = new SettingsConfig.Builder().setSkipDestIndexCreation(false).setUnattended(false).build(); + assertFalse(TransportStartTransformAction.shouldSkipDestIndexCreation(settingsConfig)); + + settingsConfig = new SettingsConfig.Builder().setSkipDestIndexCreation(false).setUnattended(true).build(); + assertFalse(TransportStartTransformAction.shouldSkipDestIndexCreation(settingsConfig)); + + settingsConfig = new SettingsConfig.Builder().setSkipDestIndexCreation(true).build(); + assertTrue(TransportStartTransformAction.shouldSkipDestIndexCreation(settingsConfig)); + + settingsConfig = new SettingsConfig.Builder().setSkipDestIndexCreation(true).setUnattended(false).build(); + assertTrue(TransportStartTransformAction.shouldSkipDestIndexCreation(settingsConfig)); + + settingsConfig = new SettingsConfig.Builder().setSkipDestIndexCreation(true).setUnattended(true).build(); + assertTrue(TransportStartTransformAction.shouldSkipDestIndexCreation(settingsConfig)); + } +} diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java index 750e535c4330f..7c294c6c3ebd3 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java @@ -186,6 +186,7 @@ public void testStatePersistenceErrorHandling() throws InterruptedException { randomBoolean(), randomBoolean(), 2, + false, false ) ); @@ -403,6 +404,7 @@ public void testStatePersistenceRecovery() throws InterruptedException { randomBoolean(), randomBoolean(), 2, + false, false ) ); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java index 5943a9007fb7c..1eb86b813f260 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java @@ -153,7 +153,7 @@ public void testBasic() throws InterruptedException { client, emptyMap(), "my-transform", - new SettingsConfig(), + SettingsConfig.EMPTY, pivotConfig, new SourceConfig(new String[] { "source-index" }), listener @@ -233,7 +233,7 @@ public void testNested() throws InterruptedException { client, emptyMap(), "my-transform", - new SettingsConfig(), + SettingsConfig.EMPTY, pivotConfig, new SourceConfig(new String[] { "source-index" }), listener diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java index 5d58ac9904482..0a030d26016f7 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java @@ -125,14 +125,14 @@ protected NamedXContentRegistry xContentRegistry() { public void testValidateExistingIndex() throws Exception { SourceConfig source = new SourceConfig("existing_source_index"); - Function pivot = new Pivot(getValidPivotConfig(), new SettingsConfig(), TransformConfigVersion.CURRENT, Collections.emptySet()); + Function pivot = new Pivot(getValidPivotConfig(), SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet()); assertValidTransform(client, source, pivot); } public void testValidateNonExistingIndex() throws Exception { SourceConfig source = new SourceConfig("non_existing_source_index"); - Function pivot = new Pivot(getValidPivotConfig(), new SettingsConfig(), TransformConfigVersion.CURRENT, Collections.emptySet()); + Function pivot = new Pivot(getValidPivotConfig(), SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet()); assertInvalidTransform(client, source, pivot); } @@ -142,7 +142,7 @@ public void testInitialPageSize() throws Exception { Function pivot = new Pivot( new PivotConfig(GroupConfigTests.randomGroupConfig(), getValidAggregationConfig(), expectedPageSize), - new SettingsConfig(), + SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet() ); @@ -150,7 +150,7 @@ public void testInitialPageSize() throws Exception { pivot = new Pivot( new PivotConfig(GroupConfigTests.randomGroupConfig(), getValidAggregationConfig(), null), - new SettingsConfig(), + SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet() ); @@ -164,7 +164,7 @@ public void testSearchFailure() throws Exception { // search has failures although they might just be temporary SourceConfig source = new SourceConfig("existing_source_index_with_failing_shards"); - Function pivot = new Pivot(getValidPivotConfig(), new SettingsConfig(), TransformConfigVersion.CURRENT, Collections.emptySet()); + Function pivot = new Pivot(getValidPivotConfig(), SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet()); assertInvalidTransform(client, source, pivot); } @@ -177,7 +177,7 @@ public void testValidateAllSupportedAggregations() throws Exception { Function pivot = new Pivot( getValidPivotConfig(aggregationConfig), - new SettingsConfig(), + SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet() ); @@ -191,7 +191,7 @@ public void testValidateAllUnsupportedAggregations() throws Exception { Function pivot = new Pivot( getValidPivotConfig(aggregationConfig), - new SettingsConfig(), + SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet() ); @@ -233,7 +233,7 @@ public void testGetPerformanceCriticalFields() throws IOException { assertThat(groupConfig.validate(null), is(nullValue())); PivotConfig pivotConfig = new PivotConfig(groupConfig, AggregationConfigTests.randomAggregationConfig(), null); - Function pivot = new Pivot(pivotConfig, new SettingsConfig(), TransformConfigVersion.CURRENT, Collections.emptySet()); + Function pivot = new Pivot(pivotConfig, SettingsConfig.EMPTY, TransformConfigVersion.CURRENT, Collections.emptySet()); assertThat(pivot.getPerformanceCriticalFields(), contains("field-A", "field-B", "field-C")); } diff --git a/x-pack/plugin/transform/src/test/resources/rest-api-spec/schema/transform_config.schema.json b/x-pack/plugin/transform/src/test/resources/rest-api-spec/schema/transform_config.schema.json index ff41548ee9cbb..51dad57843491 100644 --- a/x-pack/plugin/transform/src/test/resources/rest-api-spec/schema/transform_config.schema.json +++ b/x-pack/plugin/transform/src/test/resources/rest-api-spec/schema/transform_config.schema.json @@ -242,6 +242,13 @@ "type": "boolean", "description": "run this transform in unattended mode", "default": false + }, + "skip_dest_index_creation": { + "$id": "#root/settings/skip_dest_index_creation", + "title": "skip dest index creation", + "type": "boolean", + "description": "skip destination index creation for this transform", + "default": false } } },