Allow resource.attributes.host.name to pass through

Only add host.name to the LogsDB mapping if it's actually the primary sort field
felixbarny · Oct 15, 2024 · bbda6f1 · bbda6f1
1 parent 81976b2
commit bbda6f1
Show file tree

Hide file tree

Showing 5 changed files with 79 additions and 12 deletions.
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
@@ -1373,7 +1373,7 @@ private static void updateIndexMappingsAndBuildSortOrder(
         MapperService mapperService = indexService.mapperService();
         IndexMode indexMode = indexService.getIndexSettings() != null ? indexService.getIndexSettings().getMode() : IndexMode.STANDARD;
         List<CompressedXContent> allMappings = new ArrayList<>();
-        final CompressedXContent defaultMapping = indexMode.getDefaultMapping();
+        final CompressedXContent defaultMapping = indexMode.getDefaultMapping(indexService.getIndexSettings());
         if (defaultMapping != null) {
             allMappings.add(defaultMapping);
         }

diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java
@@ -41,6 +41,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.BooleanSupplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -75,7 +76,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
         }
 
         @Override
-        public CompressedXContent getDefaultMapping() {
+        public CompressedXContent getDefaultMapping(IndexSettings indexSettings) {
             return null;
         }
 
@@ -171,7 +172,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
         }
 
         @Override
-        public CompressedXContent getDefaultMapping() {
+        public CompressedXContent getDefaultMapping(IndexSettings indexSettings) {
             return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
         }
 
@@ -249,8 +250,16 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
         }
 
         @Override
-        public CompressedXContent getDefaultMapping() {
-            return DEFAULT_LOGS_TIMESTAMP_MAPPING;
+        public CompressedXContent getDefaultMapping(IndexSettings indexSettings) {
+            boolean sortOnHostName = Optional.ofNullable(indexSettings)
+                .map(s -> s.getIndexSortConfig().hasPrimarySortOnField("host.name"))
+                .orElse(false);
+
+            if (sortOnHostName) {
+                return DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOST_NAME;
+            } else {
+                return DEFAULT_LOGS_TIMESTAMP_MAPPING;
+            }
         }
 
         @Override
@@ -346,11 +355,11 @@ protected static String tsdbMode() {
         }
     }
 
-    public static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING;
+    public static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOST_NAME;
 
     static {
         try {
-            DEFAULT_LOGS_TIMESTAMP_MAPPING = new CompressedXContent(
+            DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOST_NAME = new CompressedXContent(
                 ((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
                     .startObject(DataStreamTimestampFieldMapper.NAME)
                     .field("enabled", true)
@@ -371,6 +380,27 @@ protected static String tsdbMode() {
         }
     }
 
+    public static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING;
+
+    static {
+        try {
+            DEFAULT_LOGS_TIMESTAMP_MAPPING = new CompressedXContent(
+                ((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
+                    .startObject(DataStreamTimestampFieldMapper.NAME)
+                    .field("enabled", true)
+                    .endObject()
+                    .startObject("properties")
+                    .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
+                    .field("type", DateFieldMapper.CONTENT_TYPE)
+                    .endObject()
+                    .endObject()
+                    .endObject())
+            );
+        } catch (IOException e) {
+            throw new AssertionError(e);
+        }
+    }
+
     private static final List<Setting<?>> TIME_SERIES_UNSUPPORTED = List.of(
         IndexSortConfig.INDEX_SORT_FIELD_SETTING,
         IndexSortConfig.INDEX_SORT_ORDER_SETTING,
@@ -421,7 +451,7 @@ public String getName() {
      * Get default mapping for this index or {@code null} if there is none.
      */
     @Nullable
-    public abstract CompressedXContent getDefaultMapping();
+    public abstract CompressedXContent getDefaultMapping(IndexSettings indexSettings);
 
     /**
      * Build the {@link FieldMapper} for {@code _id}.

diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
@@ -13,14 +13,24 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class LogsIndexModeTests extends ESTestCase {
     public void testLogsIndexModeSetting() {
         assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGSDB));
     }
 
-    public void testSortField() {
+    public void testDefaultHostNameSortField() {
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
+        final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
+        assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true));
+        assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), containsString("host.name"));
+    }
+
+    public void testCustomSortField() {
         final Settings sortSettings = Settings.builder()
             .put(buildSettings())
             .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
@@ -29,7 +39,9 @@ public void testSortField() {
         assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
         assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB));
-        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()), equalTo("agent_id"));
+        assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(false));
+        assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), not(containsString("host")));
     }
 
     public void testSortMode() {

diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -302,8 +302,9 @@ public void onRemoval(ShardId shardId, Accountable accountable) {}
                 mapperMetrics
             );
 
-            if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping() != null) {
-                mapperService.merge(null, indexSettings.getMode().getDefaultMapping(), MapperService.MergeReason.MAPPING_UPDATE);
+            CompressedXContent defaultMapping = indexSettings.getMode().getDefaultMapping(indexSettings);
+            if (applyDefaultMapping && defaultMapping != null) {
+                mapperService.merge(null, defaultMapping, MapperService.MergeReason.MAPPING_UPDATE);
             }
 
             return mapperService;

diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml
@@ -163,3 +163,27 @@ Structured log body:
           fields: ["event.dataset"]
   - length: { hits.hits: 1 }
   - match: { hits.hits.0.fields.event\.dataset: ["generic.otel"] }
+---
+host.name pass-through:
+  - do:
+      bulk:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - "@timestamp": 2024-07-18T14:48:33.467654000Z
+            resource:
+              attributes:
+                host.name: localhost
+  - is_false: errors
+  - do:
+      search:
+        index: logs-generic.otel-default
+        body:
+          query:
+            term:
+              host.name: localhost
+          fields: [ "*" ]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.resource\.attributes\.host\.name: [ "localhost" ] }
+  - match: { hits.hits.0.fields.host\.name: [ "localhost" ] }