Skip to content

Commit

Permalink
First class ECS schema support
Browse files Browse the repository at this point in the history
Object mappers now have a schema parameters.
At the moment, the only valid value is `ecs`.
During dynamic mapping, just before date/numeric detection and dynamic templates are evaluated,
it is evaluated whether the field name is defined by the schema.
If it is, the mapping is determined by the built-in schema file.

I've used a modified version of the ecs schema generation to store a schema file with flat mappings.
  • Loading branch information
felixbarny committed Oct 28, 2024
1 parent 9b951cd commit 363a404
Show file tree
Hide file tree
Showing 17 changed files with 6,991 additions and 210 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -755,8 +755,11 @@ private static void parseArrayDynamic(DocumentParserContext context, String curr
}
return;
}
Mapper objectMapperFromTemplate = DynamicFieldsBuilder.createObjectMapperFromTemplate(context, currentFieldName);
if (objectMapperFromTemplate == null) {
Mapper dynamicMapper = DynamicFieldsBuilder.createObjectMapperFromSchema(context, currentFieldName);
if (dynamicMapper != null) {
dynamicMapper = DynamicFieldsBuilder.createObjectMapperFromTemplate(context, currentFieldName);
}
if (dynamicMapper == null) {
if (context.indexSettings().isIgnoreDynamicFieldsBeyondLimit()
&& context.mappingLookup().exceedsLimit(context.indexSettings().getMappingTotalFieldsLimit(), 1)) {
if (context.canAddIgnoredField()) {
Expand All @@ -775,18 +778,18 @@ private static void parseArrayDynamic(DocumentParserContext context, String curr
context.addIgnoredField(currentFieldName);
return;
}
parseNonDynamicArray(context, objectMapperFromTemplate, currentFieldName, currentFieldName);
parseNonDynamicArray(context, null, currentFieldName, currentFieldName);
} else {
if (parsesArrayValue(objectMapperFromTemplate)) {
if (context.addDynamicMapper(objectMapperFromTemplate) == false) {
if (parsesArrayValue(dynamicMapper)) {
if (context.addDynamicMapper(dynamicMapper) == false) {
context.parser().skipChildren();
return;
}
context.path().add(currentFieldName);
parseObjectOrField(context, objectMapperFromTemplate);
parseObjectOrField(context, dynamicMapper);
context.path().remove();
} else {
parseNonDynamicArray(context, objectMapperFromTemplate, currentFieldName, currentFieldName);
parseNonDynamicArray(context, dynamicMapper, currentFieldName, currentFieldName);
}
}
}
Expand Down Expand Up @@ -1141,7 +1144,16 @@ protected SyntheticSourceSupport syntheticSourceSupport() {

private static class NoOpObjectMapper extends ObjectMapper {
NoOpObjectMapper(String name, String fullPath) {
super(name, fullPath, Explicit.IMPLICIT_TRUE, Optional.empty(), Optional.empty(), Dynamic.RUNTIME, Collections.emptyMap());
super(
name,
fullPath,
Explicit.IMPLICIT_TRUE,
Optional.empty(),
Optional.empty(),
Dynamic.RUNTIME,
Collections.emptyMap(),
null
);
}

@Override
Expand Down Expand Up @@ -1174,7 +1186,8 @@ private static class RootDocumentParserContext extends DocumentParserContext {
mappingParserContext,
source,
mappingLookup.getMapping().getRoot(),
ObjectMapper.Dynamic.getRootDynamic(mappingLookup)
ObjectMapper.Dynamic.getRootDynamic(mappingLookup),
mappingLookup.getMapping().getRoot().schema()
);
if (mappingLookup.getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.ENABLED) {
this.parser = DotExpandingXContentParser.expandDots(parser, this.path);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,13 @@ private static class Wrapper extends DocumentParserContext {
private final DocumentParserContext in;

private Wrapper(ObjectMapper parent, DocumentParserContext in) {
super(parent, parent.dynamic == null ? in.dynamic : parent.dynamic, in);
super(
parent,
parent.dynamic == null ? in.dynamic : parent.dynamic,
parent.schema == null ? in.schema : parent.schema,
parent.schema == null ? in.schemaRoot : parent.fullPath(),
in
);
this.in = in;
}

Expand Down Expand Up @@ -131,6 +137,8 @@ private enum Scope {
private final DocumentDimensions dimensions;
private final ObjectMapper parent;
private final ObjectMapper.Dynamic dynamic;
private final String schemaRoot;
private final DynamicMappingSchema schema;
private String id;
private Field version;
private final SeqNoFieldMapper.SequenceIDFields seqID;
Expand Down Expand Up @@ -165,6 +173,8 @@ private DocumentParserContext(
DocumentDimensions dimensions,
ObjectMapper parent,
ObjectMapper.Dynamic dynamic,
String schemaRoot,
DynamicMappingSchema schema,
Set<String> fieldsAppliedFromTemplates,
Set<String> copyToFields,
DynamicMapperSize dynamicMapperSize,
Expand All @@ -187,13 +197,21 @@ private DocumentParserContext(
this.dimensions = dimensions;
this.parent = parent;
this.dynamic = dynamic;
this.schemaRoot = schemaRoot;
this.schema = schema;
this.fieldsAppliedFromTemplates = fieldsAppliedFromTemplates;
this.copyToFields = copyToFields;
this.dynamicMappersSize = dynamicMapperSize;
this.recordedSource = recordedSource;
}

private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, DocumentParserContext in) {
private DocumentParserContext(
ObjectMapper parent,
ObjectMapper.Dynamic dynamic,
DynamicMappingSchema schema,
String schemaRoot,
DocumentParserContext in
) {
this(
in.mappingLookup,
in.mappingParserContext,
Expand All @@ -212,6 +230,8 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic,
in.dimensions,
parent,
dynamic,
schemaRoot,
schema,
in.fieldsAppliedFromTemplates,
in.copyToFields,
in.dynamicMappersSize,
Expand All @@ -224,7 +244,8 @@ protected DocumentParserContext(
MappingParserContext mappingParserContext,
SourceToParse source,
ObjectMapper parent,
ObjectMapper.Dynamic dynamic
ObjectMapper.Dynamic dynamic,
DynamicMappingSchema schema
) {
this(
mappingLookup,
Expand All @@ -244,6 +265,8 @@ protected DocumentParserContext(
DocumentDimensions.fromIndexSettings(mappingParserContext.getIndexSettings()),
parent,
dynamic,
"",
schema,
new HashSet<>(),
new HashSet<>(mappingLookup.fieldTypesLookup().getCopyToDestinationFields()),
new DynamicMapperSize(),
Expand Down Expand Up @@ -864,6 +887,17 @@ public final DynamicTemplate findDynamicTemplate(String fieldName, DynamicTempla
return null;
}

public Map<String, Object> findMappingFromSchema(String fieldName) {
if (schema == null) {
return null;
}
String path = path().pathAsText(fieldName);
if (schemaRoot.isEmpty() == false) {
path = path.substring(schemaRoot.length() + 1);
}
return schema.mappingForPath(path);
}

// XContentParser that wraps an existing parser positioned on a value,
// and a field name, and returns a stream that looks like { 'field' : 'value' }
private static class CopyToParser extends FilterXContentParserWrapper {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ private DynamicFieldsBuilder(Strategy strategy) {
* The strategy defines if fields are going to be mapped as ordinary or runtime fields.
*/
boolean createDynamicFieldFromValue(final DocumentParserContext context, String name) throws IOException {
Mapper.Builder builder = findMapperBuilderBuilderFromSchema(context, name);
if (builder != null) {
// if the field is defined by a schema, we can short-circuit the dynamic mapping process
CONCRETE.createDynamicField(builder, context);
return true;
}
XContentParser.Token token = context.parser().currentToken();
if (token == XContentParser.Token.VALUE_STRING) {
String text = context.parser().text();
Expand Down Expand Up @@ -159,11 +165,15 @@ boolean createDynamicFieldFromValue(final DocumentParserContext context, String
* Returns a dynamically created object mapper, eventually based on a matching dynamic template.
*/
static Mapper createDynamicObjectMapper(DocumentParserContext context, String name) {
Mapper mapper = createObjectMapperFromTemplate(context, name);
return mapper != null
? mapper
: new ObjectMapper.Builder(name, context.parent().subobjects).enabled(ObjectMapper.Defaults.ENABLED)
Mapper mapper = createObjectMapperFromSchema(context, name);
if (mapper == null) {
mapper = createObjectMapperFromTemplate(context, name);
}
if (mapper == null) {
mapper = new ObjectMapper.Builder(name, context.parent().subobjects).enabled(ObjectMapper.Defaults.ENABLED)
.build(context.createDynamicMapperBuilderContext());
}
return mapper;
}

/**
Expand All @@ -174,6 +184,25 @@ static Mapper createObjectMapperFromTemplate(DocumentParserContext context, Stri
return templateBuilder == null ? null : templateBuilder.build(context.createDynamicMapperBuilderContext());
}

/**
* Returns a dynamically created object mapper, based exclusively on a {@link DynamicMappingSchema}, null otherwise.
*/
static Mapper createObjectMapperFromSchema(DocumentParserContext context, String name) {
Mapper.Builder builder = findMapperBuilderBuilderFromSchema(context, name);
return builder == null ? null : builder.build(context.createDynamicMapperBuilderContext());
}

private static Mapper.Builder findMapperBuilderBuilderFromSchema(DocumentParserContext context, String name) {
Map<String, Object> mappingFromSchema = context.findMappingFromSchema(name);
if (mappingFromSchema != null) {
String type = (String) mappingFromSchema.get("type");
if (type != null) {
return parseDynamicTemplateMapping(name, type, mappingFromSchema, null, context);
}
}
return null;
}

/**
* Creates a dynamic string field based on a matching dynamic template.
* No field is created in case there is no matching dynamic template.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.index.mapper;

import org.elasticsearch.common.util.Maps;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentParserConfiguration;
import org.elasticsearch.xcontent.XContentType;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

public enum DynamicMappingSchema {
ECS {
private static final Map<String, Object> schema;

static {
try (
XContentParser parser = XContentType.JSON.xContent()
.createParser(XContentParserConfiguration.EMPTY, DynamicMappingSchema.class.getResourceAsStream("/schema/ecs.json"))
) {
schema = parser.map();
} catch (IOException e) {
throw new AssertionError(e);
}
}

@Override
public Map<String, Object> mappingForPath(String path) {
@SuppressWarnings("unchecked")
Map<String, Object> mapping = (Map<String, Object>) schema.get(path);
if (mapping != null) {
if (path.endsWith("message")) {
System.out.println(mapping);
}
return deepCopy(mapping);
} else {
return null;
}
}

private static Map<String, Object> deepCopy(Map<String, Object> mapping) {
return Maps.copyOf(mapping, v -> {
if (v instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> m = (Map<String, Object>) v;
return new HashMap<>(deepCopy(m));
} else {
return v;
}
});
}

@Override
public String toString() {
return "ecs";
}
};

public abstract Map<String, Object> mappingForPath(String path);

public static DynamicMappingSchema fromString(String value) {
for (DynamicMappingSchema v : values()) {
if (v.toString().equals(value)) {
return v;
}
}
throw new IllegalArgumentException("No matching schema for [" + value + "]");
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,9 @@ public NestedObjectMapper build(MapperBuilderContext context) {
+ "]"
);
}
if (schema != null) {
throw new MapperException("parameter [schema] is not supported for nested object [" + fullPath + "]");
}
final Query nestedTypeFilter = NestedPathFieldMapper.filter(indexCreatedVersion, nestedTypePath);
NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext(
context.buildFullName(leafName()),
Expand Down Expand Up @@ -233,7 +236,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) {
Function<Query, BitSetProducer> bitsetProducer,
IndexSettings indexSettings
) {
super(name, fullPath, enabled, Optional.empty(), sourceKeepMode, dynamic, mappers);
super(name, fullPath, enabled, Optional.empty(), sourceKeepMode, dynamic, mappers, null);
this.parentTypeFilter = parentTypeFilter;
this.nestedTypePath = nestedTypePath;
this.nestedTypeFilter = nestedTypeFilter;
Expand Down
Loading

0 comments on commit 363a404

Please sign in to comment.