Added reverse_nested aggregation.

The `reverse_nested` aggregation allows to aggregate on properties outside of the nested scope of a `nested` aggregation.

Closes elastic#5507

docs/reference/search/aggregations/bucket.asciidoc

@@ -8,6 +8,8 @@ include::bucket/missing-aggregation.asciidoc[]
 
 include::bucket/nested-aggregation.asciidoc[]
 
+include::bucket/reverse-nested-aggregation.asciidoc[]
+
 include::bucket/terms-aggregation.asciidoc[]
 
 include::bucket/significantterms-aggregation.asciidoc[]

docs/reference/search/aggregations/bucket/reverse-nested-aggregation.asciidoc

@@ -0,0 +1,115 @@
+[[search-aggregations-bucket-reverse-nested-aggregation]]
+=== Reverse nested
+
+A special single bucket aggregation that enables aggregating on parent docs from nested documents. Effectively this
+aggregation can break out of the nested block structure and link to other nested structures or the root document,
+which allows nesting other aggregations that aren't part of the nested object in a nested aggregation.
+
+The `reverse_nested` aggregation must be defined inside a `nested` aggregation.
+
+.Options:
+* `path` - Which defines to what nested object field should be joined back. The default is empty,
+which means that it joins back to the root / main document level. The path cannot contain a reference to
+a nested object field that falls outside the `nested` aggregation's nested structure a `reverse_nested` is in.
+
+For example, lets say we have an index for a ticket system which issues and comments. The comments are inlined into
+the issue documents as nested documents. The mapping could look like:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+
+    "issue" : {
+        "properties" : {
+            "tags" : { "type" : "string" }
+            "comments" : { <1>
+                "type" : "nested"
+                "properties" : {
+                    "username" : { "type" : "string", "index" : "not_analyzed" },
+                    "comment" : { "type" : "string" }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+<1> The `comments` is an array that holds nested documents under the `issue` object.
+
+The following aggregations will return the top commenters' username that have commented and per top commenter the top
+tags that issues have the commenter has commented to:
+
+[source,js]
+--------------------------------------------------
+{
+    "query" : {
+        "match" : { "name" : "led tv" }
+    }
+    "aggs" : {
+        "comments" : {
+            "nested" : {
+                "path" : "comments"
+            },
+            "aggs" : {
+                "top_usernames" : {
+                    "terms" : {
+                        "field" : "comments.username"
+                    }
+                },
+                "aggs" : {
+                    "comment_to_issue" : {
+                        "reverse_nested" : { <1>
+                        },
+                        "aggs" : {
+                            "top_tags_per_comment" : {
+                                "terms" : { "field" : "tags" }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+As you can see above, the the `reverse_nested` aggregation is put in to a `nested` aggregation as this is the only place
+in the dsl where the `reversed_nested` aggregation can be used. Its sole purpose is to join back to a parent doc higher
+up in the nested structure.
+
+<1> A `reverse_nested` aggregation that joins back to the root / main document level, because no `path` has been defined.
+Via the `path` option the `reverse_nested` aggregation can join back to a different level, if multiple layered nested
+object types have been defined in the mapping
+
+Possible response snippet:
+
+[source,js]
+--------------------------------------------------
+{
+    "aggregations": {
+        "comments": {
+            "top_usernames": {
+                "buckets" : [
+                    {
+                        "key" : "username_1",
+                        "doc_count" : 12,
+                        "comment_to_issue" : {
+                            "top_tags_per_comment" : {
+                                "buckets" : [
+                                    {
+                                        "key" : "tag1",
+                                        "doc_count" : 9
+                                    },
+                                    ...
+                                ]
+                            }
+                        },
+                        ...
+                    }
+                ]
+            }
+        }
+    }
+}
+--------------------------------------------------

src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.ipv4.IPv4RangeBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedBuilder;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgBuilder;
@@ -93,6 +94,10 @@ public static NestedBuilder nested(String name) {
         return new NestedBuilder(name);
     }
 
+    public static ReverseNestedBuilder reverseNested(String name) {
+        return new ReverseNestedBuilder(name);
+    }
+
     public static GeoDistanceBuilder geoDistance(String name) {
         return new GeoDistanceBuilder(name);
     }

src/main/java/org/elasticsearch/search/aggregations/AggregationModule.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeParser;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser;
 import org.elasticsearch.search.aggregations.bucket.range.ipv4.IpRangeParser;
+import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedParser;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsParser;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsParser;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgParser;
@@ -77,6 +78,7 @@ public AggregationModule() {
         parsers.add(GeoDistanceParser.class);
         parsers.add(GeoHashGridParser.class);
         parsers.add(NestedParser.class);
+        parsers.add(ReverseNestedParser.class);
     }
 
     /**

src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java

@@ -165,7 +165,7 @@ public void setNextReader(AtomicReaderContext context) throws IOException {
 
         @Override
         public boolean acceptsDocsOutOfOrder() {
-            return true;
+            return !aggregationContext.scoreDocsInOrder();
         }
 
         @Override

src/main/java/org/elasticsearch/search/aggregations/TransportAggregationModule.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.search.aggregations.bucket.range.date.InternalDateRange;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.InternalGeoDistance;
 import org.elasticsearch.search.aggregations.bucket.range.ipv4.InternalIPv4Range;
+import org.elasticsearch.search.aggregations.bucket.nested.InternalReverseNested;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms;
 import org.elasticsearch.search.aggregations.bucket.significant.UnmappedSignificantTerms;
@@ -85,5 +86,6 @@ protected void configure() {
         InternalDateHistogram.registerStream();
         InternalGeoDistance.registerStream();
         InternalNested.registerStream();
+        InternalReverseNested.registerStream();
     }
 }

src/main/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNested.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.nested;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.search.aggregations.AggregationStreams;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class InternalReverseNested extends InternalSingleBucketAggregation implements ReverseNested {
+
+    public static final Type TYPE = new Type("reverse_nested");
+
+    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+        @Override
+        public InternalReverseNested readResult(StreamInput in) throws IOException {
+            InternalReverseNested result = new InternalReverseNested();
+            result.readFrom(in);
+            return result;
+        }
+    };
+
+    public static void registerStream() {
+        AggregationStreams.registerStream(STREAM, TYPE.stream());
+    }
+
+    public InternalReverseNested() {
+    }
+
+    public InternalReverseNested(String name, long docCount, InternalAggregations aggregations) {
+        super(name, docCount, aggregations);
+    }
+
+    @Override
+    public Type type() {
+        return TYPE;
+    }
+
+}

src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java

@@ -40,6 +40,7 @@
  */
 public class NestedAggregator extends SingleBucketAggregator implements ReaderContextAware {
 
+    private final String nestedPath;
     private final Aggregator parentAggregator;
     private Filter parentFilter;
     private final Filter childFilter;
@@ -49,31 +50,23 @@ public class NestedAggregator extends SingleBucketAggregator implements ReaderCo
 
     public NestedAggregator(String name, AggregatorFactories factories, String nestedPath, AggregationContext aggregationContext, Aggregator parentAggregator) {
         super(name, factories, aggregationContext, parentAggregator);
+        this.nestedPath = nestedPath;
         this.parentAggregator = parentAggregator;
         MapperService.SmartNameObjectMapper mapper = aggregationContext.searchContext().smartNameObjectMapper(nestedPath);
         if (mapper == null) {
-            throw new AggregationExecutionException("facet nested path [" + nestedPath + "] not found");
+            throw new AggregationExecutionException("[nested] nested path [" + nestedPath + "] not found");
         }
         ObjectMapper objectMapper = mapper.mapper();
         if (objectMapper == null) {
-            throw new AggregationExecutionException("facet nested path [" + nestedPath + "] not found");
+            throw new AggregationExecutionException("[nested] nested path [" + nestedPath + "] not found");
         }
         if (!objectMapper.nested().isNested()) {
-            throw new AggregationExecutionException("facet nested path [" + nestedPath + "] is not nested");
+            throw new AggregationExecutionException("[nested] nested path [" + nestedPath + "] is not nested");
         }
 
         childFilter = aggregationContext.searchContext().filterCache().cache(objectMapper.nestedTypeFilter());
     }
 
-    private NestedAggregator findClosestNestedAggregator(Aggregator parent) {
-        for (; parent != null; parent = parent.parent()) {
-            if (parent instanceof NestedAggregator) {
-                return (NestedAggregator) parent;
-            }
-        }
-        return null;
-    }
-
     @Override
     public void setNextReader(AtomicReaderContext reader) {
         if (parentFilter == null) {
@@ -135,6 +128,19 @@ public InternalAggregation buildEmptyAggregation() {
         return new InternalNested(name, 0, buildEmptySubAggregations());
     }
 
+    public String getNestedPath() {
+        return nestedPath;
+    }
+
+    static NestedAggregator findClosestNestedAggregator(Aggregator parent) {
+        for (; parent != null; parent = parent.parent()) {
+            if (parent instanceof NestedAggregator) {
+                return (NestedAggregator) parent;
+            }
+        }
+        return null;
+    }
+
     public static class Factory extends AggregatorFactory {
 
         private final String path;

src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNested.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.nested;
+
+import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
+
+/**
+ */
+public interface ReverseNested extends SingleBucketAggregation {
+}