68
68
import java .util .HashMap ;
69
69
import java .util .HashSet ;
70
70
import java .util .Iterator ;
71
+ import java .util .LinkedHashSet ;
71
72
import java .util .LinkedList ;
72
73
import java .util .List ;
73
74
import java .util .Locale ;
@@ -714,21 +715,8 @@ protected void doRun() {
714
715
continue ;
715
716
}
716
717
717
- final String pipelineId = indexRequest .getPipeline ();
718
- indexRequest .setPipeline (NOOP_PIPELINE_NAME );
719
- final String finalPipelineId = indexRequest .getFinalPipeline ();
720
- indexRequest .setFinalPipeline (NOOP_PIPELINE_NAME );
721
- boolean hasFinalPipeline = true ;
722
- final List <String > pipelines ;
723
- if (IngestService .NOOP_PIPELINE_NAME .equals (pipelineId ) == false
724
- && IngestService .NOOP_PIPELINE_NAME .equals (finalPipelineId ) == false ) {
725
- pipelines = List .of (pipelineId , finalPipelineId );
726
- } else if (IngestService .NOOP_PIPELINE_NAME .equals (pipelineId ) == false ) {
727
- pipelines = List .of (pipelineId );
728
- hasFinalPipeline = false ;
729
- } else if (IngestService .NOOP_PIPELINE_NAME .equals (finalPipelineId ) == false ) {
730
- pipelines = List .of (finalPipelineId );
731
- } else {
718
+ Pipelines pipelines = getPipelines (indexRequest );
719
+ if (pipelines .isEmpty ()) {
732
720
i ++;
733
721
continue ;
734
722
}
@@ -763,21 +751,80 @@ public void onFailure(Exception e) {
763
751
});
764
752
765
753
IngestDocument ingestDocument = newIngestDocument (indexRequest );
766
- executePipelines (pipelines .iterator (), hasFinalPipeline , indexRequest , ingestDocument , documentListener );
767
-
754
+ LinkedHashSet <String > indexRecursionDetection = new LinkedHashSet <>();
755
+ indexRecursionDetection .add (indexRequest .index ());
756
+ executePipelines (
757
+ pipelines .iterator (),
758
+ pipelines .hasFinalPipeline (),
759
+ indexRequest ,
760
+ ingestDocument ,
761
+ documentListener ,
762
+ indexRecursionDetection
763
+ );
768
764
i ++;
769
765
}
770
766
}
771
767
}
772
768
});
773
769
}
774
770
771
+ private Pipelines getPipelines (IndexRequest indexRequest ) {
772
+ indexRequest .isPipelineResolved (false );
773
+ resolvePipelines (null , indexRequest , state .metadata ());
774
+ final String pipelineId = indexRequest .getPipeline ();
775
+ indexRequest .setPipeline (NOOP_PIPELINE_NAME );
776
+ final String finalPipelineId = indexRequest .getFinalPipeline ();
777
+ indexRequest .setFinalPipeline (NOOP_PIPELINE_NAME );
778
+ return new Pipelines (pipelineId , finalPipelineId );
779
+ }
780
+
781
+ private static class Pipelines implements Iterable <String > {
782
+ private String defaultPipeline ;
783
+ private String finalPipeline ;
784
+
785
+ private Pipelines (String defaultPipeline , String finalPipeline ) {
786
+ if (NOOP_PIPELINE_NAME .equals (defaultPipeline ) == false ) {
787
+ this .defaultPipeline = defaultPipeline ;
788
+ }
789
+ if (NOOP_PIPELINE_NAME .equals (finalPipeline ) == false ) {
790
+ this .finalPipeline = finalPipeline ;
791
+ }
792
+ }
793
+
794
+ public boolean hasFinalPipeline () {
795
+ return finalPipeline != null ;
796
+ }
797
+
798
+ public boolean isEmpty () {
799
+ return defaultPipeline == null && finalPipeline == null ;
800
+ }
801
+
802
+ public void withoutDefaultPipeline () {
803
+ defaultPipeline = null ;
804
+ }
805
+
806
+ @ Override
807
+ public Iterator <String > iterator () {
808
+ if (defaultPipeline != null && finalPipeline != null ) {
809
+ return List .of (defaultPipeline , finalPipeline ).iterator ();
810
+ }
811
+ if (finalPipeline != null ) {
812
+ return List .of (finalPipeline ).iterator ();
813
+ }
814
+ if (defaultPipeline != null ) {
815
+ return List .of (defaultPipeline ).iterator ();
816
+ }
817
+ return Collections .emptyIterator ();
818
+ }
819
+ }
820
+
775
821
private void executePipelines (
776
822
final Iterator <String > pipelineIds ,
777
823
final boolean hasFinalPipeline ,
778
824
final IndexRequest indexRequest ,
779
825
final IngestDocument ingestDocument ,
780
- final ActionListener <Boolean > listener
826
+ final ActionListener <Boolean > listener ,
827
+ final Set <String > indexRecursionDetection
781
828
) {
782
829
assert pipelineIds .hasNext ();
783
830
final String pipelineId = pipelineIds .next ();
@@ -840,6 +887,14 @@ private void executePipelines(
840
887
final String newIndex = indexRequest .indices ()[0 ];
841
888
842
889
if (Objects .equals (originalIndex , newIndex ) == false ) {
890
+ if (indexRecursionDetection .add (newIndex ) == false ) {
891
+ List <String > indexRoute = new ArrayList <>(indexRecursionDetection );
892
+ indexRoute .add (newIndex );
893
+ listener .onFailure (
894
+ new IllegalStateException (format ("index cycle detected while processing pipelines: %s" , indexRoute ))
895
+ );
896
+ return ; // document failed!
897
+ }
843
898
if (hasFinalPipeline && pipelineIds .hasNext () == false ) {
844
899
listener .onFailure (
845
900
new IllegalStateException (
@@ -854,19 +909,16 @@ private void executePipelines(
854
909
);
855
910
return ; // document failed!
856
911
} else {
857
- indexRequest .isPipelineResolved (false );
858
- resolvePipelines (null , indexRequest , state .metadata ());
859
- if (IngestService .NOOP_PIPELINE_NAME .equals (indexRequest .getFinalPipeline ()) == false ) {
860
- newPipelineIds = Collections .singleton (indexRequest .getFinalPipeline ()).iterator ();
861
- newHasFinalPipeline = true ;
862
- } else {
863
- newPipelineIds = Collections .emptyIterator ();
864
- }
912
+ // reset request pipeline that is set to _none that would override the default pipeline
913
+ indexRequest .setPipeline (null );
914
+ Pipelines pipelines = getPipelines (indexRequest );
915
+ newHasFinalPipeline = pipelines .hasFinalPipeline ();
916
+ newPipelineIds = pipelines .iterator ();
865
917
}
866
918
}
867
919
868
920
if (newPipelineIds .hasNext ()) {
869
- executePipelines (newPipelineIds , newHasFinalPipeline , indexRequest , ingestDocument , listener );
921
+ executePipelines (newPipelineIds , newHasFinalPipeline , indexRequest , ingestDocument , listener , indexRecursionDetection );
870
922
} else {
871
923
// update the index request's source and (potentially) cache the timestamp for TSDB
872
924
updateIndexRequestSource (indexRequest , ingestDocument );
0 commit comments