Add support for ClassGraphDepthMax in Ice for Java

cpp/test/Ice/objects/AllTests.cpp

@@ -306,38 +306,28 @@ allTests(Test::TestHelper* helper)
 
     cout << "testing recursive type... " << flush;
     RecursivePtr top = make_shared<Recursive>();
-    int depth = 0;
+    RecursivePtr bottom = top;
+    int maxDepth = 99;
+    for (int i = 0; i < maxDepth; i++)
+    {
+        bottom->v = make_shared<Recursive>();
+        bottom = bottom->v;
+    }
+    initial->setRecursive(top);
+
+    // Adding one more level would exceed the max class graph depth
+    bottom->v = make_shared<Recursive>();
+    bottom = bottom->v;
+
     try
     {
-        RecursivePtr p = top;
-#if defined(NDEBUG) || !defined(__APPLE__)
-        const int maxDepth = 2000;
-#else
-        // With debug, marshaling a graph of 2000 elements can cause a stack overflow on macOS
-        const int maxDepth = 1500;
-#endif
-        for (; depth <= maxDepth; ++depth)
-        {
-            p->v = make_shared<Recursive>();
-            p = p->v;
-            if ((depth < 10 && (depth % 10) == 0) || (depth < 1000 && (depth % 100) == 0) ||
-                (depth < 10000 && (depth % 1000) == 0) || (depth % 10000) == 0)
-            {
-                initial->setRecursive(top);
-            }
-        }
-        test(!initial->supportsClassGraphDepthMax());
+        initial->setRecursive(top);
+        test(false);
     }
     catch (const Ice::UnknownLocalException&)
     {
         // Expected marshal exception from the server (max class graph depth reached)
-        test(depth == 100); // The default is 100.
-    }
-    catch (const Ice::UnknownException&)
-    {
-        // Expected stack overflow from the server (Java only)
     }
-    initial->setRecursive(make_shared<Recursive>());
     cout << "ok" << endl;
 
     cout << "testing compact ID..." << flush;

cpp/test/Ice/objects/Test.ice

@@ -216,7 +216,6 @@ interface Initial
     F getF();
 
     void setRecursive(Recursive p);
-    bool supportsClassGraphDepthMax();
 
     void setCycle(Recursive r);
     bool acceptsClassCycles();

cpp/test/Ice/objects/TestI.cpp

@@ -164,12 +164,6 @@ InitialI::setRecursive(RecursivePtr, const Current&)
 {
 }
 
-bool
-InitialI::supportsClassGraphDepthMax(const Current&)
-{
-    return true;
-}
-
 void
 InitialI::setCycle(RecursivePtr r, const Current&)
 {

cpp/test/Ice/objects/TestI.h

@@ -63,7 +63,6 @@ class InitialI final : public Test::Initial
     Test::FPtr getF(const Ice::Current&) final;
 
     void setRecursive(Test::RecursivePtr, const Ice::Current&) final;
-    bool supportsClassGraphDepthMax(const Ice::Current&) final;
 
     void setCycle(Test::RecursivePtr, const Ice::Current&) final;
     bool acceptsClassCycles(const Ice::Current&) final;

csharp/test/Ice/objects/AllTests.cs

@@ -309,32 +309,27 @@ public static Test.InitialPrx allTests(global::Test.TestHelper helper)
                     output.Write("testing recursive type... ");
                     output.Flush();
                     var top = new Test.Recursive();
-                    var p = top;
-                    int depth = 0;
+                    var bottom = top;
+                    int maxDepth = 99;
+                    for (int i = 0; i < maxDepth; i++)
+                    {
+                        bottom.v = new Test.Recursive();
+                        bottom = bottom.v;
+                    }
+                    initial.setRecursive(top);
+
+                    // Adding one more level would exceed the max class graph depth
+                    bottom.v = new Test.Recursive();
+                    bottom = bottom.v;
                     try
                     {
-                        for (; depth <= 1000; ++depth)
-                        {
-                            p.v = new Test.Recursive();
-                            p = p.v;
-                            if ((depth < 10 && (depth % 10) == 0) ||
-                              (depth < 1000 && (depth % 100) == 0) ||
-                              (depth < 10000 && (depth % 1000) == 0) ||
-                              (depth % 10000) == 0)
-                            {
-                                initial.setRecursive(top);
-                            }
-                        }
-                        test(!initial.supportsClassGraphDepthMax());
+                        initial.setRecursive(top);
+                        test(false);
                     }
                     catch (Ice.UnknownLocalException)
                     {
                         // Expected marshal exception from the server(max class graph depth reached)
                     }
-                    catch (Ice.UnknownException)
-                    {
-                        // Expected stack overflow from the server(Java only)
-                    }
                     initial.setRecursive(new Test.Recursive());
                     output.WriteLine("ok");
 

csharp/test/Ice/objects/InitialI.cs

@@ -98,11 +98,6 @@ public override void setRecursive(Test.Recursive r, Ice.Current current)
             {
             }
 
-            public override bool supportsClassGraphDepthMax(Ice.Current current)
-            {
-                return true;
-            }
-
             public override void setCycle(Test.Recursive r, Ice.Current current)
             {
             }

csharp/test/Ice/objects/Test.ice

@@ -195,7 +195,6 @@ interface Initial
     F getF();
 
     void setRecursive(Recursive p);
-    bool supportsClassGraphDepthMax();
 
     void setCycle(Recursive r);
     bool acceptsClassCycles();

java/src/Ice/src/main/java/com/zeroc/Ice/InputStream.java

@@ -238,6 +238,7 @@ private void initialize(Instance instance, EncodingVersion encoding) {
 
     _instance = instance;
     _traceSlicing = _instance.traceLevels().slicing > 0;
+    _classGraphDepthMax = _instance.classGraphDepthMax();
 
     _valueFactoryManager = _instance.initializationData().valueFactoryManager;
     _logger = _instance.initializationData().logger;
@@ -250,6 +251,7 @@ private void initialize(EncodingVersion encoding) {
     _encapsStack = null;
     _encapsCache = null;
     _traceSlicing = false;
+    _classGraphDepthMax = 0x7fffffff;
     _closure = null;
     _sliceValues = true;
     _startSeq = -1;
@@ -348,6 +350,19 @@ public void setTraceSlicing(boolean b) {
     _traceSlicing = b;
   }
 
+  /**
+   * Set the maximum depth allowed for graph of Slice class instances.
+   *
+   * @param classGraphDepthMax The maximum depth.
+   */
+  public void setClassGraphDepthMax(int classGraphDepthMax) {
+    if (classGraphDepthMax < 1) {
+      _classGraphDepthMax = 0x7fffffff;
+    } else {
+      _classGraphDepthMax = classGraphDepthMax;
+    }
+  }
+
   /**
    * Retrieves the closure object associated with this stream.
    *
@@ -401,6 +416,10 @@ public void swap(InputStream other) {
     other._sliceValues = _sliceValues;
     _sliceValues = tmpSliceValues;
 
+    int tmpClassGraphDepthMax = other._classGraphDepthMax;
+    other._classGraphDepthMax = _classGraphDepthMax;
+    _classGraphDepthMax = tmpClassGraphDepthMax;
+
     //
     // Swap is never called for streams that have encapsulations being read. However,
     // encapsulations might still be set in case unmarshaling failed. We just
@@ -1831,13 +1850,27 @@ private enum SliceType {
   }
 
   private abstract static class EncapsDecoder {
+
+    protected class PatchEntry {
+      public PatchEntry(java.util.function.Consumer<Value> cb, int classGraphDepth) {
+        this.cb = cb;
+        this.classGraphDepth = classGraphDepth;
+      }
+
+      public java.util.function.Consumer<Value> cb;
+      public int classGraphDepth;
+    }
+
     EncapsDecoder(
         InputStream stream,
         boolean sliceValues,
+        int classGraphDepthMax,
         ValueFactoryManager f,
         java.util.function.Function<String, Class<?>> cr) {
       _stream = stream;
       _sliceValues = sliceValues;
+      _classGraphDepthMax = classGraphDepthMax;
+      _classGraphDepth = 0;
       _valueFactoryManager = f;
       _classResolver = cr;
       _typeIdIndex = 0;
@@ -1971,7 +2004,7 @@ protected void addPatchEntry(int index, java.util.function.Consumer<Value> cb) {
       // the callback will be called when the instance is
       // unmarshaled.
       //
-      java.util.LinkedList<java.util.function.Consumer<Value>> l = _patchMap.get(index);
+      java.util.LinkedList<PatchEntry> l = _patchMap.get(index);
       if (l == null) {
         //
         // We have no outstanding instances to be patched for this
@@ -1984,7 +2017,7 @@ protected void addPatchEntry(int index, java.util.function.Consumer<Value> cb) {
       //
       // Append a patch entry for this instance.
       //
-      l.add(cb);
+      l.add(new PatchEntry(cb, _classGraphDepth));
     }
 
     protected void unmarshal(int index, Value v) {
@@ -2003,15 +2036,15 @@ protected void unmarshal(int index, Value v) {
         //
         // Patch all instances now that the instance is unmarshaled.
         //
-        java.util.LinkedList<java.util.function.Consumer<Value>> l = _patchMap.get(index);
+        java.util.LinkedList<PatchEntry> l = _patchMap.get(index);
         if (l != null) {
           assert (l.size() > 0);
 
           //
           // Patch all pointers that refer to the instance.
           //
-          for (java.util.function.Consumer<Value> cb : l) {
-            cb.accept(v);
+          for (PatchEntry entry : l) {
+            entry.cb.accept(v);
           }
 
           //
@@ -2061,14 +2094,15 @@ protected void unmarshal(int index, Value v) {
 
     protected final InputStream _stream;
     protected final boolean _sliceValues;
+    protected final int _classGraphDepthMax;
+    protected int _classGraphDepth;
     protected ValueFactoryManager _valueFactoryManager;
     protected java.util.function.Function<String, Class<?>> _classResolver;
 
     //
     // Encapsulation attributes for value unmarshaling.
     //
-    protected java.util.TreeMap<Integer, java.util.LinkedList<java.util.function.Consumer<Value>>>
-        _patchMap;
+    protected java.util.TreeMap<Integer, java.util.LinkedList<PatchEntry>> _patchMap;
     private java.util.TreeMap<Integer, Value> _unmarshaledMap;
     private java.util.TreeMap<Integer, String> _typeIdMap;
     private int _typeIdIndex;
@@ -2080,9 +2114,10 @@ private static final class EncapsDecoder10 extends EncapsDecoder {
     EncapsDecoder10(
         InputStream stream,
         boolean sliceValues,
+        int classGraphDepthMax,
         ValueFactoryManager f,
         java.util.function.Function<String, Class<?>> cr) {
-      super(stream, sliceValues, f, cr);
+      super(stream, sliceValues, classGraphDepthMax, f, cr);
       _sliceType = SliceType.NoSlice;
     }
 
@@ -2314,6 +2349,26 @@ private void readInstance() {
         startSlice(); // Read next Slice header for next iteration.
       }
 
+      //
+      // Compute the biggest class graph depth of this object. To compute this,
+      // we get the class graph depth of each ancestor from the patch map and
+      // keep the biggest one.
+      //
+      _classGraphDepth = 0;
+      var l = _patchMap != null ? _patchMap.get(index) : null;
+      if (l != null) {
+        assert (l.size() > 0);
+        for (PatchEntry entry : l) {
+          if (entry.classGraphDepth > _classGraphDepth) {
+            _classGraphDepth = entry.classGraphDepth;
+          }
+        }
+      }
+
+      if (++_classGraphDepth > _classGraphDepthMax) {
+        throw new MarshalException("maximum class graph depth reached");
+      }
+
       //
       // Unmarshal the instance and add it to the map of unmarshaled instances.
       //
@@ -2333,10 +2388,11 @@ private static class EncapsDecoder11 extends EncapsDecoder {
     EncapsDecoder11(
         InputStream stream,
         boolean sliceValues,
+        int classGraphDepthMax,
         ValueFactoryManager f,
         java.util.function.Function<String, Class<?>> cr,
         java.util.function.IntFunction<String> r) {
-      super(stream, sliceValues, f, cr);
+      super(stream, sliceValues, classGraphDepthMax, f, cr);
       _compactIdResolver = r;
       _current = null;
       _valueIdIndex = 1;
@@ -2767,11 +2823,17 @@ private int readInstance(int index, java.util.function.Consumer<Value> cb) {
         startSlice(); // Read next Slice header for next iteration.
       }
 
+      if (++_classGraphDepth > _classGraphDepthMax) {
+        throw new MarshalException("maximum class graph depth reached");
+      }
+
       //
       // Unmarshal the instance.
       //
       unmarshal(index, v);
 
+      --_classGraphDepth;
+
       if (_current == null && _patchMap != null && !_patchMap.isEmpty()) {
         //
         // If any entries remain in the patch map, the sender has sent an index for an instance, but
@@ -2917,11 +2979,17 @@ private void initEncaps() {
     {
       if (_encapsStack.encoding_1_0) {
         _encapsStack.decoder =
-            new EncapsDecoder10(this, _sliceValues, _valueFactoryManager, _classResolver);
+            new EncapsDecoder10(
+                this, _sliceValues, _classGraphDepthMax, _valueFactoryManager, _classResolver);
       } else {
         _encapsStack.decoder =
             new EncapsDecoder11(
-                this, _sliceValues, _valueFactoryManager, _classResolver, _compactIdResolver);
+                this,
+                _sliceValues,
+                _classGraphDepthMax,
+                _valueFactoryManager,
+                _classResolver,
+                _compactIdResolver);
       }
     }
   }
@@ -2945,6 +3013,7 @@ public static interface Unmarshaler {
   }
 
   private boolean _sliceValues;
+  private int _classGraphDepthMax;
   private boolean _traceSlicing;
 
   private int _startSeq;