Bulk loader hangs if message is lost

[marmbrus]

"main" prio=10 tid=0x0000000001955000 nid=0x1135 in Object.wait() [0x00007fbc6323b000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007fbab2d73a08> (a edu.berkeley.cs.scads.comm.NioEndpoint$ConnectFuture)
at java.lang.Object.wait(Object.java:502)
at edu.berkeley.cs.scads.comm.NioEndpoint$ConnectFuture.await(AbstractNioEndpoint.scala:72)
- locked <0x00007fbab2d73a08> (a edu.berkeley.cs.scads.comm.NioEndpoint$ConnectFuture)
at edu.berkeley.cs.scads.comm.NioAvroChannelManagerBase.getChannel(avro.scala:69)
at edu.berkeley.cs.scads.comm.NioAvroChannelManagerBase.sendMessage(avro.scala:83)
at edu.berkeley.cs.scads.comm.MessageHandler$.sendMessage(MessageHandler.scala:101)
at edu.berkeley.cs.scads.comm.RemoteActorProxy$class.$bang(Actors.scala:66)
at edu.berkeley.cs.scads.comm.PartitionService.$bang(Actors.scala:35)
at edu.berkeley.cs.scads.comm.RemoteActorProxy$class.$bang$bang(Actors.scala:90)
at edu.berkeley.cs.scads.comm.PartitionService.$bang$bang(Actors.scala:35)
at edu.berkeley.cs.scads.storage.QuorumProtocol$$anonfun$writeServerBuffers$1$1.apply(QuorumProtocol.scala:319)
at edu.berkeley.cs.scads.storage.QuorumProtocol$$anonfun$writeServerBuffers$1$1.apply(QuorumProtocol.scala:318)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:206)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:206)
at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:80)
at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:80)
at scala.collection.Iterator$class.foreach(Iterator.scala:631)
at scala.collection.mutable.HashTable$$anon$1.foreach(HashTable.scala:161)
at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:194)
at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)
at scala.collection.mutable.HashMap.foreach(HashMap.scala:80)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:206)
at scala.collection.mutable.HashMap.map(HashMap.scala:39)
at edu.berkeley.cs.scads.storage.QuorumProtocol.writeServerBuffers$1(QuorumProtocol.scala:318)
at edu.berkeley.cs.scads.storage.QuorumProtocol$$anonfun$$plus$plus$eq$1.apply(QuorumProtocol.scala:341)
at edu.berkeley.cs.scads.storage.QuorumProtocol$$anonfun$$plus$plus$eq$1.apply(QuorumProtocol.scala:329)
at scala.collection.TraversableViewLike$Mapped$$anonfun$foreach$2.apply(TraversableViewLike.scala:104)
at scala.collection.Iterator$class.foreach(Iterator.scala:631)
at scala.collection.IndexedSeqLike$Elements.foreach(IndexedSeqLike.scala:52)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:79)
at scala.collection.SeqLike$$anon$2.foreach(SeqLike.scala:848)
at scala.collection.TraversableViewLike$Mapped$class.foreach(TraversableViewLike.scala:103)
at scala.collection.SeqViewLike$$anon$3.foreach(SeqViewLike.scala:175)
at edu.berkeley.cs.scads.storage.QuorumProtocol.$plus$plus$eq(QuorumProtocol.scala:329)
at edu.berkeley.cs.scads.piql.ScadrLoader$ScadrData.load(ScadrLoader.scala:111)
at edu.berkeley.cs.scads.perf.CardinalityLoadClient.run(ScadrCardinality.scala:91)
at edu.berkeley.cs.scads.perf.Experiment$class.main(Experiment.scala:55)
at edu.berkeley.cs.scads.perf.ScadrCardinalityTest$.main(ScadrCardinality.scala:17)
at edu.berkeley.cs.scads.perf.ScadrCardinalityTest.main(ScadrCardinality.scala)