Merge pull request #1346 from scottpurdy/nup-2341

Add serialization for Cells4, Cell, Segment, SegmentUpdate, CState, etc

src/CMakeLists.txt

@@ -214,6 +214,8 @@ set(src_py_support_files
 set(src_capnp_specs_rel
   nupic/proto/ArrayProto.capnp
   nupic/proto/BitHistory.capnp
+  nupic/proto/Cell.capnp
+  nupic/proto/Cells4.capnp
   nupic/proto/ClaClassifier.capnp
   nupic/proto/ConnectionsProto.capnp
   nupic/proto/LinkProto.capnp
@@ -222,6 +224,8 @@ set(src_capnp_specs_rel
   nupic/proto/PyRegionProto.capnp
   nupic/proto/RandomProto.capnp
   nupic/proto/RegionProto.capnp
+  nupic/proto/Segment.capnp
+  nupic/proto/SegmentUpdate.capnp
   nupic/proto/SparseBinaryMatrixProto.capnp
   nupic/proto/SparseMatrixProto.capnp
   nupic/proto/SpatialPoolerProto.capnp

src/nupic/algorithms/Cell.cpp

@@ -20,6 +20,7 @@
  * ---------------------------------------------------------------------
  */
 
+#include <nupic/proto/Cell.capnp.h>
 #include <nupic/algorithms/Cell.hpp>
 
 using namespace nupic::algorithms::Cells4;
@@ -121,6 +122,34 @@ void Cell::updateDutyCycle(UInt iterations)
   }
 }
 
+//-----------------------------------------------------------------------------
+void Cell::write(CellProto::Builder& proto) const
+{
+  auto segmentsProto = proto.initSegments(_segments.size());
+  for (UInt i = 0; i < _segments.size(); ++i)
+  {
+    auto segProto = segmentsProto[i];
+    _segments[i].write(segProto);
+  }
+}
+
+//-----------------------------------------------------------------------------
+void Cell::read(CellProto::Reader& proto)
+{
+  auto segmentsProto = proto.getSegments();
+  _segments.resize(segmentsProto.size());
+  _freeSegments.resize(0);
+  for (UInt i = 0; i < segmentsProto.size(); ++i)
+  {
+    auto segProto = segmentsProto[i];
+    _segments[i].read(segProto);
+    if (_segments[i].empty())
+    {
+      _freeSegments.push_back(i);
+    }
+  }
+}
+
 //-----------------------------------------------------------------------------
 void Cell::save(std::ostream& outStream) const
 {

src/nupic/algorithms/Cell.hpp

@@ -24,6 +24,8 @@
 #define NTA_CELL_HPP
 
 #include <vector>
+#include <nupic/proto/Cell.capnp.h>
+#include <nupic/types/Serializable.hpp>
 #include <nupic/types/Types.hpp>
 #include <nupic/algorithms/Segment.hpp>
 
@@ -44,7 +46,7 @@ namespace nupic {
        * mismatches in unit testing when comparing the Python TP to the C++ down to the
        * segment level.
        */
-      class Cell
+      class Cell : Serializable<CellProto>
       {
       private:
         std::vector< Segment >   _segments;     // both 'active' and 'inactive' segments
@@ -252,6 +254,14 @@ namespace nupic {
           return buff.str().size();
         }
 
+        //----------------------------------------------------------------------
+        using Serializable::write;
+        virtual void write(CellProto::Builder& proto) const override;
+
+        //----------------------------------------------------------------------
+        using Serializable::read;
+        virtual void read(CellProto::Reader& proto) override;
+
         //----------------------------------------------------------------------
         void save(std::ostream& outStream) const;
 

src/nupic/algorithms/Cells4.cpp

@@ -35,14 +35,14 @@
 #include <assert.h>
 #include <cstring>
 #include <map>
-#include <nupic/math/ArrayAlgo.hpp> // is_in
-#include <nupic/math/StlIo.hpp> // binary_save
+#include <nupic/algorithms/Cell.hpp>
 #include <nupic/algorithms/Cells4.hpp>
 #include <nupic/algorithms/SegmentUpdate.hpp>
-#include <nupic/algorithms/Cell.hpp>
+#include <nupic/math/ArrayAlgo.hpp> // is_in
+#include <nupic/math/StlIo.hpp> // binary_save
 #include <nupic/os/FStream.hpp>
-
 #include <nupic/os/Timer.hpp>
+#include <nupic/proto/Cells4.capnp.h>
 
 using namespace nupic::algorithms::Cells4;
 
@@ -1931,6 +1931,148 @@ void Cells4::reset()
   //}
 }
 
+
+//--------------------------------------------------------------------------------
+void Cells4::write(Cells4Proto::Builder& proto) const
+{
+  proto.setVersion(version());
+  proto.setOwnsMemory(_ownsMemory);
+  auto randomProto = proto.initRng();
+  _rng.write(randomProto);
+  proto.setNColumns(_nColumns);
+  proto.setNCellsPerCol(_nCellsPerCol);
+  proto.setActivationThreshold(_activationThreshold);
+  proto.setMinThreshold(_minThreshold);
+  proto.setNewSynapseCount(_newSynapseCount);
+  proto.setNIterations(_nIterations);
+  proto.setNLrnIterations(_nLrnIterations);
+  proto.setSegUpdateValidDuration(_segUpdateValidDuration);
+  proto.setInitSegFreq(_initSegFreq);
+  proto.setPermInitial(_permInitial);
+  proto.setPermConnected(_permConnected);
+  proto.setPermMax(_permMax);
+  proto.setPermDec(_permDec);
+  proto.setPermInc(_permInc);
+  proto.setGlobalDecay(_globalDecay);
+  proto.setDoPooling(_doPooling);
+  proto.setPamLength(_pamLength);
+  proto.setMaxInfBacktrack(_maxInfBacktrack);
+  proto.setMaxLrnBacktrack(_maxLrnBacktrack);
+  proto.setMaxSeqLength(_maxSeqLength);
+  proto.setLearnedSeqLength(_learnedSeqLength);
+  proto.setAvgLearnedSeqLength(_avgLearnedSeqLength);
+  proto.setMaxAge(_maxAge);
+  proto.setVerbosity(_verbosity);
+  proto.setMaxSegmentsPerCell(_maxSegmentsPerCell);
+  proto.setMaxSynapsesPerSegment(_maxSynapsesPerSegment);
+  proto.setCheckSynapseConsistency(_checkSynapseConsistency);
+  proto.setResetCalled(_resetCalled);
+  proto.setAvgInputDensity(_avgInputDensity);
+  proto.setPamCounter(_pamCounter);
+
+  auto learnActiveStateTProto = proto.initLearnActiveStateT();
+  _learnActiveStateT.write(learnActiveStateTProto);
+  auto learnActiveStateT1Proto = proto.initLearnActiveStateT1();
+  _learnActiveStateT1.write(learnActiveStateT1Proto);
+  auto learnPredictedStateTProto = proto.initLearnPredictedStateT();
+  _learnPredictedStateT.write(learnPredictedStateTProto);
+  auto learnPredictedStateT1Proto = proto.initLearnPredictedStateT1();
+  _learnPredictedStateT1.write(learnPredictedStateT1Proto);
+
+  auto cellListProto = proto.initCells(_nCells);
+  for (UInt i = 0; i < _nCells; ++i)
+  {
+    auto cellProto = cellListProto[i];
+    _cells[i].write(cellProto);
+  }
+
+  auto segmentUpdatesListProto = proto.initSegmentUpdates(
+      _segmentUpdates.size());
+  for (UInt i = 0; i < _segmentUpdates.size(); ++i)
+  {
+    auto segmentUpdateProto = segmentUpdatesListProto[i];
+    _segmentUpdates[i].write(segmentUpdateProto);
+  }
+}
+
+
+//--------------------------------------------------------------------------------
+void Cells4::read(Cells4Proto::Reader& proto)
+{
+  NTA_CHECK(proto.getVersion() == 2);
+  _ownsMemory = proto.getOwnsMemory();
+  auto randomProto = proto.getRng();
+  _rng.read(randomProto);
+  _nColumns = proto.getNColumns();
+  _nCellsPerCol = proto.getNCellsPerCol();
+  _activationThreshold = proto.getActivationThreshold();
+  _minThreshold = proto.getMinThreshold();
+  _newSynapseCount = proto.getNewSynapseCount();
+  _nIterations = proto.getNIterations();
+  _nLrnIterations = proto.getNLrnIterations();
+  _segUpdateValidDuration = proto.getSegUpdateValidDuration();
+  _initSegFreq = proto.getInitSegFreq();
+  _permInitial = proto.getPermInitial();
+  _permConnected = proto.getPermConnected();
+  _permMax = proto.getPermMax();
+  _permDec = proto.getPermDec();
+  _permInc = proto.getPermInc();
+  _globalDecay = proto.getGlobalDecay();
+  _doPooling = proto.getDoPooling();
+  _pamLength = proto.getPamLength();
+  _maxInfBacktrack = proto.getMaxInfBacktrack();
+  _maxLrnBacktrack = proto.getMaxLrnBacktrack();
+  _maxSeqLength = proto.getMaxSeqLength();
+  _learnedSeqLength = proto.getLearnedSeqLength();
+  _avgLearnedSeqLength = proto.getAvgLearnedSeqLength();
+  _maxAge = proto.getMaxAge();
+  _verbosity = proto.getVerbosity();
+  _maxSegmentsPerCell = proto.getMaxSegmentsPerCell();
+  _maxSynapsesPerSegment = proto.getMaxSynapsesPerSegment();
+  _checkSynapseConsistency = proto.getCheckSynapseConsistency();
+
+  _resetCalled = proto.getResetCalled();
+
+  _avgInputDensity = proto.getAvgInputDensity();
+  _pamCounter = proto.getPamCounter();
+
+  auto learnActiveStateTProto = proto.getLearnActiveStateT();
+  _learnActiveStateT.read(learnActiveStateTProto);
+  auto learnActiveStateT1Proto = proto.getLearnActiveStateT1();
+  _learnActiveStateT1.read(learnActiveStateT1Proto);
+  auto learnPredictedStateTProto = proto.getLearnPredictedStateT();
+  _learnPredictedStateT.read(learnPredictedStateTProto);
+  auto learnPredictedStateT1Proto = proto.getLearnPredictedStateT1();
+  _learnPredictedStateT1.read(learnPredictedStateT1Proto);
+
+  auto cellListProto = proto.getCells();
+  _nCells = cellListProto.size();
+  _cells.resize(_nCells);
+  for (UInt i = 0; i < cellListProto.size(); ++i)
+  {
+    auto cellProto = cellListProto[i];
+    _cells[i].read(cellProto);
+  }
+
+  auto segmentUpdatesListProto = proto.getSegmentUpdates();
+  _segmentUpdates.clear();
+  _segmentUpdates.resize(segmentUpdatesListProto.size());
+  for (UInt i = 0; i < segmentUpdatesListProto.size(); ++i)
+  {
+    auto segmentUpdateProto = segmentUpdatesListProto[i];
+    _segmentUpdates[i].read(segmentUpdateProto);
+  }
+
+  rebuildOutSynapses();
+  if (_checkSynapseConsistency || (_nCells * _maxSegmentsPerCell < 100000))
+  {
+    NTA_CHECK(invariants(true));
+  }
+
+  _version = VERSION;
+}
+
+
 //--------------------------------------------------------------------------------
 void Cells4::save(std::ostream& outStream) const
 {

src/nupic/algorithms/Cells4.hpp

@@ -26,9 +26,11 @@
 #include <ostream>
 #include <sstream>
 #include <fstream>
-#include <nupic/types/Types.hpp>
 #include <nupic/algorithms/Segment.hpp>
 #include <nupic/algorithms/OutSynapse.hpp>
+#include <nupic/proto/Cells4.capnp.h>
+#include <nupic/types/Serializable.hpp>
+#include <nupic/types/Types.hpp>
 #include <queue>
 #include <cstring>
 
@@ -262,7 +264,7 @@ namespace nupic {
         CBasicActivity<It> _seg;
       };
 
-      class Cells4
+      class Cells4 : public Serializable<Cells4Proto>
       {
       public:
 
@@ -1073,6 +1075,20 @@ namespace nupic {
           return tmp.str().size();
         }
 
+        //----------------------------------------------------------------------
+        /**
+         * Write the state to a proto or file
+         */
+        using Serializable::write;
+        virtual void write(Cells4Proto::Builder& proto) const override;
+
+        //----------------------------------------------------------------------
+        /**
+         * Read the state into a proto or file
+         */
+        using Serializable::read;
+        virtual void read(Cells4Proto::Reader& proto) override;
+
         //----------------------------------------------------------------------
         /**
          * Save the state to the given file