initial version of GoDec algorithm

.gitignore

@@ -0,0 +1,32 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app

README.md

@@ -0,0 +1,3 @@
+This repository contains a C++ implementation of the GoDec algorithm for decomposing a matrix into low-rank and sparse components.  See [T. Zhou and D. Tao., _Godec: Randomized Low-Rank & Sparse Matrix Decomposition in Noisy Case_](http://www.icml-2011.org/papers/41_icmlpaper.pdf).  The algorithm has been used for compression of deep neural networks, whose weight tensors can often be approximated well as sums of low-rank and sparse tensors.  See [Yu et al., _On Compressing Deep Models by Low Rank and Sparse Decomposition_](https://pdfs.semanticscholar.org/7551/cc8e398b34cba3ec46569326273da3e9a3df.pdf).
+
+This code requires the [Eigen](eigen.tuxfamily.org/) linear algebra library.

include/godec.h

@@ -0,0 +1,8 @@
+#ifndef __GODEC_H_INCLUDED__
+#define __GODEC_H_INCLUDED__
+
+#include <eigen3/Eigen/Dense>
+
+Eigen::MatrixXd godec(const Eigen::MatrixXd &x, int r, int k, int num_iterations);
+
+#endif

src/godec.cc

@@ -0,0 +1,86 @@
+#include "godec.h"
+
+#include <iostream>
+#include <queue>
+#include <eigen3/Eigen/Dense>
+#include <eigen3/Eigen/SVD>
+
+Eigen::MatrixXd reducerank(const Eigen::MatrixXd &m, int r)
+{
+  Eigen::BDCSVD<Eigen::MatrixXd> decomposition(m, Eigen::ComputeThinU | Eigen::ComputeThinV);
+  Eigen::MatrixXd s = decomposition.singularValues().asDiagonal();
+  Eigen::MatrixXd u = decomposition.matrixU();
+  Eigen::MatrixXd v = decomposition.matrixV();
+  Eigen::MatrixXd uslice = u.block(0, 0, u.rows(), r);
+  Eigen::MatrixXd vslice = v.block(0, 0, v.rows(), r);
+  Eigen::MatrixXd sslice = s.block(0, 0, r, r);
+  Eigen::MatrixXd approx = uslice*sslice*vslice.transpose();
+  std::cout << "Original:" << std::endl << m << std::endl;
+  std::cout << "U:" << std::endl << u << std::endl;
+  std::cout << "V:" << std::endl << v << std::endl;
+  std::cout << "S:" << std::endl << s << std::endl;
+  std::cout << "recovered:" << std::endl << u*s*v.transpose() << std::endl;
+  std::cout << "U slice:" << std::endl << uslice << std::endl;
+  std::cout << "V slice:" << std::endl << vslice << std::endl;
+  std::cout << "S slice:" << std::endl << sslice << std::endl;
+  std::cout << "approximation:" << std::endl << approx << std::endl;
+
+  return approx;
+}
+
+class MatElt {
+public:
+  std::pair<int, int> index;
+  float val;
+  MatElt(std::pair<int, int> p, float x) : index(p), val(x) { }
+};
+
+Eigen::MatrixXd thresh(const Eigen::MatrixXd &x, int k)
+{
+  auto cmp = [](MatElt left, MatElt right) { return (left.val < right.val);};
+  std::priority_queue<MatElt, std::vector<MatElt>, decltype(cmp)> q(cmp);
+  for (int i = 0; i < x.rows(); i++)
+    {
+      for (int j = 0; j < x.cols(); j++)
+        {
+          q.push(MatElt(std::make_pair(i, j), x(i, j)));
+        }
+    }
+
+  Eigen::MatrixXd a = Eigen::MatrixXd::Zero(x.rows(), x.cols());
+  for (int i = 0; i < k; i++)
+    {
+      MatElt elt = q.top();
+      q.pop();
+      auto f = elt.index;
+      float s = elt.val;
+      a(f.first, f.second) = s;
+    }
+
+  std::cout << "a:" << std::endl << a;
+
+  return a;
+}
+
+Eigen::MatrixXd godec(const Eigen::MatrixXd &x, int r, int k, int num_iterations)
+{
+  Eigen::MatrixXd l(x.rows(), x.cols());
+  Eigen::MatrixXd s(x.rows(), x.cols());
+
+  for (int i = 0; i < x.rows(); i++)
+    for (int j = 0; j < x.cols(); j++)
+    {
+      {
+        l(i, j) = x(i, j);
+        s(i, j) = 0.0;
+      }
+    }
+
+  for (int i = 0; i < num_iterations; i++)
+    {
+      l = reducerank(x - s, r);
+      s = thresh(x - l, k);
+    }
+
+  return l + s;
+}

src/main.cc

@@ -0,0 +1,31 @@
+#include "godec.h"
+
+#include <iostream>
+#include <eigen3/Eigen/SVD>
+
+Eigen::MatrixXd testmatrix() {
+  Eigen::MatrixXd x(4, 3);
+  x(0, 0) = .5;
+  x(0, 1) = 0;
+  x(0, 2) = 0;
+  x(1, 0) = 0;
+  x(1, 1) = .8;
+  x(1, 2) = 0;
+  x(2, 0) = 0;
+  x(2, 1) = 0;
+  x(2, 2) = 0;
+  x(3, 0) = .2;
+  x(3, 1) = 0;
+  x(3, 2) = 0;
+  return x;
+}
+
+int main()
+{
+  Eigen::MatrixXd x = testmatrix();
+  int r = 1;
+  int k = 1;
+  int num_iterations = 20;
+  auto result = godec(x, r, k, num_iterations);
+  std::cout << "godec result is " << std::endl << result << std::endl;
+}