Improve exporting of matrices

README.md

@@ -73,13 +73,14 @@ Argument | Default | Description
 ------------ | ------------- | -------------
 updateRHS | true | whether to copy the system matrix to device on every solver call
 updateInitGuess | false |whether to copy the initial guess to device on every solver call
-export | false | write the complete system to disk
-verbose | 0 | print out extra info
+verbose | 0 | print out extra info. Valid values (0-2)
 executor | reference | the executor where to solve the system matrix, other options are `omp`, `cuda`
 adaptMinIter | true | based on the previous solution set minIter to be relaxationFactor*previousIters
 relaxationFactor | 0.8 | use relaxationFactor*previousIters as new minIters
 scaling | 1.0 | Scale the complete system by the scaling factor
 forceHostBuffer  | false | whether to copy to host before MPI calls
+export | false | write the complete system (matrix and rhs) to disk as .mtx file using controlDict/writeControl
+writeGlobal | false | convert all indices to global indices
 
 ### Supported Solver
 Currently, the following solver are supported

include/OGL/common.H

@@ -107,10 +107,6 @@ void export_mtx(const word fieldName,
                 std::shared_ptr<const gko::matrix::Coo<scalar, label>> A,
                 const objectRegistry &db);
 
-void export_system(const word fieldName, const gko::matrix::Csr<scalar> *A,
-                   const gko::matrix::Dense<scalar> *x,
-                   const gko::matrix::Dense<scalar> *b, const word time);
-
 void export_vec(const word fieldName, const gko::matrix::Dense<scalar> *x,
                 const objectRegistry &db);
 

include/OGL/lduLduBase.H

@@ -232,7 +232,7 @@ public:
 
         PersistentVector<scalar> dist_x{
             psi.begin(),
-            this->fieldName() + "_solution",
+            this->fieldName() + "_initial_guess",
             db_,
             exec_handler_,
             dist_A_v,
@@ -265,11 +265,12 @@ public:
             solver_controls_.lookupOrDefault<Switch>("export", false));
         if (export_system && db_.time().writeTime()) {
             bool write_global(
-                solver_controls_.lookupOrDefault<Switch>("writeGlobal", false));
+                solver_controls_.lookupOrDefault<Switch>("writeGlobal", true));
             LOG_0(verbose_, "Export system")
-            // dist_b.write();
             write_distributed(exec_handler_, this->fieldName(), db_, dist_A_v,
                               write_global);
+            dist_b.write();
+            dist_x.write();
         }
 
         LOG_1(verbose_, "start create solver")

src/MatrixWrapper/Distributed.C

@@ -67,12 +67,30 @@ void RepartDistMatrix::write(const ExecutorHandler &exec_handler,
             ->convert_to(non_local.get());
     }
 
-    // overwrite column indices with global indices
     if (write_global) {
+        // overwrite non_local column indices with global indices
         std::copy(non_local_sparsity_->col_idxs.get_const_data(),
                   non_local_sparsity_->col_idxs.get_const_data() +
                       non_local_sparsity_->num_nnz,
                   non_local->get_col_idxs());
+
+        auto ref_exec = exec_handler.get_ref_exec();
+        auto comm = exec_handler.get_gko_mpi_host_comm();
+        label rank{exec_handler.get_rank()};
+        auto partition = gko::share(
+            gko::experimental::distributed::build_partition_from_local_size<label, label>(ref_exec, *comm.get(), local_sparsity_->dim[0]));
+
+        label offset = partition->get_range_bounds()[rank];
+        label local_nnz = local_sparsity_->num_nnz;
+
+        std::transform(local->get_row_idxs(), local->get_row_idxs() + local_nnz, local->get_row_idxs(),
+                    [&](label idx) { return idx + offset; });
+        std::transform(local->get_col_idxs(), local->get_col_idxs() + local_nnz, local->get_col_idxs(),
+                    [&](label idx) { return idx + offset; });
+
+        label non_local_nnz = non_local_sparsity_->num_nnz;
+        std::transform(non_local->get_row_idxs(), non_local->get_row_idxs() + non_local_nnz, non_local->get_row_idxs(),
+                    [&](label idx) { return idx + offset; });
     }
 
     export_mtx(field_name + "_local", local, db);

src/common.C

@@ -21,36 +21,14 @@ void export_x(const std::string fn, const gko::matrix::Dense<scalar> *x)
     gko::write(stream_x, x);
 }
 
-void export_x(const std::string fn, const gko::matrix::Csr<scalar> *A)
-{
-    LOG_1(1, "Writing " + fn)
-    std::ofstream stream{fn};
-    gko::write(stream, A, gko::layout_type::coordinate);
-}
-
 void export_vec(const word fieldName, const gko::matrix::Dense<scalar> *x,
                 const objectRegistry &db)
 {
     std::string folder{db.time().timePath()};
-    std::string fn{folder + "/" + fieldName + "_b_.mtx"};
+    std::string fn{folder + "/" + fieldName + ".mtx"};
     export_x(fn, x);
 }
 
-void export_system(const word fieldName, const gko::matrix::Csr<scalar> *A,
-                   const gko::matrix::Dense<scalar> *x,
-                   const gko::matrix::Dense<scalar> *b, const word time)
-{
-    system("mkdir -p export/" + time);
-    std::string fn_mtx{"export/" + time + "/" + fieldName + "_A.mtx"};
-    export_x(fn_mtx, A);
-
-    std::string fn_b{"export/" + time + "/" + fieldName + "_b.mtx"};
-    export_x(fn_b, b);
-
-    std::string fn_x{"export/" + time + "/" + fieldName + "_x0.mtx"};
-    export_x(fn_x, x);
-}
-
 void export_mtx(const word fieldName,
                 std::shared_ptr<const gko::matrix::Coo<scalar, label>> A,
                 const objectRegistry &db)