From 8c17d9f19c8e8b473dc3a19159b7ff92eb224697 Mon Sep 17 00:00:00 2001 From: Robert Smith Date: Mon, 11 May 2020 11:45:26 +1000 Subject: [PATCH 1/4] fixelconnectivity: Enhancements - Option -tck_weights_in now supported, allowing CFE to be performed using SIFT2. Generation of the fixel-fixel connectivity matrix will be done using floating-point if such weights are provided, integers otherwise. - New options -count and -extent export fixel data files corresponding to the number of connections, and sum of connection weights, of each fixel respectively. - Modify error message on pre-existing output directory, as output target is not a file. --- cmd/fixelconnectivity.cpp | 59 ++++- core/app.h | 2 +- core/fixel/helpers.h | 16 +- core/fixel/types.h | 2 + docs/reference/commands/fixelconnectivity.rst | 9 + src/dwi/tractography/weights.h | 1 - src/fixel/filter/smooth.h | 2 + src/fixel/matrix.cpp | 200 ++++++++++------ src/fixel/matrix.h | 214 +++++++++++++----- 9 files changed, 361 insertions(+), 144 deletions(-) diff --git a/cmd/fixelconnectivity.cpp b/cmd/fixelconnectivity.cpp index 19ea2e5d38..be2fd03dff 100644 --- a/cmd/fixelconnectivity.cpp +++ b/cmd/fixelconnectivity.cpp @@ -15,10 +15,12 @@ #include "command.h" #include "fixel/helpers.h" -#include "fixel/index_remapper.h" + +#include "dwi/tractography/weights.h" #include "fixel/matrix.h" + #define DEFAULT_ANGLE_THRESHOLD 45.0 #define DEFAULT_CONNECTIVITY_THRESHOLD 0.01 @@ -55,7 +57,17 @@ void usage () + Argument ("value").type_float (0.0, 90.0) + Option ("mask", "provide a fixel data file containing a mask of those fixels to be computed; fixels outside the mask will be empty in the output matrix") - + Argument ("file").type_image_in(); + + Argument ("file").type_image_in() + + + DWI::Tractography::TrackWeightsInOption + + + OptionGroup ("Options for additional outputs to be generated") + + + Option ("count", "export a fixel data file encoding the number of connections for each fixel") + + Argument ("path").type_image_out() + + + Option ("extent", "export a fixel data file encoding the extent of connectivity (sum of weights) for each fixel") + + Argument ("path").type_image_out(); } @@ -64,6 +76,20 @@ using value_type = float; using Fixel::index_type; + +template +void set_optional_outputs (WriterType& writer) +{ + auto opt = get_options ("count"); + if (opt.size()) + writer.set_count_path (opt[0][0]); + opt = get_options ("extent"); + if (opt.size()) + writer.set_extent_path (opt[0][0]); +} + + + void run() { const value_type connectivity_threshold = get_option_value ("connectivity", value_type(DEFAULT_CONNECTIVITY_THRESHOLD)); @@ -91,14 +117,29 @@ void run() fixel_mask.value() = true; } - auto connectivity_matrix = Fixel::Matrix::generate (argument[1], - index_image, - fixel_mask, - angular_threshold); + if (get_options ("tck_weights_in").size()) { + + auto connectivity_matrix = Fixel::Matrix::generate_weighted (argument[1], + index_image, + fixel_mask, + angular_threshold); - Fixel::Matrix::normalise_and_write (connectivity_matrix, - connectivity_threshold, - argument[2]); + Fixel::Matrix::Writer writer (connectivity_matrix, connectivity_threshold); + set_optional_outputs (writer); + writer.save (argument[2]); + + } else { + + auto connectivity_matrix = Fixel::Matrix::generate_unweighted (argument[1], + index_image, + fixel_mask, + angular_threshold); + + Fixel::Matrix::Writer writer (connectivity_matrix, connectivity_threshold); + set_optional_outputs (writer); + writer.save (argument[2]); + + } } diff --git a/core/app.h b/core/app.h index 10442aa90a..71aec4cba4 100644 --- a/core/app.h +++ b/core/app.h @@ -171,7 +171,7 @@ namespace MR if (check_overwrite_files_func) check_overwrite_files_func (name); else - throw Exception ("output file \"" + name + "\" already exists (use -force option to force overwrite)"); + throw Exception ("output path \"" + name + "\" already exists (use -force option to force overwrite)"); } } diff --git a/core/fixel/helpers.h b/core/fixel/helpers.h index 116194a682..8e973d47c2 100644 --- a/core/fixel/helpers.h +++ b/core/fixel/helpers.h @@ -296,14 +296,14 @@ namespace MR return header; } - //! Generate a header for a sparse data file (Nx1x1) using an index image as a template - template - FORCE_INLINE Header data_header_from_index (IndexHeaderType& index) { - Header header (index); + //! Generate a header for a sparse data file (Nx1x1) with a known number of fixels + FORCE_INLINE Header make_data_header (const size_t num_fixels) { + Header header; header.ndim() = 3; - header.size(0) = get_number_of_fixels (index); + header.size(0) = num_fixels; header.size(1) = 1; header.size(2) = 1; + header.spacing(0) = header.spacing(1) = header.spacing(2) = 1.0; header.stride(0) = 1; header.stride(1) = 2; header.stride(2) = 3; @@ -313,6 +313,12 @@ namespace MR return header; } + //! Generate a header for a sparse data file (Nx1x1) using an index image as a template + template + FORCE_INLINE Header data_header_from_index (IndexHeaderType& index) { + return make_data_header (get_number_of_fixels (index)); + } + //! Generate a header for a fixel directions data file (Nx3x1) using an index image as a template template FORCE_INLINE Header directions_header_from_index (IndexHeaderType& index) { diff --git a/core/fixel/types.h b/core/fixel/types.h index 81a06d618e..54434eb7d5 100644 --- a/core/fixel/types.h +++ b/core/fixel/types.h @@ -16,6 +16,8 @@ #ifndef __fixel_types_h__ #define __fixel_types_h__ +#include + namespace MR { namespace Fixel diff --git a/docs/reference/commands/fixelconnectivity.rst b/docs/reference/commands/fixelconnectivity.rst index e98540ced3..eac6f2123c 100644 --- a/docs/reference/commands/fixelconnectivity.rst +++ b/docs/reference/commands/fixelconnectivity.rst @@ -36,6 +36,15 @@ Options that influence generation of the connectivity matrix / matrices - **-mask file** provide a fixel data file containing a mask of those fixels to be computed; fixels outside the mask will be empty in the output matrix +- **-tck_weights_in path** specify a text scalar file containing the streamline weights + +Options for additional outputs to be generated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- **-count path** export a fixel data file encoding the number of connections for each fixel + +- **-extent path** export a fixel data file encoding the extent of connectivity (sum of weights) for each fixel + Standard options ^^^^^^^^^^^^^^^^ diff --git a/src/dwi/tractography/weights.h b/src/dwi/tractography/weights.h index d55a2f878d..e77b884d57 100644 --- a/src/dwi/tractography/weights.h +++ b/src/dwi/tractography/weights.h @@ -23,7 +23,6 @@ namespace MR { namespace DWI { - namespace Tractography { diff --git a/src/fixel/filter/smooth.h b/src/fixel/filter/smooth.h index 25cabe4203..5c3ba91683 100644 --- a/src/fixel/filter/smooth.h +++ b/src/fixel/filter/smooth.h @@ -16,6 +16,8 @@ #ifndef __fixel_filter_smooth_h__ #define __fixel_filter_smooth_h__ +#include "fixel/types.h" + #include "fixel/matrix.h" #include "fixel/filter/base.h" diff --git a/src/fixel/matrix.cpp b/src/fixel/matrix.cpp index 0ae2f6279b..47cfc0885e 100644 --- a/src/fixel/matrix.cpp +++ b/src/fixel/matrix.cpp @@ -37,19 +37,12 @@ namespace MR - void InitFixel::add (const vector& indices) + template + void InitFixelBase::add (const MappedTrack& mapped_track) { - if ((*this).empty()) { - (*this).reserve (indices.size()); - for (auto i : indices) - (*this).emplace_back (InitElement (i)); - track_count = 1; - return; - } - ssize_t self_index = 0, in_index = 0; - // For anything in indices that doesn't yet appear in *this, + // For anything in mapped_track that doesn't yet appear in *this, // add to this list; once completed, extend *this by the appropriate // amount, and insert these into the appropriate locations // Need to continue making use of the existing allocated memory @@ -60,15 +53,15 @@ namespace MR // - On second pass, from back to front, move elements from previous back of vector to new back, // inserting new elements at appropriate locations to retain sortedness of list const ssize_t old_size = (*this).size(); - const ssize_t in_count = indices.size(); + const ssize_t in_count = mapped_track.size(); size_t intersection = 0; while (self_index < old_size && in_index < in_count) { - if ((*this)[self_index].index() == indices[in_index]) { - ++(*this)[self_index]; + if ((*this)[self_index].index() == mapped_track[in_index]) { + increment ((*this)[self_index], mapped_track); ++self_index; ++in_index; ++intersection; - } else if ((*this)[self_index].index() > indices[in_index]) { + } else if ((*this)[self_index].index() > mapped_track[in_index]) { ++in_index; } else { ++self_index; @@ -76,58 +69,53 @@ namespace MR } self_index = old_size - 1; - in_index = indices.size() - 1; + in_index = mapped_track.size() - 1; // It's possible that a resize() call may always result in requesting // a re-assignment of memory that exactly matches the size, which may in turn // lead to memory bloat due to inability to return the old memory // If this occurs, iteratively calling push_back() may instead engage the // memory-reservation-doubling behaviour - while ((*this).size() < old_size + indices.size() - intersection) - (*this).push_back (InitElement()); + while ((*this).size() < old_size + mapped_track.size() - intersection) + (*this).push_back (ElementType()); ssize_t out_index = (*this).size() - 1; // For each output vector location, need to determine whether it should come from copying an existing entry, // or creating a new one while (out_index > self_index && self_index >= 0 && in_index >= 0) { - if ((*this)[self_index].index() == indices[in_index]) { + if ((*this)[self_index].index() == mapped_track[in_index]) { (*this)[out_index] = (*this)[self_index]; --self_index; --in_index; - } else if ((*this)[self_index].index() > indices[in_index]) { + } else if ((*this)[self_index].index() > mapped_track[in_index]) { (*this)[out_index] = (*this)[self_index]; --self_index; } else { - (*this)[out_index] = InitElement (indices[in_index]); + (*this)[out_index] = ElementType (mapped_track[in_index], mapped_track); --in_index; } --out_index; } if (self_index < 0) { while (in_index >= 0 && out_index >= 0) - (*this)[out_index--] = InitElement (indices[in_index--]); + (*this)[out_index--] = ElementType (mapped_track[in_index--], mapped_track); } - // Track total number of streamlines intersecting this fixel, + // Track total number of streamlines / sum of streamline weights intersecting this fixel, // independently of the extent of fixel-fixel connectivity - ++track_count; + increment (mapped_track); } + template class InitFixelBase; + template class InitFixelBase; - - - init_matrix_type generate ( - const std::string& track_filename, - Image& index_image, - Image& fixel_mask, - const float angular_threshold) + namespace { - class TrackProcessor { MEMALIGN(TrackProcessor) public: @@ -143,7 +131,7 @@ namespace MR angular_threshold_dp (std::cos (angular_threshold * (Math::pi/180.0))) { } bool operator() (const DWI::Tractography::Streamline<>& tck, - vector& out) const + MappedTrack& out) const { using direction_type = Eigen::Vector3; using SetVoxelDir = DWI::Tractography::Mapping::SetVoxelDir; @@ -151,9 +139,10 @@ namespace MR SetVoxelDir in; mapper (tck, in); - // For each voxel tract tangent, assign to a fixel + // For each voxel intersection, assign to a fixel out.clear(); out.reserve (in.size()); + out.set_weight (tck.weight); for (const auto& i : in) { assign_pos_of (i).to (fixel_indexer); fixel_indexer.index(3) = 0; @@ -196,33 +185,77 @@ namespace MR }; - auto directions_image = Fixel::find_directions_header (Path::dirname (index_image.name())).template get_image().with_direct_io ({+2,+1}); - DWI::Tractography::Properties properties; - DWI::Tractography::Reader track_file (track_filename, properties); - const uint32_t num_tracks = properties["count"].empty() ? 0 : to(properties["count"]); - DWI::Tractography::Mapping::TrackLoader loader (track_file, num_tracks, "computing fixel-fixel connectivity matrix"); - DWI::Tractography::Mapping::TrackMapperBase mapper (index_image); - mapper.set_upsample_ratio (DWI::Tractography::Mapping::determine_upsample_ratio (index_image, properties, 0.333f)); - mapper.set_use_precise_mapping (true); + + template + class Receiver + { NOMEMALIGN + public: + Receiver (MatrixType& data) : + data (data) { } + bool operator() (const MappedTrack& in) const + { + try { + for (auto f : in) + data[f].add (in); + return true; + } catch (...) { + throw Exception ("Error assigning memory for CFE connectivity matrix"); + return false; + } + } + private: + MatrixType& data; + }; + } + + + +#define FIXEL_MATRIX_GENERATE_SHARED \ + auto directions_image = Fixel::find_directions_header (Path::dirname (index_image.name())).template get_image().with_direct_io ({+2,+1}); \ + DWI::Tractography::Properties properties; \ + DWI::Tractography::Reader track_file (track_filename, properties); \ + const uint32_t num_tracks = properties["count"].empty() ? 0 : to(properties["count"]); \ + DWI::Tractography::Mapping::TrackLoader loader (track_file, num_tracks, "computing fixel-fixel connectivity matrix"); \ + DWI::Tractography::Mapping::TrackMapperBase mapper (index_image); \ + mapper.set_upsample_ratio (DWI::Tractography::Mapping::determine_upsample_ratio (index_image, properties, 0.333f)); \ + mapper.set_use_precise_mapping (true); \ TrackProcessor track_processor (mapper, index_image, directions_image, fixel_mask, angular_threshold); - init_matrix_type connectivity_matrix (Fixel::get_number_of_fixels (index_image)); + + + + InitMatrixUnweighted generate_unweighted ( + const std::string& track_filename, + Image& index_image, + Image& fixel_mask, + const float angular_threshold) + { + FIXEL_MATRIX_GENERATE_SHARED + InitMatrixUnweighted connectivity_matrix (Fixel::get_number_of_fixels (index_image)); + Receiver receiver (connectivity_matrix); + Thread::run_queue (loader, + Thread::batch (DWI::Tractography::Streamline()), + track_processor, + Thread::batch (MappedTrack()), + receiver); + return connectivity_matrix; + } + + + + InitMatrixWeighted generate_weighted ( + const std::string& track_filename, + Image& index_image, + Image& fixel_mask, + const float angular_threshold) + { + FIXEL_MATRIX_GENERATE_SHARED + InitMatrixWeighted connectivity_matrix (Fixel::get_number_of_fixels (index_image)); + Receiver receiver (connectivity_matrix); Thread::run_queue (loader, Thread::batch (DWI::Tractography::Streamline()), track_processor, - Thread::batch (vector()), - // Inline lambda function for receiving streamline fixel visitations and - // updating the connectivity matrix - [&] (const vector& fixels) - { - try { - for (auto f : fixels) - connectivity_matrix[f].add (fixels); - return true; - } catch (...) { - throw Exception ("Error assigning memory for CFE connectivity matrix"); - return false; - } - }); + Thread::batch (MappedTrack()), + receiver); return connectivity_matrix; } @@ -230,14 +263,34 @@ namespace MR - void normalise_and_write (init_matrix_type& matrix, - const connectivity_value_type threshold, - const std::string& path, - const KeyValues& keyvals) + + template + void Writer::set_count_path (const std::string& path) + { + assert (!count_image.valid()); + count_image = Image::create (path, MR::Fixel::make_data_header (matrix.size())); + } + + template + void Writer::set_extent_path (const std::string& path) { + assert (!extent_image.valid()); + extent_image = Image::create (path, MR::Fixel::make_data_header (matrix.size())); + } + + + template + void Writer::save (const std::string& path) const + { if (Path::exists (path)) { - if (!Path::is_dir (path)) { + if (Path::is_dir (path)) { + if (!App::overwrite_files && (Path::is_file (Path::join (path, "index.mif")) || + Path::is_file (Path::join (path, "fixels.mif")) || + Path::is_file (Path::join (path, "values.mif")))) + throw Exception ("Cannot create fixel-fixel connectivity matrix \"" + path + "\": " + "one or more files already exists (use -force to override)"); + } else { if (App::overwrite_files) { File::remove (path); } else { @@ -263,6 +316,7 @@ namespace MR index_header.keyval() = keyvals; index_header.keyval()["nfixels"] = str(matrix.size()); index_header.datatype() = DataType::from(); + index_header.datatype().set_byte_order_native(); Image index_image = Image::create (Path::join (path, "index.mif"), index_header); // Can't use function write_mrtrix_header() as the file offset of the @@ -287,7 +341,7 @@ namespace MR if (stream_index) stream << DataType::from().specifier(); else - stream << DataType::from().specifier(); + stream << DataType::from().specifier(); stream << transform_type::Identity().matrix().topLeftCorner(3,4).format(fmt) << "\n"; stream << "scaling: 0,1\n"; stream << "nfixels: " + str(matrix.size()) + "\n"; @@ -301,7 +355,7 @@ namespace MR ProgressBar progress ("Normalising and writing fixel-fixel connectivity matrix to directory \"" + path + "\"", matrix.size()); size_t data_count = 0; - vector fixel_buffer; + vector fixel_buffer; vector value_buffer; for (size_t fixel_index = 0; fixel_index != matrix.size(); ++fixel_index) { @@ -310,12 +364,14 @@ namespace MR fixel_buffer.reserve (matrix[fixel_index].size()); value_buffer.reserve (matrix[fixel_index].size()); - const connectivity_value_type normalisation_factor = connectivity_value_type(1) / connectivity_value_type (matrix[fixel_index].count()); + const connectivity_value_type normalisation_factor = matrix[fixel_index].norm_factor(); + connectivity_value_type sum_connectivity = connectivity_value_type(0); for (auto& it : matrix[fixel_index]) { const connectivity_value_type connectivity = normalisation_factor * it.value(); if (connectivity >= threshold) { fixel_buffer.push_back (it.index()); value_buffer.push_back (connectivity); + sum_connectivity += connectivity; } } @@ -323,13 +379,22 @@ namespace MR index_image.index (3) = 0; index_image.value() = uint64_t(fixel_buffer.size()); index_image.index (3) = 1; index_image.value() = fixel_buffer.size() ? data_count : uint64_t(0); - fixel_stream.write (reinterpret_cast(fixel_buffer.data()), fixel_buffer.size() * sizeof (index_type)); + fixel_stream.write (reinterpret_cast(fixel_buffer.data()), fixel_buffer.size() * sizeof (fixel_index_type)); value_stream.write (reinterpret_cast(value_buffer.data()), value_buffer.size() * sizeof (connectivity_value_type)); data_count += fixel_buffer.size(); + if (count_image.valid()) { + count_image.index(0) = fixel_index; + count_image.value() = fixel_buffer.size(); + } + if (extent_image.valid()) { + extent_image.index(0) = fixel_index; + extent_image.value() = sum_connectivity; + } + // Force deallocation of memory used for this fixel in the generated matrix - InitFixel().swap (matrix[fixel_index]); + typename MatrixType::value_type().swap (matrix[fixel_index]); ++progress; } @@ -345,6 +410,9 @@ namespace MR } + template class Writer; + template class Writer; + diff --git a/src/fixel/matrix.h b/src/fixel/matrix.h index 323b80d20d..a721c37b1b 100644 --- a/src/fixel/matrix.h +++ b/src/fixel/matrix.h @@ -19,66 +19,158 @@ #include "image.h" #include "types.h" #include "file/ofstream.h" -#include "fixel/index_remapper.h" +#include "fixel/types.h" namespace MR { namespace Fixel { - - namespace Matrix { + using index_image_type = uint64_t; - using fixel_index_type = uint32_t; + using fixel_index_type = MR::Fixel::index_type; using count_type = uint32_t; using connectivity_value_type = float; - // Classes for dealing with dynamic multi-threaded construction of the - // fixel-fixel connectivity matrix - class InitElement + class MappedTrack : public vector + { NOMEMALIGN + public: + using BaseType = vector; + default_type get_weight() const { return weight; } + void set_weight (const default_type w) { weight = w; } + private: + default_type weight; + }; + + + + class InitElementBase + { NOMEMALIGN + public: + InitElementBase() : + fixel_index (std::numeric_limits::max()) { } + InitElementBase (const fixel_index_type fixel_index) : + fixel_index (fixel_index) { } + InitElementBase (const InitElementBase&) = default; + FORCE_INLINE InitElementBase& operator= (const InitElementBase& that) { fixel_index = that.fixel_index; return *this; } + FORCE_INLINE fixel_index_type index() const { return fixel_index; } + FORCE_INLINE bool operator< (const InitElementBase& that) const { return fixel_index < that.fixel_index; } + private: + fixel_index_type fixel_index; + }; + + + + class InitElementUnweighted : private InitElementBase { NOMEMALIGN public: - using ValueType = fixel_index_type; - InitElement() : - fixel_index (std::numeric_limits::max()), + using BaseType = InitElementBase; + using BaseType::operator<; + using ValueType = count_type; + InitElementUnweighted() : track_count (0) { } - InitElement (const fixel_index_type fixel_index) : - fixel_index (fixel_index), + InitElementUnweighted (const fixel_index_type fixel_index) : + BaseType (fixel_index), track_count (1) { } - InitElement (const fixel_index_type fixel_index, const ValueType track_count) : - fixel_index (fixel_index), - track_count (track_count) { } - InitElement (const InitElement&) = default; - FORCE_INLINE InitElement& operator++() { track_count++; return *this; } - FORCE_INLINE InitElement& operator= (const InitElement& that) { fixel_index = that.fixel_index; track_count = that.track_count; return *this; } - FORCE_INLINE fixel_index_type index() const { return fixel_index; } + InitElementUnweighted (const fixel_index_type fixel_index, const MappedTrack& all_data) : + BaseType (fixel_index), + track_count (1) { } + InitElementUnweighted (const InitElementUnweighted&) = default; + FORCE_INLINE fixel_index_type index() const { return BaseType::index(); } + FORCE_INLINE InitElementUnweighted& operator++() { track_count++; return *this; } + FORCE_INLINE InitElementUnweighted& operator= (const InitElementUnweighted& that) { BaseType::operator= (that); track_count = that.track_count; return *this; } FORCE_INLINE ValueType value() const { return track_count; } - FORCE_INLINE bool operator< (const InitElement& that) const { return fixel_index < that.fixel_index; } private: - fixel_index_type fixel_index; ValueType track_count; }; - class InitFixel : public vector - { MEMALIGN(InitFixel) + class InitElementWeighted : private InitElementBase + { NOMEMALIGN + public: + using BaseType = InitElementBase; + using BaseType::operator<; + using ValueType = connectivity_value_type; + InitElementWeighted() : + sum_weights (connectivity_value_type(0)) { } + InitElementWeighted (const fixel_index_type fixel_index) = delete; + InitElementWeighted (const fixel_index_type fixel_index, const MappedTrack& all_data) : + BaseType (fixel_index), + sum_weights (all_data.get_weight()) { } + InitElementWeighted (const InitElementWeighted&) = default; + FORCE_INLINE fixel_index_type index() const { return BaseType::index(); } + FORCE_INLINE InitElementWeighted& operator+= (const connectivity_value_type increment) { sum_weights += increment; return *this; } + FORCE_INLINE InitElementWeighted& operator= (const InitElementWeighted& that) { BaseType::operator= (that); sum_weights = that.sum_weights; return *this; } + FORCE_INLINE ValueType value() const { return sum_weights; } + private: + connectivity_value_type sum_weights; + }; + + + + template + class InitFixelBase : public vector + { NOMEMALIGN + public: + using BaseType = vector; + virtual ~InitFixelBase() { } + void add (const MappedTrack& mapped_track); + virtual default_type norm_factor() const = 0; + protected: + virtual void increment (const MappedTrack& data) = 0; + virtual void increment (ElementType& element, const MappedTrack& data) = 0; + }; + + class InitFixelUnweighted : public InitFixelBase + { NOMEMALIGN public: - using ElementType = InitElement; - using BaseType = vector; - InitFixel() : + using BaseType = InitFixelBase; + InitFixelUnweighted() : track_count (0) { } - void add (const vector& indices); - count_type count() const { return track_count; } + default_type norm_factor() const override { return 1.0 / default_type(track_count); } private: count_type track_count; + void increment (const MappedTrack& data) override { ++track_count; } + void increment (InitElementUnweighted& element, const MappedTrack& data) override { ++element; } }; + class InitFixelWeighted : public InitFixelBase + { NOMEMALIGN + public: + using BaseType = InitFixelBase; + InitFixelWeighted() : + sum_weights (default_type (0)) { } + default_type norm_factor() const override { return 1.0 / sum_weights; } + private: + default_type sum_weights; + void increment (const MappedTrack& data) override { sum_weights += data.get_weight(); } + void increment (InitElementWeighted& element, const MappedTrack& data) override { element += data.get_weight(); } + }; + + + + using InitMatrixUnweighted = vector; + using InitMatrixWeighted = vector; + + + + + + + + + + + + + + @@ -88,16 +180,16 @@ namespace MR { NOMEMALIGN public: using ValueType = connectivity_value_type; - NormElement (const index_type fixel_index, + NormElement (const fixel_index_type fixel_index, const ValueType connectivity_value) : fixel_index (fixel_index), connectivity_value (connectivity_value) { } - FORCE_INLINE index_type index() const { return fixel_index; } + FORCE_INLINE fixel_index_type index() const { return fixel_index; } FORCE_INLINE ValueType value() const { return connectivity_value; } FORCE_INLINE void exponentiate (const ValueType C) { connectivity_value = std::pow (connectivity_value, C); } FORCE_INLINE void normalise (const ValueType norm_factor) { connectivity_value *= norm_factor; } private: - index_type fixel_index; + fixel_index_type fixel_index; connectivity_value_type connectivity_value; }; @@ -136,10 +228,7 @@ namespace MR - // Different types are used depending on whether the connectivity matrix - // is in the process of being built, or whether it has been normalised - // TODO Revise - using init_matrix_type = vector; + @@ -147,7 +236,13 @@ namespace MR // Generate a fixel-fixel connectivity matrix - init_matrix_type generate ( + InitMatrixUnweighted generate_unweighted ( + const std::string& track_filename, + Image& index_image, + Image& fixel_mask, + const float angular_threshold); + + InitMatrixWeighted generate_weighted ( const std::string& track_filename, Image& index_image, Image& fixel_mask, @@ -155,35 +250,30 @@ namespace MR + template + class Writer + { MEMALIGN(Writer) + public: + Writer (MatrixType& matrix, + const connectivity_value_type threshold) : + matrix (matrix), + threshold (threshold) { } + void set_keyvals (KeyValues& kv) { keyvals = kv; } + void set_count_path (const std::string& path); + void set_extent_path (const std::string& path); + void save (const std::string& path) const; + private: + MatrixType& matrix; + const connectivity_value_type threshold; + KeyValues keyvals; + mutable Image count_image; + mutable Image extent_image; + }; + + - // New code for handling load/save of fixel-fixel connectivity matrix - // Use something akin to the fixel directory format, with a sub-directory - // within an existing fixel directory containing the following data: - // - index.mif: Similar to the index image in the fixel directory format, - // but has dimensions Nx1x1x2, where: - // - N is the number of fixels in the fixel template - // - The first volume contains the number of connected fixels for that fixel - // - The second volume contains the offset of the first connected fixel for that fixel - // - connectivity.mif: Floating-point image of dimension Cx1x1, where C is the total - // number of fixel-fixel connections stored in the entire matrix. Each value should be - // between 0.0 and 1.0, corresponding to the fraction of streamlines passing through - // the fixel that additionally pass through some other fixel. - // - fixel.mif: Unsigned integer image of dimension Cx1x1, where C is the total number of - // fixel-fixel connections stored in the entire matrix. Each value indexes the - // fixel to which the fixel-fixel connection refers. - // - // In order to avoid duplication of memory usage, the writing function should - // perform, for each fixel in turn: - // - Normalisation of the matrix - // - Writing to the three images - // - Erasing the memory used for that matrix in the initial building - - void normalise_and_write (init_matrix_type& matrix, - const connectivity_value_type threshold, - const std::string& path, - const KeyValues& keyvals = KeyValues()); From be145790d96b18d97e7d85a6a42df582d97889c5 Mon Sep 17 00:00:00 2001 From: Robert Smith Date: Tue, 18 Jul 2023 15:25:44 +1000 Subject: [PATCH 2/4] Resolving compilation after merging dev into fixelconnectivity --- core/fixel/helpers.h | 6 ------ src/fixel/filter/smooth.h | 5 ++--- src/fixel/matrix.cpp | 17 ++++++++++------- src/fixel/matrix.h | 4 ++-- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/core/fixel/helpers.h b/core/fixel/helpers.h index 39b483bfea..e932dec065 100644 --- a/core/fixel/helpers.h +++ b/core/fixel/helpers.h @@ -335,12 +335,6 @@ namespace MR return header; } - //! Generate a header for a sparse data file (Nx1x1) using an index image as a template - template - FORCE_INLINE Header data_header_from_index (IndexHeaderType& index) { - return make_data_header (get_number_of_fixels (index)); - } - //! Generate a header for a fixel directions data file (Nx3x1) using an index image as a template template FORCE_INLINE Header directions_header_from_index (IndexHeaderType& index) { diff --git a/src/fixel/filter/smooth.h b/src/fixel/filter/smooth.h index 2256a20957..7bb5a6c999 100644 --- a/src/fixel/filter/smooth.h +++ b/src/fixel/filter/smooth.h @@ -18,8 +18,7 @@ #ifndef __fixel_filter_smooth_h__ #define __fixel_filter_smooth_h__ -#include "fixel/types.h" - +#include "fixel/fixel.h" #include "fixel/matrix.h" #include "fixel/filter/base.h" @@ -53,7 +52,7 @@ namespace MR */ class Smooth : public Base - { + { public: Smooth (Image index_image, diff --git a/src/fixel/matrix.cpp b/src/fixel/matrix.cpp index 3332cdb47c..d6b04fe88e 100644 --- a/src/fixel/matrix.cpp +++ b/src/fixel/matrix.cpp @@ -190,7 +190,7 @@ namespace MR template class Receiver - { NOMEMALIGN + { public: Receiver (MatrixType& data) : data (data) { } @@ -270,14 +270,14 @@ namespace MR void Writer::set_count_path (const std::string& path) { assert (!count_image.valid()); - count_image = Image::create (path, MR::Fixel::make_data_header (matrix.size())); + count_image = Image::create (path, MR::Fixel::data_header_from_nfixels (matrix.size())); } template void Writer::set_extent_path (const std::string& path) { assert (!extent_image.valid()); - extent_image = Image::create (path, MR::Fixel::make_data_header (matrix.size())); + extent_image = Image::create (path, MR::Fixel::data_header_from_nfixels (matrix.size())); } @@ -322,10 +322,10 @@ namespace MR index_type num_connections = 0; { - ProgressBar progress ("Computing number of fixels in output", matrix.size()); + ProgressBar progress ("Computing number of fixel-fixel connections in matrix", matrix.size()); for (size_t fixel_index = 0; fixel_index != matrix.size(); ++fixel_index) { - const connectivity_value_type normalisation_factor = connectivity_value_type(1) / connectivity_value_type (matrix[fixel_index].count()); + const connectivity_value_type normalisation_factor = matrix[fixel_index].norm_factor(); for (auto& it : matrix[fixel_index]) { const connectivity_value_type connectivity = normalisation_factor * it.value(); if (connectivity >= threshold) @@ -349,6 +349,7 @@ namespace MR fixel_header.keyval()["nfixels"] = str(matrix.size()); fixel_header.datatype() = DataType::from(); fixel_header.datatype().set_byte_order_native(); + Header value_header (fixel_header); value_header.datatype() = DataType::from(); value_header.datatype().set_byte_order_native(); @@ -370,11 +371,13 @@ namespace MR const ssize_t connection_offset = fixel_image.index(0); index_type connection_count = 0; - const connectivity_value_type normalisation_factor = connectivity_value_type(1) / connectivity_value_type (matrix[fixel_index].count()); + connectivity_value_type sum_connectivity = connectivity_value_type (0.0); + const connectivity_value_type normalisation_factor = matrix[fixel_index].norm_factor(); for (auto& it : matrix[fixel_index]) { const connectivity_value_type connectivity = normalisation_factor * it.value(); if (connectivity >= threshold) { ++connection_count; + sum_connectivity += connectivity; fixel_image.value() = it.index(); ++fixel_image.index(0); value_image.value() = connectivity; @@ -388,7 +391,7 @@ namespace MR if (count_image.valid()) { count_image.index(0) = fixel_index; - count_image.value() = fixel_buffer.size(); + count_image.value() = connection_count; } if (extent_image.valid()) { extent_image.index(0) = fixel_index; diff --git a/src/fixel/matrix.h b/src/fixel/matrix.h index c109a08ece..65a234411b 100644 --- a/src/fixel/matrix.h +++ b/src/fixel/matrix.h @@ -21,7 +21,7 @@ #include "image.h" #include "types.h" #include "file/ofstream.h" -#include "fixel/types.h" +#include "fixel/fixel.h" namespace MR { @@ -254,7 +254,7 @@ namespace MR template class Writer - { MEMALIGN(Writer) + { public: Writer (MatrixType& matrix, const connectivity_value_type threshold) : From ab8bf844c7e8eff089b12b0f287aef7070dc8cfc Mon Sep 17 00:00:00 2001 From: Robert Smith Date: Wed, 19 Jul 2023 00:36:37 +1000 Subject: [PATCH 3/4] fixelconnectivity": New tests and test data to reflect enhancements --- testing/binaries/data | 2 +- testing/binaries/tests/fixelconnectivity | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/testing/binaries/data b/testing/binaries/data index 747223c963..309728c7c1 160000 --- a/testing/binaries/data +++ b/testing/binaries/data @@ -1 +1 @@ -Subproject commit 747223c9635094939d55dd6c639a6e07c1c84427 +Subproject commit 309728c7c12ac0e81eb9e572eda290b45f952f4c diff --git a/testing/binaries/tests/fixelconnectivity b/testing/binaries/tests/fixelconnectivity index 9a45f4d719..6ba11c37fc 100644 --- a/testing/binaries/tests/fixelconnectivity +++ b/testing/binaries/tests/fixelconnectivity @@ -1,3 +1,4 @@ -fixelconnectivity SIFT_phantom/fixels/ SIFT_phantom/tracks.tck tmp/ -force && testing_diff_image tmp/index.mif SIFT_phantom/matrix/index.mif && testing_diff_image tmp/fixels.mif SIFT_phantom/matrix/fixels.mif && testing_diff_image tmp/values.mif SIFT_phantom/matrix/values.mif -fixelconnectivity SIFT_phantom/fixels/ SIFT_phantom/tracks.tck tmp/ -mask SIFT_phantom/fixels/upper.mif -force && testing_diff_image tmp/index.mif fixelconnectivity/masked/index.mif && testing_diff_image tmp/fixels.mif fixelconnectivity/masked/fixels.mif && testing_diff_image tmp/values.mif fixelconnectivity/masked/values.mif +fixelconnectivity SIFT_phantom/fixels/ SIFT_phantom/tracks.tck tmp/ -count tmp-count.mif -extent tmp-extent.mif -force && testing_diff_image tmp/index.mif fixelconnectivity/default/index.mif && testing_diff_image tmp/fixels.mif fixelconnectivity/default/fixels.mif && testing_diff_image tmp/values.mif fixelconnectivity/default/values.mif && testing_diff_image tmp-count.mif fixelconnectivity/default_count.mif && testing_diff_image tmp-extent.mif fixelconnectivity/default_extent.mif +fixelconnectivity SIFT_phantom/fixels/ SIFT_phantom/tracks.tck tmp/ -count tmp-count.mif -extent tmp-extent.mif -mask SIFT_phantom/fixels/upper.mif -force && testing_diff_image tmp/index.mif fixelconnectivity/masked/index.mif && testing_diff_image tmp/fixels.mif fixelconnectivity/masked/fixels.mif && testing_diff_image tmp/values.mif fixelconnectivity/masked/values.mif && testing_diff_image tmp-count.mif fixelconnectivity/masked_count.mif && testing_diff_image tmp-extent.mif fixelconnectivity/masked_extent.mif +fixelconnectivity SIFT_phantom/fixels/ SIFT_phantom/tracks.tck tmp/ -count tmp-count.mif -extent tmp-extent.mif -tck_weights_in SIFT_phantom/weights.csv -force && testing_diff_image tmp/index.mif fixelconnectivity/weighted/index.mif && testing_diff_image tmp/fixels.mif fixelconnectivity/weighted/fixels.mif && testing_diff_image tmp/values.mif fixelconnectivity/weighted/values.mif && testing_diff_image tmp-count.mif fixelconnectivity/weighted_count.mif && testing_diff_image tmp-extent.mif fixelconnectivity/weighted_extent.mif From cdc1c3b84b55a7d48aa2205b965e287467f24745 Mon Sep 17 00:00:00 2001 From: Robert Smith Date: Wed, 19 Jul 2023 00:37:21 +1000 Subject: [PATCH 4/4] Fixel::Matrix: Neaten by using typedefs --- src/fixel/matrix.cpp | 4 ++-- src/fixel/matrix.h | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/fixel/matrix.cpp b/src/fixel/matrix.cpp index d6b04fe88e..d9dd2624d3 100644 --- a/src/fixel/matrix.cpp +++ b/src/fixel/matrix.cpp @@ -296,7 +296,7 @@ namespace MR if (App::overwrite_files) { File::remove (path); } else { - throw Exception ("Cannot create fixel-fixel connectivity matrix \"" + path + "\": Already exists as file"); + throw Exception ("Cannot create fixel-fixel connectivity matrix directory \"" + path + "\": Already exists as file"); } } } else { @@ -322,7 +322,7 @@ namespace MR index_type num_connections = 0; { - ProgressBar progress ("Computing number of fixel-fixel connections in matrix", matrix.size()); + ProgressBar progress ("Computing number of supra-threshold fixel-fixel connections", matrix.size()); for (size_t fixel_index = 0; fixel_index != matrix.size(); ++fixel_index) { const connectivity_value_type normalisation_factor = matrix[fixel_index].norm_factor(); diff --git a/src/fixel/matrix.h b/src/fixel/matrix.h index 65a234411b..3f8e1524fa 100644 --- a/src/fixel/matrix.h +++ b/src/fixel/matrix.h @@ -100,18 +100,18 @@ namespace MR using BaseType::operator<; using ValueType = connectivity_value_type; InitElementWeighted() : - sum_weights (connectivity_value_type(0)) { } + sum_weights (ValueType(0)) { } InitElementWeighted (const fixel_index_type fixel_index) = delete; InitElementWeighted (const fixel_index_type fixel_index, const MappedTrack& all_data) : BaseType (fixel_index), sum_weights (all_data.get_weight()) { } InitElementWeighted (const InitElementWeighted&) = default; FORCE_INLINE fixel_index_type index() const { return BaseType::index(); } - FORCE_INLINE InitElementWeighted& operator+= (const connectivity_value_type increment) { sum_weights += increment; return *this; } + FORCE_INLINE InitElementWeighted& operator+= (const ValueType increment) { sum_weights += increment; return *this; } FORCE_INLINE InitElementWeighted& operator= (const InitElementWeighted& that) { BaseType::operator= (that); sum_weights = that.sum_weights; return *this; } FORCE_INLINE ValueType value() const { return sum_weights; } private: - connectivity_value_type sum_weights; + ValueType sum_weights; }; @@ -192,7 +192,7 @@ namespace MR FORCE_INLINE void normalise (const ValueType norm_factor) { connectivity_value *= norm_factor; } private: fixel_index_type fixel_index; - connectivity_value_type connectivity_value; + ValueType connectivity_value; }; @@ -205,24 +205,25 @@ namespace MR public: using ElementType = NormElement; using BaseType = vector; + using ValueType = connectivity_value_type; NormFixel() : - norm_multiplier (connectivity_value_type (1)) { } + norm_multiplier (ValueType (1)) { } NormFixel (const BaseType& i) : BaseType (i), - norm_multiplier (connectivity_value_type (1)) { } + norm_multiplier (ValueType (1)) { } NormFixel (BaseType&& i) : BaseType (std::move (i)), - norm_multiplier (connectivity_value_type (1)) { } + norm_multiplier (ValueType (1)) { } void normalise() { - norm_multiplier = connectivity_value_type (0); + norm_multiplier = ValueType (0); for (const auto& c : *this) norm_multiplier += c.value(); - norm_multiplier = norm_multiplier ? (connectivity_value_type (1) / norm_multiplier) : connectivity_value_type(0); + norm_multiplier = norm_multiplier ? (ValueType (1) / norm_multiplier) : ValueType(0); } - void normalise (const connectivity_value_type sum) { - norm_multiplier = sum ? (connectivity_value_type(1) / sum) : connectivity_value_type(0); + void normalise (const ValueType sum) { + norm_multiplier = sum ? (ValueType(1) / sum) : ValueType(0); } - connectivity_value_type norm_multiplier; + ValueType norm_multiplier; };