Merge pull request #217 from eseiler/misc/split

[MISC] Split hpp/cpp

.github/workflows/ci_coverage.yml

@@ -51,7 +51,7 @@ jobs:
         uses: seqan/actions/setup-toolchain@main
         with:
           compiler: ${{ matrix.compiler }}
-          ccache_size: 125M
+          ccache_size: 200M
 
       - name: Install CMake
         uses: seqan/actions/setup-cmake@main
@@ -68,7 +68,7 @@ jobs:
           mkdir build
           cd build
           cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
-                   -DCMAKE_CXX_FLAGS="-Wno-interference-size" \
+                   -DCMAKE_CXX_FLAGS="-Werror -Wno-interference-size" \
                    -DCHOPPER_NATIVE_BUILD=OFF
           make -j2 gtest_build
 
@@ -93,8 +93,7 @@ jobs:
                 ${{ github.workspace }}/build \
                 --filter ${{ github.workspace }}/include \
                 --filter ${{ github.workspace }}/src \
-                --exclude ${{ github.workspace }}/src/measure_hyperloglog.cpp \
-                --exclude ${{ github.workspace }}/src/display_layout \
+                --exclude ${{ github.workspace }}/src/util \
                 --exclude-lines-by-pattern '^\s*$' \
                 --exclude-lines-by-pattern '^\s*};$' \
                 --exclude-lines-by-pattern '^.*GCOVR_EXCL_LINE.*$' \

.github/workflows/ci_linux.yml

@@ -31,17 +31,25 @@ jobs:
       fail-fast: true
       matrix:
         include:
+          - name: "clang17"
+            compiler: "clang-17"
+            build: unit
+            build_type: Release
+
           - name: "gcc13"
             compiler: "gcc-13"
             build_type: Release
+            cxx_flags: -Wno-interference-size
 
           - name: "gcc12"
             compiler: "gcc-12"
             build_type: Release
+            cxx_flags: -Wno-interference-size
 
           - name: "gcc11"
             compiler: "gcc-11"
             build_type: Release
+            cxx_flags: -Wno-interference-size
 
     steps:
       - name: Checkout
@@ -54,7 +62,11 @@ jobs:
         uses: seqan/actions/setup-toolchain@main
         with:
           compiler: ${{ matrix.compiler }}
-          ccache_size: 75M
+          ccache_size: 125M
+
+      - name: Install OpenMP
+        if: contains(matrix.name, 'clang')
+        run: install libomp-17-dev
 
       - name: Install CMake
         uses: seqan/actions/setup-cmake@main
@@ -66,7 +78,7 @@ jobs:
           mkdir build
           cd build
           cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
-                   -DCMAKE_CXX_FLAGS="-Wno-interference-size" \
+                   -DCMAKE_CXX_FLAGS="-Werror ${{ matrix.cxx_flags }}" \
                    -DCHOPPER_NATIVE_BUILD=OFF
           make -j2 gtest_build
 

.github/workflows/ci_macos.yml

@@ -54,7 +54,7 @@ jobs:
         uses: seqan/actions/setup-toolchain@main
         with:
           compiler: ${{ matrix.compiler }}
-          ccache_size: 75M
+          ccache_size: 125M
 
       - name: Install CMake
         uses: seqan/actions/setup-cmake@main
@@ -66,7 +66,7 @@ jobs:
           mkdir build
           cd build
           cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
-                   -DCMAKE_CXX_FLAGS="-Wno-interference-size" \
+                   -DCMAKE_CXX_FLAGS="-Werror -Wno-interference-size" \
                    -DCHOPPER_NATIVE_BUILD=OFF
           make -j3 gtest_build
 

.github/workflows/ci_misc.yml

@@ -64,7 +64,7 @@ jobs:
           mkdir build
           cd build
           cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
-                   -DCMAKE_CXX_FLAGS="-Wno-interference-size" \
+                   -DCMAKE_CXX_FLAGS="-Werror -Wno-interference-size" \
                    -DCHOPPER_HEADER_TEST_ONLY=ON \
                    -DCHOPPER_NATIVE_BUILD=OFF
 

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "lib/seqan3"]
 	path = lib/seqan3
 	url = https://github.com/seqan/seqan3.git
-[submodule "lib/robin-hood-hashing"]
-	path = lib/robin-hood-hashing
-	url = https://github.com/martinus/robin-hood-hashing
 [submodule "lib/simde"]
 	path = lib/simde
 	url = https://github.com/simd-everywhere/simde

include/chopper/configuration.hpp

@@ -7,15 +7,11 @@
 
 #pragma once
 
-#include <cassert>
+#include <cinttypes>
 #include <filesystem>
+#include <iosfwd>
 
-#include <cereal/archives/json.hpp>
 #include <cereal/cereal.hpp>
-#include <cereal/types/string.hpp>
-#include <cereal/types/vector.hpp>
-
-#include <chopper/prefixes.hpp>
 
 #include <hibf/cereal/path.hpp> // IWYU pragma: keep
 #include <hibf/config.hpp>
@@ -70,48 +66,9 @@ struct configuration
     //!\brief The HIBF config which will be used to compute the layout within the HIBF lib.
     seqan::hibf::config hibf_config;
 
-    void read_from(std::istream & stream)
-    {
-        std::string line;
-        std::stringstream config_str;
-
-        while (std::getline(stream, line) && line != chopper::prefix::meta_chopper_config_start)
-            ;
-
-        assert(line == chopper::prefix::meta_chopper_config_start);
-
-        while (std::getline(stream, line) && line != chopper::prefix::meta_chopper_config_end)
-        {
-            assert(line.size() >= 2);
-            assert(std::string_view{line}.substr(0, 1) == seqan::hibf::prefix::meta_header);
-            config_str << line.substr(1); // remove seqan::hibf::prefix::meta_header
-        }
-
-        assert(line == chopper::prefix::meta_chopper_config_end);
+    void read_from(std::istream & stream);
 
-        cereal::JSONInputArchive iarchive(config_str);
-        iarchive(*this);
-
-        hibf_config.read_from(stream);
-    }
-
-    void write_to(std::ostream & stream) const
-    {
-        // write json file to temprorary string stream with cereal
-        std::stringstream config_stream{};
-        cereal::JSONOutputArchive output(config_stream); // stream to cout
-        output(cereal::make_nvp("chopper_config", *this));
-
-        // write config
-        stream << chopper::prefix::meta_chopper_config_start << '\n';
-        std::string line;
-        while (std::getline(config_stream, line, '\n'))
-            stream << seqan::hibf::prefix::meta_header << line << '\n';
-        stream << seqan::hibf::prefix::meta_header << "}\n" // last closing bracket isn't written by loop above
-               << chopper::prefix::meta_chopper_config_end << '\n';
-
-        hibf_config.write_to(stream);
-    }
+    void write_to(std::ostream & stream) const;
 
 private:
     friend class cereal::access;

include/chopper/input_functor.hpp

@@ -7,15 +7,14 @@
 
 #pragma once
 
+#include <cinttypes>
+#include <cstddef>
 #include <string>
 #include <vector>
 
 #include <seqan3/io/sequence_file/all.hpp>
-#include <seqan3/search/views/minimiser_hash.hpp>
 
-#include <chopper/adjust_seed.hpp>
-
-#include <hibf/hierarchical_interleaved_bloom_filter.hpp>
+#include <hibf/config.hpp>
 
 namespace chopper
 {
@@ -38,36 +37,7 @@ struct input_functor
 
     uint8_t kmer_size{21};
 
-    void operator()(size_t const num, seqan::hibf::insert_iterator it)
-    {
-        assert(filenames.size() > num);
-        if (input_are_precomputed_files)
-        {
-            uint64_t hash{};
-            char * const hash_data{reinterpret_cast<char *>(&hash)};
-            std::streamsize const hash_bytes{sizeof(hash)};
-
-            std::ifstream infile{filenames[num], std::ios::binary};
-
-            while (infile.read(hash_data, hash_bytes))
-                it = hash;
-        }
-        else
-        {
-            sequence_file_type fin{filenames[num]};
-
-            seqan3::shape shape = seqan3::ungapped{kmer_size};
-            auto minimizer_view = seqan3::views::minimiser_hash(shape,
-                                                                seqan3::window_size{kmer_size},
-                                                                seqan3::seed{adjust_seed(shape.count())});
-
-            for (auto && [seq] : fin)
-            {
-                for (auto hash_value : seq | minimizer_view)
-                    it = hash_value;
-            }
-        }
-    }
+    void operator()(size_t const num, seqan::hibf::insert_iterator it);
 };
 
 } // namespace chopper

include/chopper/layout/determine_best_number_of_technical_bins.hpp

@@ -0,0 +1,22 @@
+// ---------------------------------------------------------------------------------------------------
+// Copyright (c) 2006-2023, Knut Reinert & Freie Universität Berlin
+// Copyright (c) 2016-2023, Knut Reinert & MPI für molekulare Genetik
+// This file may be used, modified and/or redistributed under the terms of the 3-clause BSD-License
+// shipped with this file and also available at: https://github.com/seqan/chopper/blob/main/LICENSE.md
+// ---------------------------------------------------------------------------------------------------
+
+#include <utility>
+#include <vector>
+
+#include <chopper/configuration.hpp>
+
+#include <hibf/layout/layout.hpp>
+#include <hibf/sketch/hyperloglog.hpp>
+
+namespace chopper::layout
+{
+
+std::pair<seqan::hibf::layout::layout, std::vector<seqan::hibf::sketch::hyperloglog>>
+determine_best_number_of_technical_bins(chopper::configuration & config);
+
+}

include/chopper/layout/execute.hpp

@@ -7,6 +7,9 @@
 
 #pragma once
 
+#include <string>
+#include <vector>
+
 #include <chopper/configuration.hpp>
 
 namespace chopper::layout