Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Place components in namespaces #17

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Place components in namespaces #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkBitReader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
#include <Statistics.hpp>


using namespace rapidgzip;


template<bool MOST_SIGNIFICANT_BITS_FIRST,
typename BitBuffer>
[[nodiscard]] std::pair<double, uint64_t>
Expand Down
4 changes: 4 additions & 0 deletions src/benchmarks/benchmarkBitStringFinder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@

//#define BENCHMARK


using namespace rapidgzip;


namespace
{
constexpr uint64_t bitStringToFind = 0x314159265359; /* bcd(pi) */
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkCRC32.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
#endif


using namespace rapidgzip;


constexpr size_t REPEAT_COUNT{ 10 };


Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkCalculations.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
#include <Statistics.hpp>


using namespace rapidgzip;


[[nodiscard]] std::string
formatBandwidth( const std::vector<double>& times,
size_t byteCount )
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkGzip.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
#include <TestHelpers.hpp>


using namespace rapidgzip;


class GzipWrapper
{
public:
Expand Down
67 changes: 35 additions & 32 deletions src/benchmarks/benchmarkGzipBlockFinder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ GZIP file format specification version 4.3
#include <TestHelpers.hpp>


using namespace rapidgzip;


std::ostream&
operator<<( std::ostream& out, std::vector<size_t> vector )
{
Expand Down Expand Up @@ -431,7 +434,7 @@ findDeflateBlocksZlib( BufferedFileReader::AlignedBuffer buffer )
[[nodiscard]] std::vector<size_t>
findDeflateBlocksZlibOptimized( BufferedFileReader::AlignedBuffer buffer )
{
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( buffer ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( buffer ) );

/**
* Deflate Block:
Expand Down Expand Up @@ -572,7 +575,7 @@ findDeflateBlocksRapidgzip( BufferedFileReader::AlignedBuffer buffer )
using DeflateBlock = rapidgzip::deflate::Block<>;

const auto nBitsToTest = buffer.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( buffer ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( buffer ) ) );

std::vector<size_t> bitOffsets;

Expand Down Expand Up @@ -621,7 +624,7 @@ findDeflateBlocksRapidgzip( BufferedFileReader::AlignedBuffer buffer )
* decoded data if we do decide that it is a valid block. The number of checks during reading is also
* pretty few because there almost are no wasted / invalid symbols. */
bitOffsets.push_back( offset );
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
break;
}
}
Expand All @@ -634,7 +637,7 @@ template<uint8_t CACHED_BIT_COUNT>
countDeflateBlocksPreselection( BufferedFileReader::AlignedBuffer data )
{
const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

uint64_t candidateCount{ 0 };

Expand All @@ -661,7 +664,7 @@ countDeflateBlocksPreselection( BufferedFileReader::AlignedBuffer data )

++candidateCount;
++offset;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* This might happen when calling readDynamicHuffmanCoding quite some bytes before the end! */
break;
}
Expand All @@ -676,7 +679,7 @@ template<uint8_t CACHED_BIT_COUNT>
countDeflateBlocksPreselectionManualSlidingBuffer( BufferedFileReader::AlignedBuffer data )
{
const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

uint64_t candidateCount{ 0 };

Expand Down Expand Up @@ -706,7 +709,7 @@ countDeflateBlocksPreselectionManualSlidingBuffer( BufferedFileReader::AlignedBu
<< static_cast<uint8_t>( CACHED_BIT_COUNT - nextPosition );
offset += nextPosition;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* This might happen when calling readDynamicHuffmanCoding quite some bytes before the end! */
}

Expand Down Expand Up @@ -894,10 +897,10 @@ checkPrecode( const uint64_t next4Bits,
/* Without "forceinline", I observed a ~10% performance degradation! */
template<CheckPrecodeMethod CHECK_PRECODE_METHOD>
[[nodiscard]] forceinline rapidgzip::Error
checkDeflateBlock( const uint64_t bitBufferForLUT,
const uint64_t bitBufferPrecodeBits,
const size_t offset,
rapidgzip::BitReader& bitReader )
checkDeflateBlock( const uint64_t bitBufferForLUT,
const uint64_t bitBufferPrecodeBits,
const size_t offset,
gzip::BitReader& bitReader )
{
using namespace rapidgzip;
using namespace deflate;
Expand Down Expand Up @@ -1105,10 +1108,10 @@ checkAndGetValidHistogramID( const uint64_t precodeBits )

template<>
[[nodiscard]] forceinline rapidgzip::Error
checkDeflateBlock<CheckPrecodeMethod::SINGLE_COMPRESSED_LUT>( const uint64_t bitBufferForLUT,
const uint64_t bitBufferPrecodeBits,
const size_t offset,
rapidgzip::BitReader& bitReader )
checkDeflateBlock<CheckPrecodeMethod::SINGLE_COMPRESSED_LUT>( const uint64_t bitBufferForLUT,
const uint64_t bitBufferPrecodeBits,
const size_t offset,
gzip::BitReader& bitReader )
{
using namespace rapidgzip;
using namespace deflate;
Expand Down Expand Up @@ -1189,7 +1192,7 @@ template<uint8_t CACHED_BIT_COUNT,
findDeflateBlocksRapidgzipLUT( BufferedFileReader::AlignedBuffer data )
{
const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

std::vector<size_t> bitOffsets;

Expand All @@ -1212,7 +1215,7 @@ findDeflateBlocksRapidgzipLUT( BufferedFileReader::AlignedBuffer data )
constexpr auto ALL_PRECODE_BITS = PRECODE_COUNT_BITS + MAX_PRECODE_COUNT * PRECODE_BITS;
static_assert( ( ALL_PRECODE_BITS == 61 ) && ( ALL_PRECODE_BITS >= CACHED_BIT_COUNT )
&& ( ALL_PRECODE_BITS <= std::numeric_limits<uint64_t>::digits )
&& ( ALL_PRECODE_BITS <= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE ),
&& ( ALL_PRECODE_BITS <= gzip::BitReader::MAX_BIT_BUFFER_SIZE ),
"It must fit into 64-bit and it also must fit the largest possible jump in the LUT." );
auto bitBufferPrecodeBits = bitReader.read<ALL_PRECODE_BITS>();

Expand Down Expand Up @@ -1260,7 +1263,7 @@ findDeflateBlocksRapidgzipLUT( BufferedFileReader::AlignedBuffer data )

offset += nextPosition;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* This might happen when calling readDynamicHuffmanCoding quite some bytes before the end! */
}

Expand All @@ -1272,7 +1275,7 @@ findDeflateBlocksRapidgzipLUT( BufferedFileReader::AlignedBuffer data )
countFilterEfficiencies( BufferedFileReader::AlignedBuffer data )
{
const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

std::vector<size_t> bitOffsets;

Expand Down Expand Up @@ -1304,7 +1307,7 @@ countFilterEfficiencies( BufferedFileReader::AlignedBuffer data )
const auto next57Bits = bitReader.peek( rapidgzip::deflate::MAX_PRECODE_COUNT
* rapidgzip::deflate::PRECODE_BITS );
static_assert( rapidgzip::deflate::MAX_PRECODE_COUNT * rapidgzip::deflate::PRECODE_BITS
<= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE,
<= gzip::BitReader::MAX_BIT_BUFFER_SIZE,
"This optimization requires a larger BitBuffer inside BitReader!" );
using rapidgzip::PrecodeCheck::WalkTreeLUT::checkPrecode;
const auto precodeError = checkPrecode( next4Bits, next57Bits );
Expand All @@ -1331,7 +1334,7 @@ countFilterEfficiencies( BufferedFileReader::AlignedBuffer data )

bitOffsets.push_back( offset );
++offset;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* This might happen when calling readDynamicHuffmanCoding quite some bytes before the end! */
break;
}
Expand Down Expand Up @@ -1473,7 +1476,7 @@ findDeflateBlocksRapidgzipLUTTwoPass( BufferedFileReader::AlignedBuffer data )
"code length check, to avoid duplicate checks in the precode check!" );

const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

std::vector<size_t> bitOffsetCandidates;

Expand All @@ -1492,7 +1495,7 @@ findDeflateBlocksRapidgzipLUTTwoPass( BufferedFileReader::AlignedBuffer data )
offset += nextPosition;
bitReader.seekAfterPeek( nextPosition );
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
break;
}
}
Expand All @@ -1517,7 +1520,7 @@ findDeflateBlocksRapidgzipLUTTwoPass( BufferedFileReader::AlignedBuffer data )
const auto next57Bits = bitReader.peek( rapidgzip::deflate::MAX_PRECODE_COUNT
* rapidgzip::deflate::PRECODE_BITS );
static_assert( rapidgzip::deflate::MAX_PRECODE_COUNT * rapidgzip::deflate::PRECODE_BITS
<= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE,
<= gzip::BitReader::MAX_BIT_BUFFER_SIZE,
"This optimization requires a larger BitBuffer inside BitReader!" );

auto error = rapidgzip::Error::NONE;
Expand All @@ -1538,12 +1541,12 @@ findDeflateBlocksRapidgzipLUTTwoPass( BufferedFileReader::AlignedBuffer data )
if ( error != rapidgzip::Error::NONE ) {
return false;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {}
} catch ( const gzip::BitReader::EndOfFileReached& ) {}

try {
bitReader.seek( static_cast<long long int>( offset ) + 3 );
return block.readDynamicHuffmanCoding( bitReader ) == rapidgzip::Error::NONE;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {}
} catch ( const gzip::BitReader::EndOfFileReached& ) {}
return false;
};

Expand Down Expand Up @@ -1573,7 +1576,7 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff
"code length check, to avoid duplicate checks in the precode check!" );

const size_t nBitsToTest = data.size() * CHAR_BIT;
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( std::move( data ) ) );

std::vector<size_t> bitOffsetCandidates;

Expand All @@ -1594,7 +1597,7 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff
constexpr auto ALL_PRECODE_BITS = PRECODE_COUNT_BITS + MAX_PRECODE_COUNT * PRECODE_BITS;
static_assert( ( ALL_PRECODE_BITS == 61 ) && ( ALL_PRECODE_BITS >= CACHED_BIT_COUNT )
&& ( ALL_PRECODE_BITS <= std::numeric_limits<uint64_t>::digits )
&& ( ALL_PRECODE_BITS <= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE ),
&& ( ALL_PRECODE_BITS <= gzip::BitReader::MAX_BIT_BUFFER_SIZE ),
"It must fit into 64-bit and it also must fit the largest possible jump in the LUT." );
auto bitBufferPrecodeBits = bitReader.read<ALL_PRECODE_BITS>();

Expand Down Expand Up @@ -1650,7 +1653,7 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff

offset += nextPosition;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
/* Might happen when testing close to the end. */
}

Expand All @@ -1674,7 +1677,7 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff
const auto next57Bits = bitReader.peek( rapidgzip::deflate::MAX_PRECODE_COUNT
* rapidgzip::deflate::PRECODE_BITS );
static_assert( rapidgzip::deflate::MAX_PRECODE_COUNT * rapidgzip::deflate::PRECODE_BITS
<= rapidgzip::BitReader::MAX_BIT_BUFFER_SIZE,
<= gzip::BitReader::MAX_BIT_BUFFER_SIZE,
"This optimization requires a larger BitBuffer inside BitReader!" );

using rapidgzip::PrecodeCheck::WalkTreeLUT::checkPrecode;
Expand All @@ -1683,12 +1686,12 @@ findDeflateBlocksRapidgzipLUTTwoPassWithPrecode( BufferedFileReader::AlignedBuff
if ( error != rapidgzip::Error::NONE ) {
return false;
}
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {}
} catch ( const gzip::BitReader::EndOfFileReached& ) {}

try {
bitReader.seek( static_cast<long long int>( offset ) + 3 );
return block.readDynamicHuffmanCoding( bitReader ) == rapidgzip::Error::NONE;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {}
} catch ( const gzip::BitReader::EndOfFileReached& ) {}
return false;
};

Expand Down
4 changes: 2 additions & 2 deletions src/benchmarks/benchmarkHuffmanCoding.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ template<typename HuffmanCoding>
benchmarkHuffmanCoding( const std::vector<typename HuffmanCoding::BitCount>& codeLengths,
const BufferedFileReader::AlignedBuffer& encoded )
{
rapidgzip::BitReader bitReader( std::make_unique<BufferedFileReader>( encoded ) );
gzip::BitReader bitReader( std::make_unique<BufferedFileReader>( encoded ) );

Result result;
result.encodedSizeInBits = bitReader.size().value();
Expand All @@ -67,7 +67,7 @@ benchmarkHuffmanCoding( const std::vector<typename HuffmanCoding::BitCount>& cod
break;
}
++count;
} catch ( const rapidgzip::BitReader::EndOfFileReached& ) {
} catch ( const gzip::BitReader::EndOfFileReached& ) {
break;
}
}
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkIORead.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
#include <FileUtils.hpp>


using namespace rapidgzip;


/* Create a temporary file for benchmarking that is cleaned up with RAII. */
struct TemporaryFile
{
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkIOWrite.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
#include <ThreadPool.hpp>


using namespace rapidgzip;


/* Alignment to the filesystem block size is necessary for direct I/O. */
using DataBuffer = std::vector<char, AlignedAllocator<char, 4096> >;

Expand Down
5 changes: 4 additions & 1 deletion src/benchmarks/benchmarkIndexCompression.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
#include <ParallelGzipReader.hpp>


using namespace rapidgzip;


template<typename ResultContainer = std::vector<uint8_t> >
[[nodiscard]] ResultContainer
compress( const VectorView<uint8_t> toCompress )
Expand Down Expand Up @@ -64,7 +67,7 @@ main( int argc,

std::array<uint8_t, 64_Ki> windowPatches;

rapidgzip::BitReader bitReader( file->clone() );
gzip::BitReader bitReader( file->clone() );
WindowMap windows;
for ( const auto& checkpoint : index.checkpoints ) {
windowCount++;
Expand Down
5 changes: 4 additions & 1 deletion src/benchmarks/benchmarkInflate.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
#include <TestHelpers.hpp>


using namespace rapidgzip;


[[nodiscard]] size_t
decompressWithRapidgzip( UniqueFileReader fileReader )
{
Expand Down Expand Up @@ -125,7 +128,7 @@ compressWithZlib( const std::vector<std::byte>& toCompress,
output.resize( output.size() + CHUNK_SIZE );
stream.next_out = reinterpret_cast<Bytef*>( output.data() + output.size() - CHUNK_SIZE );
stream.avail_out = CHUNK_SIZE;
status = deflate( &stream, Z_FINISH );
status = ::deflate( &stream, Z_FINISH );
}

deflateEnd( &stream );
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkMarkerReplacement.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
#include <common.hpp>


using namespace rapidgzip;


void
replaceInPlace( std::vector<std::uint16_t>& buffer,
const std::vector<std::uint8_t>& window )
Expand Down
3 changes: 3 additions & 0 deletions src/benchmarks/benchmarkPigzBlockFinder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
#include <TestHelpers.hpp>


using namespace rapidgzip;


struct BenchmarkResults
{
double duration{ 0 };
Expand Down
Loading
Loading