Skip to content

Commit

Permalink
fix format
Browse files Browse the repository at this point in the history
  • Loading branch information
hubgeter committed Nov 22, 2023
1 parent 509a96f commit 6a0385f
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 36 deletions.
19 changes: 4 additions & 15 deletions be/src/exec/decompressor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -383,21 +383,10 @@ Status Lz4BlockDecompressor::decompress(uint8_t* input, size_t input_len, size_t

// Decompress
auto uncompressed_len = LZ4_decompress_safe(reinterpret_cast<const char*>(src),
reinterpret_cast<char*>(output), compressed_len,
remaining_output_size);

if (uncompressed_len < 0 || uncompressed_len != uncompressed_block_len ) {
static std::mutex mtx;
{
std::unique_lock<std::mutex> lck(mtx);
std::cout <<"Input len = " << input_len<<"\n";
std::cout <<"Output len = " << output_max_len<<"\n";
std::cout << "compressed_len = " << compressed_len <<"\n";
std::cout << "remaining_output_size = " << remaining_output_size <<"\n";
std::cout << "uncompressed_len = " << uncompressed_len <<"\n";
std::cout <<"is error = " << LZ4F_isError(uncompressed_len) <<"\n";
std::cout <<"error code = "<< LZ4F_getErrorName(uncompressed_len)<<"\n";
}
reinterpret_cast<char*>(output), compressed_len,
remaining_output_size);

if (uncompressed_len < 0 || uncompressed_len != uncompressed_block_len) {
return Status::InternalError(
"lz4 block decompress failed. uncompressed_len: {}, expected: {}",
uncompressed_len, uncompressed_block_len);
Expand Down
29 changes: 8 additions & 21 deletions be/src/util/block_compression.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@
#include <ostream>

#include "common/config.h"
#include "exec/decompressor.h"
#include "gutil/strings/substitute.h"
#include "util/bit_util.h"
#include "util/defer_op.h"
#include "util/faststring.h"
#include "util/bit_util.h"
#include "exec/decompressor.h"

namespace doris {

Expand Down Expand Up @@ -192,33 +192,20 @@ class HadoopLz4BlockCompression : public Lz4BlockCompression {
return &s_instance;
}
Status decompress(const Slice& input, Slice* output) override {
RETURN_IF_ERROR(Decompressor::create_decompressor(CompressType::LZ4BLOCK,&_decompressor));
RETURN_IF_ERROR(Decompressor::create_decompressor(CompressType::LZ4BLOCK, &_decompressor));
size_t input_bytes_read = 0;
size_t decompressed_len = 0;
size_t more_input_bytes = 0;
size_t more_output_bytes = 0;
bool stream_end = false;
auto st = _decompressor->decompress( (uint8_t*)input.data,input.size,&input_bytes_read,
(uint8_t*)output->data,output->size,
&decompressed_len,&stream_end,&more_input_bytes,&more_output_bytes);
static std::mutex mtx;
{
std::unique_lock<std::mutex> lck(mtx);
if (st != Status::OK()) {
std::cout << st.to_string() << "\n";
}

}

// if (st!=Status::OK()){
// auto st2 = Lz4BlockCompression::decompress(input,output);
// }

return Status::OK();
auto st = _decompressor->decompress((uint8_t*)input.data, input.size, &input_bytes_read,
(uint8_t*)output->data, output->size, &decompressed_len,
&stream_end, &more_input_bytes, &more_output_bytes);
return (st != Status::OK()) ? Lz4BlockCompression::decompress(input, output) : Status::OK();
}

private:
Decompressor* _decompressor;

};
// Used for LZ4 frame format, decompress speed is two times faster than LZ4.
class Lz4fBlockCompression : public BlockCompressionCodec {
Expand Down

0 comments on commit 6a0385f

Please sign in to comment.