diff --git a/README.md b/README.md index 3ba3e67..7a3e511 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ The compression level in QATzip could be mapped to standard zlib\* as below: * For 7z format, decompression only supports \*.7z archives compressed by qzip. * For 7z format, decompression only supports software. * For 7z format, the header compression is not supported. - +* Stream APIs do not support deflate_4B compression/decompression now. ## Installation Instructions diff --git a/src/qatzip.c b/src/qatzip.c index 8e74662..5c97415 100755 --- a/src/qatzip.c +++ b/src/qatzip.c @@ -1375,7 +1375,7 @@ static void *doCompressOut(void *in) QZ_DEBUG("\tlen = 0x%x\n", resl->produced); //Append footer outputFooterGen(qz_sess, resl, data_fmt); - qz_sess->next_dest += stdGzipFooterSz(); + qz_sess->next_dest += outputFooterSz(data_fmt); if (1 == g_process.qz_inst[i].stream[j].src_pinned) { g_process.qz_inst[i].src_buffers[j]->pBuffers->pData = g_process.qz_inst[i].stream[j].orig_src; @@ -1772,8 +1772,12 @@ static int checkHeader(QzSess_T *qz_sess, unsigned char *src, if ((unsigned char *)qzFooter == src_ptr + src_avail_len - stdGzipFooterSz()) { isEndWithFooter = 1; } - } else if (QZ_OK != qzGzipHeaderExt(src_ptr, hdr)) { - return QZ_FAIL; + } else if (QZ_DEFLATE_4B == data_fmt) { + hdr->extra.qz_e.dest_sz = *(int *)src_ptr; + hdr->extra.qz_e.src_sz = qz_sess->sess_params.hw_buff_sz; + } else if (QZ_DEFLATE_GZIP_EXT == data_fmt) { + if (QZ_OK != qzGzipHeaderExt(src_ptr, hdr)) + return QZ_FAIL; } src_send_sz = (long)(hdr->extra.qz_e.dest_sz); @@ -1981,13 +1985,14 @@ static void *doDecompressIn(void *in) } g_process.qz_inst[i].num_retries = 0; - src_avail_len -= (outputHeaderSz(data_fmt) + src_send_sz + stdGzipFooterSz()); + src_avail_len -= (outputHeaderSz(data_fmt) + src_send_sz + + outputFooterSz(data_fmt)); dest_avail_len -= dest_receive_sz; dest_ptr += dest_receive_sz; - src_ptr += (src_send_sz + stdGzipFooterSz()); - remaining -= (src_send_sz + stdGzipFooterSz()); + src_ptr += (src_send_sz + outputFooterSz(data_fmt)); + remaining -= (src_send_sz + outputFooterSz(data_fmt)); break; default: @@ -2121,29 +2126,32 @@ static void *__attribute__((cold)) doDecompressOut(void *in) g_process.qz_inst[i].stream[j].src_pinned = 0; } - if (unlikely(resl->checksum != - g_process.qz_inst[i].stream[j].gzip_footer_checksum || - resl->produced != g_process.qz_inst[i].stream[j].gzip_footer_orgdatalen)) { - QZ_ERROR("Error in check footer, inst %ld, stream %ld\n", i, j); - QZ_DEBUG("resp checksum: %x data checksum %x\n", - resl->checksum, - g_process.qz_inst[i].stream[j].gzip_footer_checksum); - QZ_DEBUG("resp produced :%d data produced: %d\n", - resl->produced, - g_process.qz_inst[i].stream[j].gzip_footer_orgdatalen); - - swapDataBuffer(i, j); - sess->thd_sess_stat = QZ_DATA_ERROR; - qz_sess->processed++; - qz_sess->stop_submitting = 1; - g_process.qz_inst[i].stream[j].sink2++; - continue; + + if (data_fmt != QZ_DEFLATE_4B) { + if (unlikely(resl->checksum != + g_process.qz_inst[i].stream[j].gzip_footer_checksum || + resl->produced != g_process.qz_inst[i].stream[j].gzip_footer_orgdatalen)) { + QZ_ERROR("Error in check footer, inst %ld, stream %ld\n", i, j); + QZ_DEBUG("resp checksum: %x data checksum %x\n", + resl->checksum, + g_process.qz_inst[i].stream[j].gzip_footer_checksum); + QZ_DEBUG("resp produced :%d data produced: %d\n", + resl->produced, + g_process.qz_inst[i].stream[j].gzip_footer_orgdatalen); + + swapDataBuffer(i, j); + sess->thd_sess_stat = QZ_DATA_ERROR; + qz_sess->processed++; + qz_sess->stop_submitting = 1; + g_process.qz_inst[i].stream[j].sink2++; + continue; + } } src_send_sz = g_process.qz_inst[i].src_buffers[j]->pBuffers->dataLenInBytes; qz_sess->next_dest += resl->produced; qz_sess->qz_in_len += (outputHeaderSz(data_fmt) + src_send_sz + - stdGzipFooterSz()); + outputFooterSz(data_fmt)); qz_sess->qz_out_len += resl->produced; QZ_DEBUG("qz_sess->next_dest = %p\n", qz_sess->next_dest); @@ -2268,6 +2276,7 @@ int qzDecompress(QzSession_T *sess, const unsigned char *src, QzDataFormat_T data_fmt = qz_sess->sess_params.data_fmt; if (unlikely(data_fmt != QZ_DEFLATE_RAW && + data_fmt != QZ_DEFLATE_4B && data_fmt != QZ_DEFLATE_GZIP && data_fmt != QZ_DEFLATE_GZIP_EXT)) { QZ_ERROR("Unknown data formt: %d\n", data_fmt); @@ -2277,12 +2286,13 @@ int qzDecompress(QzSession_T *sess, const unsigned char *src, } QZ_DEBUG("qzDecompress data_fmt: %d\n", data_fmt); - if (hdr->extra.qz_e.src_sz < qz_sess->sess_params.input_sz_thrshold || + if (data_fmt == QZ_DEFLATE_RAW || + (data_fmt == QZ_DEFLATE_GZIP_EXT && + hdr->extra.qz_e.src_sz < qz_sess->sess_params.input_sz_thrshold) || g_process.qz_init_status == QZ_NO_HW || sess->hw_session_stat == QZ_NO_HW || !(isQATProcessable(src, src_len, qz_sess)) || - qz_sess->inflate_stat == InflateOK || - QZ_DEFLATE_RAW == data_fmt) { + qz_sess->inflate_stat == InflateOK) { QZ_DEBUG("decompression src_len=%u, hdr->extra.qz_e.src_sz = %u, " "g_process.qz_init_status = %d, sess->hw_session_stat = %d, " "isQATProcessable = %d, switch to software.\n", diff --git a/src/qatzip_gzip.c b/src/qatzip_gzip.c index 7d7b5c1..cdc6b70 100755 --- a/src/qatzip_gzip.c +++ b/src/qatzip_gzip.c @@ -58,6 +58,11 @@ inline unsigned long stdGzipHeaderSz(void) return sizeof(StdGzH_T); } +inline unsigned long qz4BHeaderSz(void) +{ + return sizeof(Qz4BH_T); +} + inline unsigned long stdGzipFooterSz(void) { return sizeof(StdGzF_T); @@ -67,6 +72,8 @@ inline unsigned long outputFooterSz(QzDataFormat_T data_fmt) { unsigned long size = 0; switch (data_fmt) { + case QZ_DEFLATE_4B: + /* fall through */ case QZ_DEFLATE_RAW: size = 0; break; @@ -84,6 +91,9 @@ unsigned long outputHeaderSz(QzDataFormat_T data_fmt) unsigned long size = 0; switch (data_fmt) { + case QZ_DEFLATE_4B: + size = qz4BHeaderSz(); + break; case QZ_DEFLATE_RAW: break; case QZ_DEFLATE_GZIP: @@ -150,6 +160,13 @@ void stdGzipHeaderGen(unsigned char *ptr, CpaDcRqResults *res) hdr->os = 255; } +void qz4BHeaderGen(unsigned char *ptr, CpaDcRqResults *res) +{ + Qz4BH_T *hdr; + hdr = (Qz4BH_T *)ptr; + hdr->blk_size = res->produced; +} + void outputHeaderGen(unsigned char *ptr, CpaDcRqResults *res, QzDataFormat_T data_fmt) @@ -157,6 +174,9 @@ void outputHeaderGen(unsigned char *ptr, QZ_DEBUG("Generate header\n"); switch (data_fmt) { + case QZ_DEFLATE_4B: + qz4BHeaderGen(ptr, res); + break; case QZ_DEFLATE_RAW: break; case QZ_DEFLATE_GZIP: @@ -174,10 +194,19 @@ int isQATProcessable(const unsigned char *ptr, QzSess_T *const qz_sess) { QzGzH_T *h = (QzGzH_T *)ptr; + Qz4BH_T *h_4B; StdGzF_T *qzFooter = NULL; long buff_sz = (DEST_SZ(qz_sess->sess_params.hw_buff_sz) < *src_len ? DEST_SZ( qz_sess->sess_params.hw_buff_sz) : *src_len); + if (qz_sess->sess_params.data_fmt == QZ_DEFLATE_4B) { + h_4B = (Qz4BH_T *)ptr; + if (h_4B->blk_size > DEST_SZ(qz_sess->sess_params.hw_buff_sz)) { + return 0; + } + return 1; + } + /*check if HW can process*/ if (h->std_hdr.id1 == 0x1f && \ h->std_hdr.id2 == 0x8b && \ diff --git a/src/qatzip_internal.h b/src/qatzip_internal.h index 3935f07..621826b 100755 --- a/src/qatzip_internal.h +++ b/src/qatzip_internal.h @@ -233,6 +233,10 @@ typedef struct QzGzH_S { QzExtraField_T extra; } QzGzH_T; +typedef struct Qz4BH_S { + uint32_t blk_size; +} Qz4BH_T; + typedef struct StdGzF_S { uint32_t crc32; uint32_t i_size; @@ -248,11 +252,13 @@ typedef struct QzMem_S { void dumpAllCounters(void); int qzSetupHW(QzSession_T *sess, int i); unsigned long qzGzipHeaderSz(void); +unsigned long qz4BHeaderSz(void); unsigned long stdGzipHeaderSz(void); unsigned long stdGzipFooterSz(void); unsigned long outputHeaderSz(QzDataFormat_T data_fmt); unsigned long outputFooterSz(QzDataFormat_T data_fmt); void qzGzipHeaderGen(unsigned char *ptr, CpaDcRqResults *res); +void qz4BHeaderGen(unsigned char *ptr, CpaDcRqResults *res); void stdGzipHeaderGen(unsigned char *ptr, CpaDcRqResults *res); int qzGzipHeaderExt(const unsigned char *const ptr, QzGzH_T *hdr); void outputHeaderGen(unsigned char *ptr, diff --git a/src/qatzip_sw.c b/src/qatzip_sw.c index e8aee41..c868904 100755 --- a/src/qatzip_sw.c +++ b/src/qatzip_sw.c @@ -90,6 +90,7 @@ int qzSWCompress(QzSession_T *sess, const unsigned char *src, QzDataFormat_T data_fmt = QZ_DATA_FORMAT_DEFAULT; unsigned int chunk_sz = QZ_HW_BUFF_SZ; QzGzH_T *qz_hdr = NULL; + Qz4BH_T *qz4B_header = NULL; *src_len = 0; *dest_len = 0; @@ -127,6 +128,7 @@ int qzSWCompress(QzSession_T *sess, const unsigned char *src, stream->total_out = 0; switch (data_fmt) { + case QZ_DEFLATE_4B: case QZ_DEFLATE_RAW: windows_bits = -MAX_WBITS; break; @@ -158,6 +160,10 @@ int qzSWCompress(QzSession_T *sess, const unsigned char *src, return QZ_FAIL; } qz_hdr = (QzGzH_T *)dest; + } else if (QZ_DEFLATE_4B == data_fmt) { + /* Need to reserve 4 bytes to fill the compressed length. */ + qz4B_header = (Qz4BH_T *)dest; + dest = dest + sizeof(Qz4BH_T); } } @@ -223,10 +229,13 @@ int qzSWCompress(QzSession_T *sess, const unsigned char *src, * When data_fmt is QZ_DEFLATE_GZIP_EXT, * we should fill src_sz & dest_sz in gzipext header field. */ - if (QZ_DEFLATE_GZIP_EXT == data_fmt && qz_hdr) { + if (QZ_DEFLATE_GZIP_EXT == data_fmt && qz4B_header) { qz_hdr->extra.qz_e.src_sz = stream->total_in; qz_hdr->extra.qz_e.dest_sz = stream->total_out - outputHeaderSz(data_fmt) - outputFooterSz(data_fmt); + } else if (QZ_DEFLATE_4B == data_fmt && qz4B_header) { + qz4B_header->blk_size = stream->total_out; + *dest_len = *dest_len + sizeof(Qz4BH_T); } ret = deflateEnd(stream); stream->total_in = 0; @@ -252,6 +261,7 @@ int qzSWDecompress(QzSession_T *sess, const unsigned char *src, int windows_bits = 0; unsigned int total_in; unsigned int total_out; + unsigned int qz4B_header_len = 0; QzSess_T *qz_sess = (QzSess_T *) sess->internal; qz_sess->force_sw = 1; @@ -283,6 +293,7 @@ int qzSWDecompress(QzSession_T *sess, const unsigned char *src, QZ_DEBUG("decomp_sw data_fmt: %d\n", data_fmt); switch (data_fmt) { + case QZ_DEFLATE_4B: case QZ_DEFLATE_RAW: windows_bits = -MAX_WBITS; break; @@ -294,6 +305,12 @@ int qzSWDecompress(QzSession_T *sess, const unsigned char *src, } if (InflateNull == qz_sess->inflate_stat) { + if (QZ_DEFLATE_4B == data_fmt) { + /* For QZ_DEFLATE_4B, we need to skip the header. */ + stream->next_in = (z_const Bytef *)stream->next_in + sizeof(Qz4BH_T); + stream->avail_in = stream->avail_in - sizeof(Qz4BH_T); + qz4B_header_len = sizeof(Qz4BH_T); + } ret = inflateInit2(stream, windows_bits); if (Z_OK != ret) { ret = QZ_FAIL; @@ -339,7 +356,8 @@ int qzSWDecompress(QzSession_T *sess, const unsigned char *src, } *dest_len = GET_LOWER_32BITS(stream->total_out - total_out); - *src_len = GET_LOWER_32BITS(stream->total_in - total_in); + /* for Deflate_4B, we need to add the length of Deflate 4B header. */ + *src_len = GET_LOWER_32BITS(stream->total_in - total_in + qz4B_header_len); done: QZ_DEBUG("Exit qzSWDecompress total_in: %u total_out: %u " diff --git a/test/main.c b/test/main.c index a514ffc..6356220 100755 --- a/test/main.c +++ b/test/main.c @@ -3712,7 +3712,7 @@ void *qzDecompressStreamWithBufferError(void *thd_arg) " -D direction comp | decomp | both\n" \ " -F format [comp format]:[orig data size]/...\n" \ " -L comp_lvl 1 - " STR(MAX_LVL) "\n" \ - " -O data_fmt deflate | gzip | gzipext\n" \ + " -O data_fmt deflate | gzip | gzipext | deflate_4B\n" \ " -T huffmanType static | dynamic\n" \ " -r req_cnt_thrshold max inflight request num, default is 16\n" \ " -S thread_sleep the unit is milliseconds, default is a random time\n" \ @@ -3805,6 +3805,8 @@ int main(int argc, char *argv[]) g_params_th.data_fmt = QZ_DEFLATE_GZIP; } else if (strcmp(optarg, "gzipext") == 0) { g_params_th.data_fmt = QZ_DEFLATE_GZIP_EXT; + } else if (strcmp(optarg, "deflate_4B") == 0) { + g_params_th.data_fmt = QZ_DEFLATE_4B; } else { QZ_ERROR("Error service arg: %s\n", optarg); return -1; diff --git a/utils/qzip.c b/utils/qzip.c index efd912d..cc97e4a 100644 --- a/utils/qzip.c +++ b/utils/qzip.c @@ -62,7 +62,8 @@ const struct option g_long_opts[] = { {"huffmanhdr", 1, 0, 'H'}, /* set huffman header type */ {"level", 1, 0, 'L'}, /* set compression level */ {"chunksz", 1, 0, 'C'}, /* set chunk size */ - {"output", 1, 0, 'O'}, /* set output header format(gzip, gzipext, 7z)*/ + {"output", 1, 0, 'O'}, /* set output header format(gzip, gzipext, 7z, + deflate_4B) */ {"recursive", 0, 0, 'R'}, /* set recursive mode when compressing a directory */ {"polling", 1, 0, 'P'}, /* set polling mode when compressing and @@ -96,7 +97,7 @@ void help(void) " -V, --version display version number", " -L, --level set compression level", " -C, --chunksz set chunk size", - " -O, --output set output header format(gzip|gzipext|7z)", + " -O, --output set output header format(gzip|gzipext|7z|deflate_4B)", " -r, set max inflight request number", " -R, set Recursive mode for a directory", " -o, set output file name", @@ -450,7 +451,7 @@ int makeOutName(const char *in_name, const char *out_name, SUFFIX_GZ); } else { if (!hasSuffix(in_name)) { - QZ_ERROR("Error: %s: Wrong suffix. Supported suffix: 7z/gz.\n", + QZ_ERROR("%s: Wrong suffix. Supported suffix: 7z/gz.\n", in_name); return -1; } diff --git a/utils/qzip_main.c b/utils/qzip_main.c index 52e3888..059c8ad 100644 --- a/utils/qzip_main.c +++ b/utils/qzip_main.c @@ -104,6 +104,8 @@ int main(int argc, char **argv) g_params_th.data_fmt = QZ_DEFLATE_GZIP_EXT; } else if (strcmp(optarg, "7z") == 0) { g_params_th.data_fmt = QZ_DEFLATE_RAW; + } else if (strcmp(optarg, "deflate_4B") == 0) { + g_params_th.data_fmt = QZ_DEFLATE_4B; } else { QZ_ERROR("Error gzip header format arg: %s\n", optarg); return -1;