Skip to content

Commit

Permalink
QATAPP-24793: Add QZ_DEFLATE_4B header format support
Browse files Browse the repository at this point in the history 
Change-Id: I5ef0635d593091a29fdac6ee8946126d04f9f532
Signed-off-by: Chengfei Zhu <[email protected]>
  • Loading branch information
@cfzhu
Xue Fei authored and cfzhu committed Dec 14, 2021
1 parent 5d1c4ff commit ecb32e7
Show file tree
Hide file tree
Showing 8 changed files with 102 additions and 34 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ The compression level in QATzip could be mapped to standard zlib\* as below:
* For 7z format, decompression only supports \*.7z archives compressed by qzip.
* For 7z format, decompression only supports software.
* For 7z format, the header compression is not supported.

* Stream APIs do not support deflate_4B compression/decompression now.


## Installation Instructions
Expand Down
64 changes: 37 additions & 27 deletions src/qatzip.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1375,7 +1375,7 @@ static void *doCompressOut(void *in)
QZ_DEBUG("\tlen = 0x%x\n", resl->produced);
//Append footer
outputFooterGen(qz_sess, resl, data_fmt);
qz_sess->next_dest += stdGzipFooterSz();
qz_sess->next_dest += outputFooterSz(data_fmt);
if (1 == g_process.qz_inst[i].stream[j].src_pinned) {
g_process.qz_inst[i].src_buffers[j]->pBuffers->pData =
g_process.qz_inst[i].stream[j].orig_src;
Expand Down Expand Up @@ -1772,8 +1772,12 @@ static int checkHeader(QzSess_T *qz_sess, unsigned char *src,
if ((unsigned char *)qzFooter == src_ptr + src_avail_len - stdGzipFooterSz()) {
isEndWithFooter = 1;
}
} else if (QZ_OK != qzGzipHeaderExt(src_ptr, hdr)) {
return QZ_FAIL;
} else if (QZ_DEFLATE_4B == data_fmt) {
hdr->extra.qz_e.dest_sz = *(int *)src_ptr;
hdr->extra.qz_e.src_sz = qz_sess->sess_params.hw_buff_sz;
} else if (QZ_DEFLATE_GZIP_EXT == data_fmt) {
if (QZ_OK != qzGzipHeaderExt(src_ptr, hdr))
return QZ_FAIL;
}

src_send_sz = (long)(hdr->extra.qz_e.dest_sz);
Expand Down Expand Up @@ -1981,13 +1985,14 @@ static void *doDecompressIn(void *in)
}

g_process.qz_inst[i].num_retries = 0;
src_avail_len -= (outputHeaderSz(data_fmt) + src_send_sz + stdGzipFooterSz());
src_avail_len -= (outputHeaderSz(data_fmt) + src_send_sz +
outputFooterSz(data_fmt));
dest_avail_len -= dest_receive_sz;

dest_ptr += dest_receive_sz;

src_ptr += (src_send_sz + stdGzipFooterSz());
remaining -= (src_send_sz + stdGzipFooterSz());
src_ptr += (src_send_sz + outputFooterSz(data_fmt));
remaining -= (src_send_sz + outputFooterSz(data_fmt));
break;

default:
Expand Down Expand Up @@ -2121,29 +2126,32 @@ static void *__attribute__((cold)) doDecompressOut(void *in)
g_process.qz_inst[i].stream[j].src_pinned = 0;
}

if (unlikely(resl->checksum !=
g_process.qz_inst[i].stream[j].gzip_footer_checksum ||
resl->produced != g_process.qz_inst[i].stream[j].gzip_footer_orgdatalen)) {
QZ_ERROR("Error in check footer, inst %ld, stream %ld\n", i, j);
QZ_DEBUG("resp checksum: %x data checksum %x\n",
resl->checksum,
g_process.qz_inst[i].stream[j].gzip_footer_checksum);
QZ_DEBUG("resp produced :%d data produced: %d\n",
resl->produced,
g_process.qz_inst[i].stream[j].gzip_footer_orgdatalen);

swapDataBuffer(i, j);
sess->thd_sess_stat = QZ_DATA_ERROR;
qz_sess->processed++;
qz_sess->stop_submitting = 1;
g_process.qz_inst[i].stream[j].sink2++;
continue;

if (data_fmt != QZ_DEFLATE_4B) {
if (unlikely(resl->checksum !=
g_process.qz_inst[i].stream[j].gzip_footer_checksum ||
resl->produced != g_process.qz_inst[i].stream[j].gzip_footer_orgdatalen)) {
QZ_ERROR("Error in check footer, inst %ld, stream %ld\n", i, j);
QZ_DEBUG("resp checksum: %x data checksum %x\n",
resl->checksum,
g_process.qz_inst[i].stream[j].gzip_footer_checksum);
QZ_DEBUG("resp produced :%d data produced: %d\n",
resl->produced,
g_process.qz_inst[i].stream[j].gzip_footer_orgdatalen);

swapDataBuffer(i, j);
sess->thd_sess_stat = QZ_DATA_ERROR;
qz_sess->processed++;
qz_sess->stop_submitting = 1;
g_process.qz_inst[i].stream[j].sink2++;
continue;
}
}

src_send_sz = g_process.qz_inst[i].src_buffers[j]->pBuffers->dataLenInBytes;
qz_sess->next_dest += resl->produced;
qz_sess->qz_in_len += (outputHeaderSz(data_fmt) + src_send_sz +
stdGzipFooterSz());
outputFooterSz(data_fmt));
qz_sess->qz_out_len += resl->produced;

QZ_DEBUG("qz_sess->next_dest = %p\n", qz_sess->next_dest);
Expand Down Expand Up @@ -2268,6 +2276,7 @@ int qzDecompress(QzSession_T *sess, const unsigned char *src,

QzDataFormat_T data_fmt = qz_sess->sess_params.data_fmt;
if (unlikely(data_fmt != QZ_DEFLATE_RAW &&
data_fmt != QZ_DEFLATE_4B &&
data_fmt != QZ_DEFLATE_GZIP &&
data_fmt != QZ_DEFLATE_GZIP_EXT)) {
QZ_ERROR("Unknown data formt: %d\n", data_fmt);
Expand All @@ -2277,12 +2286,13 @@ int qzDecompress(QzSession_T *sess, const unsigned char *src,
}

QZ_DEBUG("qzDecompress data_fmt: %d\n", data_fmt);
if (hdr->extra.qz_e.src_sz < qz_sess->sess_params.input_sz_thrshold ||
if (data_fmt == QZ_DEFLATE_RAW ||
(data_fmt == QZ_DEFLATE_GZIP_EXT &&
hdr->extra.qz_e.src_sz < qz_sess->sess_params.input_sz_thrshold) ||
g_process.qz_init_status == QZ_NO_HW ||
sess->hw_session_stat == QZ_NO_HW ||
!(isQATProcessable(src, src_len, qz_sess)) ||
qz_sess->inflate_stat == InflateOK ||
QZ_DEFLATE_RAW == data_fmt) {
qz_sess->inflate_stat == InflateOK) {
QZ_DEBUG("decompression src_len=%u, hdr->extra.qz_e.src_sz = %u, "
"g_process.qz_init_status = %d, sess->hw_session_stat = %d, "
"isQATProcessable = %d, switch to software.\n",
Expand Down
29 changes: 29 additions & 0 deletions src/qatzip_gzip.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ inline unsigned long stdGzipHeaderSz(void)
return sizeof(StdGzH_T);
}

inline unsigned long qz4BHeaderSz(void)
{
return sizeof(Qz4BH_T);
}

inline unsigned long stdGzipFooterSz(void)
{
return sizeof(StdGzF_T);
Expand All @@ -67,6 +72,8 @@ inline unsigned long outputFooterSz(QzDataFormat_T data_fmt)
{
unsigned long size = 0;
switch (data_fmt) {
case QZ_DEFLATE_4B:
/* fall through */
case QZ_DEFLATE_RAW:
size = 0;
break;
Expand All @@ -84,6 +91,9 @@ unsigned long outputHeaderSz(QzDataFormat_T data_fmt)
unsigned long size = 0;

switch (data_fmt) {
case QZ_DEFLATE_4B:
size = qz4BHeaderSz();
break;
case QZ_DEFLATE_RAW:
break;
case QZ_DEFLATE_GZIP:
Expand Down Expand Up @@ -150,13 +160,23 @@ void stdGzipHeaderGen(unsigned char *ptr, CpaDcRqResults *res)
hdr->os = 255;
}

void qz4BHeaderGen(unsigned char *ptr, CpaDcRqResults *res)
{
Qz4BH_T *hdr;
hdr = (Qz4BH_T *)ptr;
hdr->blk_size = res->produced;
}

void outputHeaderGen(unsigned char *ptr,
CpaDcRqResults *res,
QzDataFormat_T data_fmt)
{
QZ_DEBUG("Generate header\n");

switch (data_fmt) {
case QZ_DEFLATE_4B:
qz4BHeaderGen(ptr, res);
break;
case QZ_DEFLATE_RAW:
break;
case QZ_DEFLATE_GZIP:
Expand All @@ -174,10 +194,19 @@ int isQATProcessable(const unsigned char *ptr,
QzSess_T *const qz_sess)
{
QzGzH_T *h = (QzGzH_T *)ptr;
Qz4BH_T *h_4B;
StdGzF_T *qzFooter = NULL;
long buff_sz = (DEST_SZ(qz_sess->sess_params.hw_buff_sz) < *src_len ? DEST_SZ(
qz_sess->sess_params.hw_buff_sz) : *src_len);

if (qz_sess->sess_params.data_fmt == QZ_DEFLATE_4B) {
h_4B = (Qz4BH_T *)ptr;
if (h_4B->blk_size > DEST_SZ(qz_sess->sess_params.hw_buff_sz)) {
return 0;
}
return 1;
}

/*check if HW can process*/
if (h->std_hdr.id1 == 0x1f && \
h->std_hdr.id2 == 0x8b && \
Expand Down
6 changes: 6 additions & 0 deletions src/qatzip_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,10 @@ typedef struct QzGzH_S {
QzExtraField_T extra;
} QzGzH_T;

typedef struct Qz4BH_S {
uint32_t blk_size;
} Qz4BH_T;

typedef struct StdGzF_S {
uint32_t crc32;
uint32_t i_size;
Expand All @@ -248,11 +252,13 @@ typedef struct QzMem_S {
void dumpAllCounters(void);
int qzSetupHW(QzSession_T *sess, int i);
unsigned long qzGzipHeaderSz(void);
unsigned long qz4BHeaderSz(void);
unsigned long stdGzipHeaderSz(void);
unsigned long stdGzipFooterSz(void);
unsigned long outputHeaderSz(QzDataFormat_T data_fmt);
unsigned long outputFooterSz(QzDataFormat_T data_fmt);
void qzGzipHeaderGen(unsigned char *ptr, CpaDcRqResults *res);
void qz4BHeaderGen(unsigned char *ptr, CpaDcRqResults *res);
void stdGzipHeaderGen(unsigned char *ptr, CpaDcRqResults *res);
int qzGzipHeaderExt(const unsigned char *const ptr, QzGzH_T *hdr);
void outputHeaderGen(unsigned char *ptr,
Expand Down
22 changes: 20 additions & 2 deletions src/qatzip_sw.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ int qzSWCompress(QzSession_T *sess, const unsigned char *src,
QzDataFormat_T data_fmt = QZ_DATA_FORMAT_DEFAULT;
unsigned int chunk_sz = QZ_HW_BUFF_SZ;
QzGzH_T *qz_hdr = NULL;
Qz4BH_T *qz4B_header = NULL;

*src_len = 0;
*dest_len = 0;
Expand Down Expand Up @@ -127,6 +128,7 @@ int qzSWCompress(QzSession_T *sess, const unsigned char *src,
stream->total_out = 0;

switch (data_fmt) {
case QZ_DEFLATE_4B:
case QZ_DEFLATE_RAW:
windows_bits = -MAX_WBITS;
break;
Expand Down Expand Up @@ -158,6 +160,10 @@ int qzSWCompress(QzSession_T *sess, const unsigned char *src,
return QZ_FAIL;
}
qz_hdr = (QzGzH_T *)dest;
} else if (QZ_DEFLATE_4B == data_fmt) {
/* Need to reserve 4 bytes to fill the compressed length. */
qz4B_header = (Qz4BH_T *)dest;
dest = dest + sizeof(Qz4BH_T);
}
}

Expand Down Expand Up @@ -223,10 +229,13 @@ int qzSWCompress(QzSession_T *sess, const unsigned char *src,
* When data_fmt is QZ_DEFLATE_GZIP_EXT,
* we should fill src_sz & dest_sz in gzipext header field.
*/
if (QZ_DEFLATE_GZIP_EXT == data_fmt && qz_hdr) {
if (QZ_DEFLATE_GZIP_EXT == data_fmt && qz4B_header) {
qz_hdr->extra.qz_e.src_sz = stream->total_in;
qz_hdr->extra.qz_e.dest_sz = stream->total_out -
outputHeaderSz(data_fmt) - outputFooterSz(data_fmt);
} else if (QZ_DEFLATE_4B == data_fmt && qz4B_header) {
qz4B_header->blk_size = stream->total_out;
*dest_len = *dest_len + sizeof(Qz4BH_T);
}
ret = deflateEnd(stream);
stream->total_in = 0;
Expand All @@ -252,6 +261,7 @@ int qzSWDecompress(QzSession_T *sess, const unsigned char *src,
int windows_bits = 0;
unsigned int total_in;
unsigned int total_out;
unsigned int qz4B_header_len = 0;

QzSess_T *qz_sess = (QzSess_T *) sess->internal;
qz_sess->force_sw = 1;
Expand Down Expand Up @@ -283,6 +293,7 @@ int qzSWDecompress(QzSession_T *sess, const unsigned char *src,

QZ_DEBUG("decomp_sw data_fmt: %d\n", data_fmt);
switch (data_fmt) {
case QZ_DEFLATE_4B:
case QZ_DEFLATE_RAW:
windows_bits = -MAX_WBITS;
break;
Expand All @@ -294,6 +305,12 @@ int qzSWDecompress(QzSession_T *sess, const unsigned char *src,
}

if (InflateNull == qz_sess->inflate_stat) {
if (QZ_DEFLATE_4B == data_fmt) {
/* For QZ_DEFLATE_4B, we need to skip the header. */
stream->next_in = (z_const Bytef *)stream->next_in + sizeof(Qz4BH_T);
stream->avail_in = stream->avail_in - sizeof(Qz4BH_T);
qz4B_header_len = sizeof(Qz4BH_T);
}
ret = inflateInit2(stream, windows_bits);
if (Z_OK != ret) {
ret = QZ_FAIL;
Expand Down Expand Up @@ -339,7 +356,8 @@ int qzSWDecompress(QzSession_T *sess, const unsigned char *src,
}

*dest_len = GET_LOWER_32BITS(stream->total_out - total_out);
*src_len = GET_LOWER_32BITS(stream->total_in - total_in);
/* for Deflate_4B, we need to add the length of Deflate 4B header. */
*src_len = GET_LOWER_32BITS(stream->total_in - total_in + qz4B_header_len);

done:
QZ_DEBUG("Exit qzSWDecompress total_in: %u total_out: %u "
Expand Down
4 changes: 3 additions & 1 deletion test/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3712,7 +3712,7 @@ void *qzDecompressStreamWithBufferError(void *thd_arg)
" -D direction comp | decomp | both\n" \
" -F format [comp format]:[orig data size]/...\n" \
" -L comp_lvl 1 - " STR(MAX_LVL) "\n" \
" -O data_fmt deflate | gzip | gzipext\n" \
" -O data_fmt deflate | gzip | gzipext | deflate_4B\n" \
" -T huffmanType static | dynamic\n" \
" -r req_cnt_thrshold max inflight request num, default is 16\n" \
" -S thread_sleep the unit is milliseconds, default is a random time\n" \
Expand Down Expand Up @@ -3805,6 +3805,8 @@ int main(int argc, char *argv[])
g_params_th.data_fmt = QZ_DEFLATE_GZIP;
} else if (strcmp(optarg, "gzipext") == 0) {
g_params_th.data_fmt = QZ_DEFLATE_GZIP_EXT;
} else if (strcmp(optarg, "deflate_4B") == 0) {
g_params_th.data_fmt = QZ_DEFLATE_4B;
} else {
QZ_ERROR("Error service arg: %s\n", optarg);
return -1;
Expand Down
7 changes: 4 additions & 3 deletions utils/qzip.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ const struct option g_long_opts[] = {
{"huffmanhdr", 1, 0, 'H'}, /* set huffman header type */
{"level", 1, 0, 'L'}, /* set compression level */
{"chunksz", 1, 0, 'C'}, /* set chunk size */
{"output", 1, 0, 'O'}, /* set output header format(gzip, gzipext, 7z)*/
{"output", 1, 0, 'O'}, /* set output header format(gzip, gzipext, 7z,
deflate_4B) */
{"recursive", 0, 0, 'R'}, /* set recursive mode when compressing a
directory */
{"polling", 1, 0, 'P'}, /* set polling mode when compressing and
Expand Down Expand Up @@ -96,7 +97,7 @@ void help(void)
" -V, --version display version number",
" -L, --level set compression level",
" -C, --chunksz set chunk size",
" -O, --output set output header format(gzip|gzipext|7z)",
" -O, --output set output header format(gzip|gzipext|7z|deflate_4B)",
" -r, set max inflight request number",
" -R, set Recursive mode for a directory",
" -o, set output file name",
Expand Down Expand Up @@ -450,7 +451,7 @@ int makeOutName(const char *in_name, const char *out_name,
SUFFIX_GZ);
} else {
if (!hasSuffix(in_name)) {
QZ_ERROR("Error: %s: Wrong suffix. Supported suffix: 7z/gz.\n",
QZ_ERROR("%s: Wrong suffix. Supported suffix: 7z/gz.\n",
in_name);
return -1;
}
Expand Down
2 changes: 2 additions & 0 deletions utils/qzip_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ int main(int argc, char **argv)
g_params_th.data_fmt = QZ_DEFLATE_GZIP_EXT;
} else if (strcmp(optarg, "7z") == 0) {
g_params_th.data_fmt = QZ_DEFLATE_RAW;
} else if (strcmp(optarg, "deflate_4B") == 0) {
g_params_th.data_fmt = QZ_DEFLATE_4B;
} else {
QZ_ERROR("Error gzip header format arg: %s\n", optarg);
return -1;
Expand Down

0 comments on commit ecb32e7

Please sign in to comment.