Add Zstandard compression codec

.eslintrc.json

@@ -0,0 +1,14 @@
+{
+    "plugins": [
+        "prettier"
+    ],
+    "rules": {
+        "prettier/prettier": "error"
+    },
+    "env": {
+        "node": true
+    },
+    "parserOptions": {
+        "ecmaVersion": 2020
+    }
+}

.github/funding.yml

@@ -0,0 +1 @@
+github: [Nevon]

.github/workflows/test.yml

@@ -0,0 +1,37 @@
+name: CI
+
+on:
+  push:
+    branches: [ $default-branch ]
+  pull_request:
+    branches: [ $default-branch ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [10.x, 12.x, 14.x]
+
+    env:
+      CI: true
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Install dependencies
+      run: npm ci
+    - name: Lint
+      run: npm run lint
+    - name: Run Kafka
+      run: docker-compose up -d
+    - name: Wait for dependencies
+      run: node scripts/waitForKafka.js
+    - name: Run tests
+      run: npm test
+    - name: Verify types
+      run: npm run test:types

.gitignore

@@ -0,0 +1,8 @@
+.DS_Store
+node_modules
+test-report.xml
+junit.xml
+coverage/
+yarn-error.log
+.vscode
+.vs

.npmignore

@@ -0,0 +1,16 @@
+.DS_Store
+test-report.xml
+junit.xml
+coverage/
+yarn-error.log
+**/*.spec.js
+__tests__
+__snapshots__
+scripts/
+docker-compose.yml
+jest.config.js
+.eslintrc.json
+.eslintignore
+.github/
+types/test.ts
+*.tgz

LICENSE

@@ -0,0 +1,24 @@
+The MIT License
+
+Copyright (c) 2020 Tommy Brunn ([email protected])
+
+Permission is hereby granted, free of charge,
+to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to
+deal in the Software without restriction, including
+without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom
+the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
+ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,79 @@
+# `@kafkajs/zstd`
+
+ZStandard codec for KafkaJS.
+
+**WIP: Currently publishing compressed messages is failing. Not ready for usage yet!**
+
+## Installation
+
+```sh
+npm install --save @kafkajs/zstd
+```
+
+## Configuration
+
+```javascript
+const {  CompressionTypes, CompressionCodecs } = require('kafkajs')
+const ZstdCodec = require('@kafkajs/zstd')
+
+// Both compressionParams and decompressionParams are optional
+const compressionParams = { level: 1 }
+const decompressionParams = {}
+CompressionCodecs[CompressionTypes.ZSTD] = ZstdCodec(compressionParams, decompressionParams)
+```
+
+### `compressionParams`
+
+**Optional** Allows you to configure the compression level and training data.
+
+```js
+{
+    /*  
+     * Higher compression level means faster at the cost of compression ratio or memory usage.
+     * See https://facebook.github.io/zstd/
+     */
+    level: 3,
+
+    /*
+     * Training data for improving performance on small payloads.
+     * See https://facebook.github.io/zstd/#small-data
+     */
+    dict: trainingData,
+    dictSize: Buffer.byteLength(trainingData)
+}
+```
+
+### `decompressionParams`
+
+**Optional** Allows you to configure the training data.
+
+```js
+{
+    /*
+     * Training data for improving performance on small payloads.
+     * See https://facebook.github.io/zstd/#small-data
+     */
+    dict: trainingData,
+    dictSize: Buffer.byteLength(trainingData)
+}
+```
+
+## Testing
+
+```sh
+$ docker-compose up -d && node scripts/waitForKafka.js
+$ npm test
+
+# Interactive mode
+$ npm run test:watch
+
+# Verify type definitions
+$ npm run test:types
+
+# Lint
+$ npm run lint
+```
+
+## License
+
+See [LICENSE](https://github.com/kafkajs/zstd/blob/master/LICENSE) for more details.

docker-compose.yml

@@ -0,0 +1,30 @@
+---
+version: '2'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  kafka:
+    image: confluentinc/cp-kafka:5.4.2
+    hostname: kafka
+    container_name: kafka
+    labels:
+      - 'custom.project=kafkajs-zstd'
+      - 'custom.service=kafka'
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
+      KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO'
+      CONFLUENT_SUPPORT_METRICS_ENABLE: 'false'