Moleculer logger for ClickHouse

This is a fork from native Datadog logger

Description

Easy to save logs to ClickHouse with moleculer

Install

$ npm install @1xtr/moleculer-ch-logger --save

Usage

const ClickHouseLogger = require('@1xtr/moleculer-ch-logger')

module.exports = {
    logger: new ClickHouseLogger({
      // put here your options
    })
}

Default options

const defaultOptions = {
  url: 'http://localhost',
  port: 8123,
  dbName: 'default',
  dbUser: 'default',
  dbPassword: '',
  dbTableName: 'logs',
  // use source like TAG
  source: process.env.MOL_NODE_NAME || 'moleculer',
  hostname: hostname(),
  objectPrinter: null,
  interval: 10 * 1000,
  timeZone: 'Europe/Istanbul',
  tableTTL: 'date + INTERVAL 1 DAY RECOMPRESS CODEC(ZSTD(3))',
  useBuffer: false,
}

Log table

const body = `CREATE TABLE IF NOT EXISTS ${this.opts.dbTableName} (
          timestamp DateTime64(3, '${this.opts.timeZone}') DEFAULT now(),
          requestID String,
          subdomain String,
          caller String,
          level String,
          message String,
          nodeID String,
          namespace String,
          service String,
          version String,
          source String,
          hostname String,
          date Date DEFAULT today())
      ENGINE = MergeTree()
      ORDER BY (toStartOfHour(timestamp), service, level, subdomain, requestID, timestamp)
      PRIMARY KEY (toStartOfHour(timestamp), service, level, subdomain, requestID)
      PARTITION BY (date, toStartOfDay(timestamp))
      TTL ${this.opts.tableTTL}
      SETTINGS index_granularity = 8192;`

Buffer

const body = `CREATE TABLE IF NOT EXISTS ${this.opts.dbTableName}_buffer
      as ${this.opts.dbTableName}
      ENGINE = Buffer('${this.opts.dbName}', '${this.opts.dbTableName}', 16, 10, 100, 1000, 10000, 10000, 100000);`

Logger mixin

That mixin I use for logging (Click for open)

const defaultContext = {
  requestID: '',
  meta: { customer: { subdomain: '' } },
}

module.exports = {
  name: 'logger',
  methods: {
    log(data, ctx = defaultContext) {
      this.sendLog(data, ctx, 'info')
    },
    error(title, error, ctx = defaultContext) {
      this.logger.error({
        title,
        subdomain: ctx.meta.customer ? ctx.meta.customer.subdomain : '',
        caller: ctx.caller || '',
        error,
        requestID: ctx.requestID || '',
      })
    },
    err(error, ctx = defaultContext) {
      this.logger.error({
        title: error.message,
        subdomain: ctx.meta.customer ? ctx.meta.customer.subdomain : '',
        caller: ctx.caller || '',
        error,
        requestID: ctx.requestID || '',
      })
    },
    warn(data, ctx = defaultContext) {
      this.sendLog(data, ctx, 'warn')
    },
    sendLog(data, ctx, logType) {
      if (!ctx.meta.customer) {
        ctx.meta.customer = { subdomain: '' }
      }

      if (typeof data === 'string') {
        return this.logger[logType]({
          requestID: ctx.requestID || '',
          subdomain: ctx.meta.customer.subdomain || '',
          caller: ctx.caller || '',
          title: data,
        })
      }

      if (typeof data !== 'object') {
        return this.logger[logType]({
          requestID: ctx.requestID || '',
          subdomain: ctx.meta.customer.subdomain || '',
          caller: ctx.caller || '',
          data,
        })
      }

      return this.logger[logType]({
        requestID: ctx.requestID || '',
        subdomain: ctx.meta.customer.subdomain || '',
        caller: ctx.caller || '',
        ...data,
      })
    },
  },
}

Documentation

For more details read docs