Add binary/zstd encoding method.

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.2] - 2021-12-04
+
+### Added
+
+- Add support for encoding asset to seq[byte]
+- Add zstd compression
+- Add zstd and base64 combined encoding (zstd -> byteArrayToString -> base64)
+
 ## [0.2.1] - 2021-07-28
 
 ### Changed

README.md

@@ -4,12 +4,14 @@ nimassets `Nim Assets` is heavily inspired by [go-bindata](https://github.com/jt
 
 ## Usage
 ```bash
-nimassets 0.2.1 (Bundle your assets into nim file)
-    -h | --help         : show help
-    -v | --version      : show version
-    -o | --output       : output filename
-    -f | --fast         : faster generation
-    -d | --dir          : dir to include (recursively) [can be used multiple times -d=DIR1 -d=DIR2 ...]
+nimassets 0.2.2 (Bundle your assets into nim file)
+    -h  | --help          : show help
+    -v  | --version       : show version
+    -o  | --output        : output filename
+    -f  | --fast          : faster generation
+    -d  | --dir           : dir to include (recursively) [can be used multiple times -d=DIR1 -d=DIR2 ...]
+    -t  | --type          : binary | base64 | zstd | base64zstd
+    -cl | --compresslevel : compress level for zstd, default is 3
 ```
 
 ### Bundle
@@ -21,7 +23,7 @@ nimassets -d=templatesdir -o=assetsfile.nim
 ```
 
 `-f` or `--fast` flag can help with large assets directories
-
+`-t` or `--type` encoding method, default is base64
 
 
 ### Use Assets

nimassets.nimble

@@ -13,6 +13,7 @@ skipDirs      = @["examples", "tests"]
 # Dependencies
 
 requires "nim >= 1.4.0"
+requires "zstd >= 0.5.0"
 
 proc build() =
   exec "nimble build --threads:on -d:release"

src/nimassets.nim

@@ -4,6 +4,7 @@ import
   strformat,
   strutils,
   base64,
+  zstd/compress,
   parseopt,
   threadpool
 
@@ -16,26 +17,155 @@ const buildCommit* = staticExec("git rev-parse HEAD")  ## \
 # const latestTag* = staticExec("git describe --abbrev=0 --tags") ## \
 ## `latestTag` latest tag on this branch
 
-const versionString* = &"0.2.1 ({buildBranchName}/{buildCommit})"
+const versionString* = &"0.2.2 ({buildBranchName}/{buildCommit})"
 
-const assetsFileHeader = """
+const assetsFileHeaderBinary = """
+import tables
+
+var assets: Table[string, seq[byte]]
+
+proc getAsset*(path: string): seq[byte] =
+  result = assets[path]
+
+proc getAssetToStr*(path: string): string =
+  proc toString(bytes: openArray[byte]): string =
+    result = newString(bytes.len)
+    copyMem(result[0].addr, bytes[0].unsafeAddr, bytes.len)
+  result = toString(getAsset(path))
+
+"""
+
+const assetsFileHeaderBase64 = """
 import tables, base64
 
 var assets: Table[string, string]
 
 proc getAsset*(path: string): string =
   result = assets[path].decode()
 
+func toByteSeq(str: string): seq[byte] {.inline.} =
+  ## Copy ``string`` memory into an immutable``seq[byte]``.
+  let length = str.len
+  if length > 0:
+    result = newSeq[byte](length)
+    copyMem(result[0].unsafeAddr, str[0].unsafeAddr, length)
+
+proc getAssetToByteSeq*(path: string): string =
+  result = toByteSeq (getAsset path)
+
+"""
+
+const assetsFileHeaderZstd = """
+import tables, zstd/decompress
+
+var assets: Table[string, seq[byte]]
+
+proc getAsset*(path: string): seq[byte] =
+  result = decompress(assets[path])
+
+proc getAssetToStr*(path: string): string =
+  proc toString(bytes: openArray[byte]): string =
+    result = newString(bytes.len)
+    copyMem(result[0].addr, bytes[0].unsafeAddr, bytes.len)
+  result = toString(getAsset(path))
+
+"""
+
+const assetsFileHeaderBase64Zstd = """
+import tables, base64, zstd/decompress
+
+var assets: Table[string, string]
+
+func stringToByteSeq(str: string): seq[byte] {.inline.} =
+  ## Copy ``string`` memory into an immutable``seq[byte]``.
+  let length = str.len
+  if length > 0:
+    result = newSeq[byte](length)
+    copyMem(result[0].unsafeAddr, str[0].unsafeAddr, length)
+
+proc byteArrayToString(bytes: openArray[byte]): string =
+  result = newString(bytes.len)
+  copyMem(result[0].addr, bytes[0].unsafeAddr, bytes.len)
+
+proc getAsset*(path: string): seq[byte] =
+  result = decompress(stringToByteSeq(assets[path].decode()))
+
+proc getAssetToStr*(path: string): string =
+  result = byteArrayToString(getAsset path)
+
 """
 
+type DataType = enum
+  tBinary = 1
+  tBase64 = 2
+  tZstd = 3
+  tBase64Zstd = 4
+
+var dataType : DataType = tBase64
+var compressLevel : int = 3
+
+proc byteArrayToString(bytes: openArray[byte]): string =
+  result = newString(bytes.len)
+  copyMem(result[0].addr, bytes[0].unsafeAddr, bytes.len)
+
+func stringToByteSeq(str: string): seq[byte] {.inline.} =
+  ## Copy ``string`` memory into an immutable``seq[byte]``.
+  let length = str.len
+  if length > 0:
+    result = newSeq[byte](length)
+    copyMem(result[0].unsafeAddr, str[0].unsafeAddr, length)
+
 proc handleFile(path: string): string {.thread.} =
   var val, valString: string
-  val = readFile(path).encode()
-  valString = "\"\"\"" & val & "\"\"\""
+
+  stdout.write fmt"{path} ... "
+  case dataType
+  of tBinary:
+    let file : File = open(path)
+    var input = newSeq[byte](file.getFileSize())
+    discard file.readBytes(input, 0, file.getFileSize())
+    file.close()
+    valString = "@[byte "
+    for i, b in input:
+      if i < input.len - 1:
+        valString &= fmt"0x{toHex(b)},"
+      else:
+        valString &= fmt"0x{toHex(b)}]"
+  of tBase64:
+    val = readFile(path).encode()
+    valString = "\"\"\"" & val & "\"\"\""
+  of tZstd:
+    let file : File = open(path)
+    var input = newSeq[byte](file.getFileSize())
+    discard file.readBytes(input, 0, file.getFileSize())
+    file.close()
+
+    stdout.write fmt"zstd-level:{compressLevel} ... "
+    let compressed = compress(input, level=compressLevel)
+
+    stdout.write "build_string ... "
+    valString = "@[byte "
+    for i, b in compressed:
+      if i < compressed.len - 1:
+        valString &= fmt"0x{toHex(b)},"
+      else:
+        valString &= fmt"0x{toHex(b)}]"
+  of tBase64Zstd:
+    # zstd -> byteArrayToString -> base64
+    let file : File = open(path)
+    var input = newSeq[byte](file.getFileSize())
+    discard file.readBytes(input, 0, file.getFileSize())
+    file.close()
+
+    val = byteArrayToString(compress(input, level=compressLevel)).encode()
+    stdout.write fmt"zstd-level:{compressLevel} ... "
+    valString = "\"\"\"" & val & "\"\"\""
+
   if detectOs(Windows):
     result = &"""assets["{escape(path, prefix="", suffix="")}"] = {valString}""" & "\n\n"
   else:
     result = &"""assets["{path}"] = {valString}""" & "\n\n"
+    stdout.write "ok\n"
 
 proc generateDirAssetsSimple*(dir: string): string =
   for path in expandTilde(dir).walkDirRec():
@@ -56,7 +186,12 @@ proc generateDirAssetsSpawn*(dir: string): string =
 proc createAssetsFile*(dirs:seq[string], outputfile="assets.nim", fast=false, compress=false) =
   var
     generator: proc(s:string): string
-    data = assetsFileHeader
+    data =
+      case dataType
+      of tBinary: assetsFileHeaderBinary
+      of tBase64: assetsFileHeaderBase64
+      of tZstd: assetsFileHeaderZstd
+      of tBase64Zstd: assetsFileHeaderBase64Zstd
 
   if fast:
     generator = generateDirAssetsSpawn
@@ -72,11 +207,13 @@ proc writeHelp() =
   #-c | --compress     : compress
   echo &"""
 nimassets {versionString} (Bundle your assets into nim file)
-    -h | --help         : show help
-    -v | --version      : show version
-    -o | --output       : output filename
-    -f | --fast         : faster generation
-    -d | --dir          : dir to include (recursively)
+    -h  | --help          : show help
+    -v  | --version       : show version
+    -o  | --output        : output filename
+    -f  | --fast          : faster generation
+    -d  | --dir           : dir to include (recursively)
+    -t  | --type          : binary | base64 | zstd | base64zstd
+    -cl | --compresslevel : compress level for zstd
 """
 
 proc writeVersion() =
@@ -103,6 +240,17 @@ proc cli*() =
               writeVersion()
               quit()
           # of "compress", "c": compress= true
+          of "type", "t": 
+            case val
+            of "binary":
+              dataType = tBinary
+            of "base64":
+              dataType = tBase64
+            of "zstd":
+              dataType = tZstd
+            of "base64zstd":
+              dataType = tBase64Zstd
+          of "compresslevel", "cl": compressLevel = parseInt(val)
           of "fast", "f": fast = true
           of "dir", "d": dirs.add(val)
           of "output", "o": output = val