From 4bf3a9b38183ba36c5b156504ebcf6afa9913ba3 Mon Sep 17 00:00:00 2001 From: Kanbaru Date: Sat, 4 Dec 2021 17:08:30 +0800 Subject: [PATCH] Add binary/zstd encoding method. --- CHANGELOG.md | 8 +++ README.md | 16 +++-- nimassets.nimble | 1 + src/nimassets.nim | 168 +++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 176 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1315144..be00c70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.2.2] - 2021-12-04 + +### Added + +- Add support for encoding asset to seq[byte] +- Add zstd compression +- Add zstd and base64 combined encoding (zstd -> byteArrayToString -> base64) + ## [0.2.1] - 2021-07-28 ### Changed diff --git a/README.md b/README.md index d74b035..dc2cb8e 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,14 @@ nimassets `Nim Assets` is heavily inspired by [go-bindata](https://github.com/jt ## Usage ```bash -nimassets 0.2.1 (Bundle your assets into nim file) - -h | --help : show help - -v | --version : show version - -o | --output : output filename - -f | --fast : faster generation - -d | --dir : dir to include (recursively) [can be used multiple times -d=DIR1 -d=DIR2 ...] +nimassets 0.2.2 (Bundle your assets into nim file) + -h | --help : show help + -v | --version : show version + -o | --output : output filename + -f | --fast : faster generation + -d | --dir : dir to include (recursively) [can be used multiple times -d=DIR1 -d=DIR2 ...] + -t | --type : binary | base64 | zstd | base64zstd + -cl | --compresslevel : compress level for zstd, default is 3 ``` ### Bundle @@ -21,7 +23,7 @@ nimassets -d=templatesdir -o=assetsfile.nim ``` `-f` or `--fast` flag can help with large assets directories - +`-t` or `--type` encoding method, default is base64 ### Use Assets diff --git a/nimassets.nimble b/nimassets.nimble index 1210b73..923dd20 100644 --- a/nimassets.nimble +++ b/nimassets.nimble @@ -13,6 +13,7 @@ skipDirs = @["examples", "tests"] # Dependencies requires "nim >= 1.4.0" +requires "zstd >= 0.5.0" proc build() = exec "nimble build --threads:on -d:release" diff --git a/src/nimassets.nim b/src/nimassets.nim index 8c4109f..a2dc576 100644 --- a/src/nimassets.nim +++ b/src/nimassets.nim @@ -4,6 +4,7 @@ import strformat, strutils, base64, + zstd/compress, parseopt, threadpool @@ -16,9 +17,25 @@ const buildCommit* = staticExec("git rev-parse HEAD") ## \ # const latestTag* = staticExec("git describe --abbrev=0 --tags") ## \ ## `latestTag` latest tag on this branch -const versionString* = &"0.2.1 ({buildBranchName}/{buildCommit})" +const versionString* = &"0.2.2 ({buildBranchName}/{buildCommit})" -const assetsFileHeader = """ +const assetsFileHeaderBinary = """ +import tables + +var assets: Table[string, seq[byte]] + +proc getAsset*(path: string): seq[byte] = + result = assets[path] + +proc getAssetToStr*(path: string): string = + proc toString(bytes: openArray[byte]): string = + result = newString(bytes.len) + copyMem(result[0].addr, bytes[0].unsafeAddr, bytes.len) + result = toString(getAsset(path)) + +""" + +const assetsFileHeaderBase64 = """ import tables, base64 var assets: Table[string, string] @@ -26,16 +43,129 @@ var assets: Table[string, string] proc getAsset*(path: string): string = result = assets[path].decode() +func toByteSeq(str: string): seq[byte] {.inline.} = + ## Copy ``string`` memory into an immutable``seq[byte]``. + let length = str.len + if length > 0: + result = newSeq[byte](length) + copyMem(result[0].unsafeAddr, str[0].unsafeAddr, length) + +proc getAssetToByteSeq*(path: string): string = + result = toByteSeq (getAsset path) + +""" + +const assetsFileHeaderZstd = """ +import tables, zstd/decompress + +var assets: Table[string, seq[byte]] + +proc getAsset*(path: string): seq[byte] = + result = decompress(assets[path]) + +proc getAssetToStr*(path: string): string = + proc toString(bytes: openArray[byte]): string = + result = newString(bytes.len) + copyMem(result[0].addr, bytes[0].unsafeAddr, bytes.len) + result = toString(getAsset(path)) + +""" + +const assetsFileHeaderBase64Zstd = """ +import tables, base64, zstd/decompress + +var assets: Table[string, string] + +func stringToByteSeq(str: string): seq[byte] {.inline.} = + ## Copy ``string`` memory into an immutable``seq[byte]``. + let length = str.len + if length > 0: + result = newSeq[byte](length) + copyMem(result[0].unsafeAddr, str[0].unsafeAddr, length) + +proc byteArrayToString(bytes: openArray[byte]): string = + result = newString(bytes.len) + copyMem(result[0].addr, bytes[0].unsafeAddr, bytes.len) + +proc getAsset*(path: string): seq[byte] = + result = decompress(stringToByteSeq(assets[path].decode())) + +proc getAssetToStr*(path: string): string = + result = byteArrayToString(getAsset path) + """ +type DataType = enum + tBinary = 1 + tBase64 = 2 + tZstd = 3 + tBase64Zstd = 4 + +var dataType : DataType = tBase64 +var compressLevel : int = 3 + +proc byteArrayToString(bytes: openArray[byte]): string = + result = newString(bytes.len) + copyMem(result[0].addr, bytes[0].unsafeAddr, bytes.len) + +func stringToByteSeq(str: string): seq[byte] {.inline.} = + ## Copy ``string`` memory into an immutable``seq[byte]``. + let length = str.len + if length > 0: + result = newSeq[byte](length) + copyMem(result[0].unsafeAddr, str[0].unsafeAddr, length) + proc handleFile(path: string): string {.thread.} = var val, valString: string - val = readFile(path).encode() - valString = "\"\"\"" & val & "\"\"\"" + + stdout.write fmt"{path} ... " + case dataType + of tBinary: + let file : File = open(path) + var input = newSeq[byte](file.getFileSize()) + discard file.readBytes(input, 0, file.getFileSize()) + file.close() + valString = "@[byte " + for i, b in input: + if i < input.len - 1: + valString &= fmt"0x{toHex(b)}," + else: + valString &= fmt"0x{toHex(b)}]" + of tBase64: + val = readFile(path).encode() + valString = "\"\"\"" & val & "\"\"\"" + of tZstd: + let file : File = open(path) + var input = newSeq[byte](file.getFileSize()) + discard file.readBytes(input, 0, file.getFileSize()) + file.close() + + stdout.write fmt"zstd-level:{compressLevel} ... " + let compressed = compress(input, level=compressLevel) + + stdout.write "build_string ... " + valString = "@[byte " + for i, b in compressed: + if i < compressed.len - 1: + valString &= fmt"0x{toHex(b)}," + else: + valString &= fmt"0x{toHex(b)}]" + of tBase64Zstd: + # zstd -> byteArrayToString -> base64 + let file : File = open(path) + var input = newSeq[byte](file.getFileSize()) + discard file.readBytes(input, 0, file.getFileSize()) + file.close() + + val = byteArrayToString(compress(input, level=compressLevel)).encode() + stdout.write fmt"zstd-level:{compressLevel} ... " + valString = "\"\"\"" & val & "\"\"\"" + if detectOs(Windows): result = &"""assets["{escape(path, prefix="", suffix="")}"] = {valString}""" & "\n\n" else: result = &"""assets["{path}"] = {valString}""" & "\n\n" + stdout.write "ok\n" proc generateDirAssetsSimple*(dir: string): string = for path in expandTilde(dir).walkDirRec(): @@ -56,7 +186,12 @@ proc generateDirAssetsSpawn*(dir: string): string = proc createAssetsFile*(dirs:seq[string], outputfile="assets.nim", fast=false, compress=false) = var generator: proc(s:string): string - data = assetsFileHeader + data = + case dataType + of tBinary: assetsFileHeaderBinary + of tBase64: assetsFileHeaderBase64 + of tZstd: assetsFileHeaderZstd + of tBase64Zstd: assetsFileHeaderBase64Zstd if fast: generator = generateDirAssetsSpawn @@ -72,11 +207,13 @@ proc writeHelp() = #-c | --compress : compress echo &""" nimassets {versionString} (Bundle your assets into nim file) - -h | --help : show help - -v | --version : show version - -o | --output : output filename - -f | --fast : faster generation - -d | --dir : dir to include (recursively) + -h | --help : show help + -v | --version : show version + -o | --output : output filename + -f | --fast : faster generation + -d | --dir : dir to include (recursively) + -t | --type : binary | base64 | zstd | base64zstd + -cl | --compresslevel : compress level for zstd """ proc writeVersion() = @@ -103,6 +240,17 @@ proc cli*() = writeVersion() quit() # of "compress", "c": compress= true + of "type", "t": + case val + of "binary": + dataType = tBinary + of "base64": + dataType = tBase64 + of "zstd": + dataType = tZstd + of "base64zstd": + dataType = tBase64Zstd + of "compresslevel", "cl": compressLevel = parseInt(val) of "fast", "f": fast = true of "dir", "d": dirs.add(val) of "output", "o": output = val