From 00f9764e1fa0b8bfc1817f900520ee16deb0a384 Mon Sep 17 00:00:00 2001 From: itanka9 Date: Wed, 6 Nov 2024 10:56:39 +0300 Subject: [PATCH] json2pbf --- .github/workflows/deploy.yaml | 50 ++ index.html | 2 +- index.ts | 66 ++- package-lock.json | 880 +++++++++++++++++++++++++++++++++- package.json | 6 + src/bfsm.ts | 115 +++++ src/config.ts | 28 +- src/json2pbf.ts | 537 +++++++++++---------- src/worker.ts | 40 +- tsconfig.json | 1 + 10 files changed, 1440 insertions(+), 285 deletions(-) create mode 100644 .github/workflows/deploy.yaml create mode 100644 src/bfsm.ts diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..043d0ed --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,50 @@ +# Simple workflow for deploying static content to GitHub Pages +name: Deploy static content to Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ['main'] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets the GITHUB_TOKEN permissions to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow one concurrent deployment +concurrency: + group: 'pages' + cancel-in-progress: true + +jobs: + # Single deploy job since we're just deploying + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Node + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'npm' + - name: Install dependencies + run: npm install + - name: Build + run: npm run build -- --base=/json2pbf/ + - name: Setup Pages + uses: actions/configure-pages@v3 + - name: Upload artifact + uses: actions/upload-pages-artifact@v1 + with: + path: './dist' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v1 \ No newline at end of file diff --git a/index.html b/index.html index 27cd9ba..4cc1bbb 100644 --- a/index.html +++ b/index.html @@ -7,6 +7,6 @@ Wroker test - + diff --git a/index.ts b/index.ts index de89cc5..1880b40 100644 --- a/index.ts +++ b/index.ts @@ -1,10 +1,37 @@ -import { unpackJson } from './src/json2pbf'; +import { unpack } from './src/json2pbf'; import { ITERATIONS_COUNT, JSON_TEMPLATE } from './src/config'; +import { unpackBfsm } from './src/bfsm'; -const worker = new Worker(new URL('./src/worker.ts', import.meta.url)); +const worker = new Worker(new URL('./src/worker.ts', import.meta.url), { + type: 'module', + }); const decoder = new TextDecoder('utf-8'); + +function unpackAndBuildMap(data: ArrayBuffer, columnar = false) { + const unpacked = unpack(data); + const map = new Map(); + if (columnar) { + for (let i = 0; i < unpacked.id.length; i++) { + const featureState = { + hidden: unpacked.hidden[i] + } + map.set(unpacked.id[i], featureState); + + } + } else { + for (let i = 0; i < unpacked.length; i++) { + const item = unpacked[i]; + const featureState = { + hidden: item.hidden + } + map.set(item.id, featureState); + } + } + return map; +} + async function makeTests() { - function waitResults(type: number, unpack: (data: any) => { data: any; hash: string }) { + function waitResults(type: number, unpack: (data: any) => { data: any; }) { const results: any[] = []; const converted: any[] = []; return new Promise((res) => { @@ -34,28 +61,49 @@ async function makeTests() { console.log('Массив ', JSON_TEMPLATE); console.log(`Передаем из воркера в главный поток ${ITERATIONS_COUNT} раз`); console.group('передаем типизированный массив напрямую'); - await waitResults(0, (data) => ({ data, hash: data.toString() })); + await waitResults(0, (data) => ({ data })); console.groupEnd(); console.group('передаем объект напрямую'); - await waitResults(1, (data) => ({ data, hash: JSON.stringify(data) })); + await waitResults(1, (data) => ({ data })); console.groupEnd(); console.group('передаем как строку'); - await waitResults(2, (data) => ({ data: JSON.parse(data), hash: data })); + await waitResults(2, (data) => ({ data: JSON.parse(data) })); console.groupEnd(); console.group('JSON.parse + TextDecoder'); await waitResults(3, (data) => { const hash = decoder.decode(data); - return { data: JSON.parse(hash), hash }; + return { data: JSON.parse(hash) }; }); console.groupEnd(); console.group('PBF'); await waitResults(4, (data) => { - const unpacked = unpackJson(data); - return { data: unpacked, hash: JSON.stringify(unpacked) }; + const unpacked = unpackAndBuildMap(data); + return { data: unpacked }; + }); + console.groupEnd(); + + console.group('PBF (row)'); + await waitResults(5, (data) => { + const unpacked = unpackAndBuildMap(data); + return { data: unpacked }; + }); + console.groupEnd(); + + console.group('PBF (columnar)'); + await waitResults(6, (data) => { + const unpacked = unpackAndBuildMap(data, true); + return { data: unpacked }; + }); + console.groupEnd(); + + console.group('BFSM'); + await waitResults(7, (data) => { + const unpacked = unpackBfsm(data); + return { data: unpacked }; }); console.groupEnd(); } diff --git a/package-lock.json b/package-lock.json index 4b5aafd..f37d86f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,40 +1,900 @@ { "name": "worker-msg-test", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 3, "requires": true, - "dependencies": { - "ieee754": { + "packages": { + "": { + "name": "worker-msg-test", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "pbf": "^3.2.1", + "tslib": "^2.3.0" + }, + "devDependencies": { + "typescript": "^5.6.3", + "vite": "^5.4.10" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.24.4.tgz", + "integrity": "sha512-jfUJrFct/hTA0XDM5p/htWKoNNTbDLY0KRwEt6pyOA6k2fmk0WVwl65PdUdJZgzGEHWx+49LilkcSaumQRyNQw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.24.4.tgz", + "integrity": "sha512-j4nrEO6nHU1nZUuCfRKoCcvh7PIywQPUCBa2UsootTHvTHIoIu2BzueInGJhhvQO/2FTRdNYpf63xsgEqH9IhA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.24.4.tgz", + "integrity": "sha512-GmU/QgGtBTeraKyldC7cDVVvAJEOr3dFLKneez/n7BvX57UdhOqDsVwzU7UOnYA7AAOt+Xb26lk79PldDHgMIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.24.4.tgz", + "integrity": "sha512-N6oDBiZCBKlwYcsEPXGDE4g9RoxZLK6vT98M8111cW7VsVJFpNEqvJeIPfsCzbf0XEakPslh72X0gnlMi4Ddgg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.24.4.tgz", + "integrity": "sha512-py5oNShCCjCyjWXCZNrRGRpjWsF0ic8f4ieBNra5buQz0O/U6mMXCpC1LvrHuhJsNPgRt36tSYMidGzZiJF6mw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.24.4.tgz", + "integrity": "sha512-L7VVVW9FCnTTp4i7KrmHeDsDvjB4++KOBENYtNYAiYl96jeBThFfhP6HVxL74v4SiZEVDH/1ILscR5U9S4ms4g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.24.4.tgz", + "integrity": "sha512-10ICosOwYChROdQoQo589N5idQIisxjaFE/PAnX2i0Zr84mY0k9zul1ArH0rnJ/fpgiqfu13TFZR5A5YJLOYZA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.24.4.tgz", + "integrity": "sha512-ySAfWs69LYC7QhRDZNKqNhz2UKN8LDfbKSMAEtoEI0jitwfAG2iZwVqGACJT+kfYvvz3/JgsLlcBP+WWoKCLcw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.24.4.tgz", + "integrity": "sha512-uHYJ0HNOI6pGEeZ/5mgm5arNVTI0nLlmrbdph+pGXpC9tFHFDQmDMOEqkmUObRfosJqpU8RliYoGz06qSdtcjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.24.4.tgz", + "integrity": "sha512-38yiWLemQf7aLHDgTg85fh3hW9stJ0Muk7+s6tIkSUOMmi4Xbv5pH/5Bofnsb6spIwD5FJiR+jg71f0CH5OzoA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.24.4.tgz", + "integrity": "sha512-q73XUPnkwt9ZNF2xRS4fvneSuaHw2BXuV5rI4cw0fWYVIWIBeDZX7c7FWhFQPNTnE24172K30I+dViWRVD9TwA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.24.4.tgz", + "integrity": "sha512-Aie/TbmQi6UXokJqDZdmTJuZBCU3QBDA8oTKRGtd4ABi/nHgXICulfg1KI6n9/koDsiDbvHAiQO3YAUNa/7BCw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.24.4.tgz", + "integrity": "sha512-P8MPErVO/y8ohWSP9JY7lLQ8+YMHfTI4bAdtCi3pC2hTeqFJco2jYspzOzTUB8hwUWIIu1xwOrJE11nP+0JFAQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.24.4.tgz", + "integrity": "sha512-K03TljaaoPK5FOyNMZAAEmhlyO49LaE4qCsr0lYHUKyb6QacTNF9pnfPpXnFlFD3TXuFbFbz7tJ51FujUXkXYA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.24.4.tgz", + "integrity": "sha512-VJYl4xSl/wqG2D5xTYncVWW+26ICV4wubwN9Gs5NrqhJtayikwCXzPL8GDsLnaLU3WwhQ8W02IinYSFJfyo34Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.24.4.tgz", + "integrity": "sha512-ku2GvtPwQfCqoPFIJCqZ8o7bJcj+Y54cZSr43hHca6jLwAiCbZdBUOrqE6y29QFajNAzzpIOwsckaTFmN6/8TA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.24.4.tgz", + "integrity": "sha512-V3nCe+eTt/W6UYNr/wGvO1fLpHUrnlirlypZfKCT1fG6hWfqhPgQV/K/mRBXBpxc0eKLIF18pIOFVPh0mqHjlg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.24.4.tgz", + "integrity": "sha512-LTw1Dfd0mBIEqUVCxbvTE/LLo+9ZxVC9k99v1v4ahg9Aak6FpqOfNu5kRkeTAn0wphoC4JU7No1/rL+bBCEwhg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/estree": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } }, - "pbf": { + "node_modules/pbf": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/pbf/-/pbf-3.2.1.tgz", "integrity": "sha512-ClrV7pNOn7rtmoQVF4TS1vyU0WhYRnP92fzbfF75jAIwpnzdJXf8iTd4CMEqO4yUenH6NDqLiwjqlh6QgZzgLQ==", - "requires": { + "dependencies": { "ieee754": "^1.1.12", "resolve-protobuf-schema": "^2.1.0" + }, + "bin": { + "pbf": "bin/pbf" } }, - "protocol-buffers-schema": { + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true + }, + "node_modules/postcss": { + "version": "8.4.47", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.47.tgz", + "integrity": "sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.7", + "picocolors": "^1.1.0", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/protocol-buffers-schema": { "version": "3.5.1", "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.5.1.tgz", "integrity": "sha512-YVCvdhxWNDP8/nJDyXLuM+UFsuPk4+1PB7WGPVDzm3HTHbzFLxQYeW2iZpS4mmnXrQJGBzt230t/BbEb7PrQaw==" }, - "resolve-protobuf-schema": { + "node_modules/resolve-protobuf-schema": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz", "integrity": "sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ==", - "requires": { + "dependencies": { "protocol-buffers-schema": "^3.3.1" } }, - "tslib": { + "node_modules/rollup": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.24.4.tgz", + "integrity": "sha512-vGorVWIsWfX3xbcyAS+I047kFKapHYivmkaT63Smj77XwvLSJos6M1xGqZnBPFQFBRZDOcG1QnYEIxAvTr/HjA==", + "dev": true, + "dependencies": { + "@types/estree": "1.0.6" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.24.4", + "@rollup/rollup-android-arm64": "4.24.4", + "@rollup/rollup-darwin-arm64": "4.24.4", + "@rollup/rollup-darwin-x64": "4.24.4", + "@rollup/rollup-freebsd-arm64": "4.24.4", + "@rollup/rollup-freebsd-x64": "4.24.4", + "@rollup/rollup-linux-arm-gnueabihf": "4.24.4", + "@rollup/rollup-linux-arm-musleabihf": "4.24.4", + "@rollup/rollup-linux-arm64-gnu": "4.24.4", + "@rollup/rollup-linux-arm64-musl": "4.24.4", + "@rollup/rollup-linux-powerpc64le-gnu": "4.24.4", + "@rollup/rollup-linux-riscv64-gnu": "4.24.4", + "@rollup/rollup-linux-s390x-gnu": "4.24.4", + "@rollup/rollup-linux-x64-gnu": "4.24.4", + "@rollup/rollup-linux-x64-musl": "4.24.4", + "@rollup/rollup-win32-arm64-msvc": "4.24.4", + "@rollup/rollup-win32-ia32-msvc": "4.24.4", + "@rollup/rollup-win32-x64-msvc": "4.24.4", + "fsevents": "~2.3.2" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tslib": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==" + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/vite": { + "version": "5.4.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.10.tgz", + "integrity": "sha512-1hvaPshuPUtxeQ0hsVH3Mud0ZanOLwVTneA1EgbAM5LhaZEqyPWGRQ7BtaMvUrTDeEaC8pxtj6a6jku3x4z6SQ==", + "dev": true, + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } } } } diff --git a/package.json b/package.json index 1fd3b78..7cb76df 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,8 @@ "description": "", "main": "index.html", "scripts": { + "start": "vite", + "build": "vite build", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { @@ -19,5 +21,9 @@ "dependencies": { "pbf": "^3.2.1", "tslib": "^2.3.0" + }, + "devDependencies": { + "typescript": "^5.6.3", + "vite": "^5.4.10" } } diff --git a/src/bfsm.ts b/src/bfsm.ts new file mode 100644 index 0000000..09c67fb --- /dev/null +++ b/src/bfsm.ts @@ -0,0 +1,115 @@ +interface BinaryFeatureStateAttribute { + buffer: ArrayBuffer; + stride: number; + offset: number; +} + +interface BinaryFeatureStateMap { + size: number; + attributes: { + id: BinaryFeatureStateAttribute; + [name: string]: BinaryFeatureStateAttribute; + }; +} + +export function packBfsm(items: any[]): { data: BinaryFeatureStateMap, transfer: ArrayBuffer[] } { + // ID у нас uint64, поэтому его размер в байтах равен 8 + const idByteLength = 8; + + // Другие атрибуты мы храним во float32 – их размер в байтах равен 4 + const attributeByteLength = 4; + + const stride = idByteLength + attributeByteLength; + const byteLength = stride * items.length; + const buffer = new ArrayBuffer(byteLength); + + let attributeByteOffset = 0; + + const attributes: BinaryFeatureStateMap['attributes'] = { + id: { + buffer, + stride, + offset: attributeByteOffset, + }, + }; + + attributeByteOffset += idByteLength; + + attributes['hidden'] = { + buffer, + stride, + offset: attributeByteOffset, + }; + + attributeByteOffset += attributeByteLength; + + const featureStateMap: BinaryFeatureStateMap = { + attributes, + size: items.length, + }; + + // Теперь заполняем данными наш буфер + const view = new DataView(buffer); + let byteOffset = 0; + + for (let i = 0; i < items.length; i++) { + const item = items[i]; + + const id = BigInt(+item['id']); + view.setBigUint64(byteOffset, id, true); + byteOffset += idByteLength; + + view.setFloat32(byteOffset, item['hidden'], true); + byteOffset += attributeByteLength; + } + + return { + data: featureStateMap, + transfer: [buffer], + }; +} + +export function unpackBfsm(binaryMap: BinaryFeatureStateMap): Map> { + const result = new Map>(); + + const { attributes, size } = binaryMap; + + const idAttribute = { + stride: attributes.id.stride, + view: new Uint32Array(attributes.id.buffer, attributes.id.offset), + }; + + const otherAttributes: Array<{ + name: keyof BinaryFeatureStateMap['attributes']; + stride: number; + view: Float32Array; + }> = []; + + for (const attributeName in binaryMap.attributes) { + if (attributeName !== 'id') { + const { stride, buffer, offset } = binaryMap.attributes[attributeName]; + + otherAttributes.push({ + name: attributeName, + stride, + view: new Float32Array(buffer, offset), + }); + } + } + + for (let i = 0; i < size; i++) { + // TODO https://jira.2gis.ru/browse/TILES-6351 Поддержать строковые идентификаторы, сейчас работает только с внутренними id карты + const id = + String(idAttribute.view[(i * idAttribute.stride) / 4]) + + String(idAttribute.view[(i * idAttribute.stride) / 4 + 1]); + + const featureState = {}; + for (const attribute of otherAttributes) { + const value = attribute.view[(i * attribute.stride) / 4]; + featureState[attribute.name] = !Number.isNaN(value) ? value : null; + } + result.set(id, featureState); + } + + return result; +} diff --git a/src/config.ts b/src/config.ts index 2aac14f..0552cef 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,13 +1,15 @@ -export const ITERATIONS_COUNT = 10000; -export const JSON_TEMPLATE = [ - 2, - 769, - 0.00000011, - NaN, - NaN, - 10000, - '#ffffff', - true, - '#ff0000', - null, -]; + +const test: any[] = []; +for (let i = 0; i < 10**6; i++) { + test.push({ id: Math.trunc(Math.random() * 10**9).toString(), hidden: Math.random() > 0.5 }) +} + +const ctest: { id: string[], hidden: boolean[] } = { id: [], hidden: [] }; +for (let i = 0; i < 10**6; i++) { + ctest.id[i] = Math.trunc(Math.random() * 10**9).toString(); + ctest.hidden[i] = Math.random() > 0.5; +} + +export const ITERATIONS_COUNT = 20; +export const JSON_TEMPLATE = test; +export const JSON_COLUMNS_TEMPLATE = ctest; diff --git a/src/json2pbf.ts b/src/json2pbf.ts index ca3f949..d5220a0 100644 --- a/src/json2pbf.ts +++ b/src/json2pbf.ts @@ -1,289 +1,332 @@ import Pbf from 'pbf'; -enum ValueTypes { - undefined = 0, - boolean = 1, - varint = 2, - double = 3, - string = 4, - null = 5, - object = 6, - array = 7, +const VERSION = 1; + +export enum JsonType { + Boolean = 1, + Number = 2, + String = 3, + Object = 4, + Array = 5, + Null = 6, +} + +export enum PackMethod { + Generic = 1, + Columnar = 2, + Row = 3 } -interface JsonToPBFPackContext { +interface PackCtx { pbf: Pbf; - keys: { [key: string]: number }; - keysArr: string[]; - index: number; + stringMap: { [str: string]: number }; + strings: string[]; } -interface JsonToPBFUnpackContext { +interface UnpackCtx { keys: string[]; pbf: Pbf; } -function getKeyIndex(ctx: JsonToPBFPackContext, key: string) { - let i = ctx.keys[key]; +function indexateString(ctx: PackCtx, key: string) { + let i = ctx.stringMap[key]; if (i === undefined) { - i = ctx.index; - ctx.keys[key] = i; - ctx.index++; - ctx.keysArr.push(key); + i = ctx.strings.length; + ctx.strings.push(key); + ctx.stringMap[key] = i; } return i; } -function writeTag(pbf: Pbf, key: number, type: ValueTypes) { +function writeTag(pbf: Pbf, key: number, type: JsonType) { pbf.writeVarint((key << 3) | type); } -function jsonToPBF(value: any, ctx: JsonToPBFPackContext, key = 0) { - const pbf = ctx.pbf; - switch (typeof value) { - case 'boolean': - writeTag(pbf, key, ValueTypes.boolean); - pbf.writeBoolean(value); - break; - case 'number': - if (value % 1 === 0) { - writeTag(pbf, key, ValueTypes.varint); - pbf.writeSVarint(value); - } else { - writeTag(pbf, key, ValueTypes.double); - pbf.writeDouble(value); - } - break; - case 'string': - writeTag(pbf, key, ValueTypes.string); - pbf.writeString(value); - break; - case 'object': { - if (value === null) { - writeTag(pbf, key, ValueTypes.null); - } else if (Array.isArray(value)) { - writeTag(pbf, key, ValueTypes.array); - const len = value.length; - pbf.writeVarint(len); - for (let i = 0; i < len; i++) { - jsonToPBF(value[i], ctx); - } - } else { - writeTag(pbf, key, ValueTypes.object); - const keys = Object.keys(value); - pbf.writeVarint(keys.length); - keys.forEach((key) => { - jsonToPBF(value[key], ctx, getKeyIndex(ctx, key)); - }); - } +const encoders: Record void> = { + [JsonType.Boolean]: ({ pbf }: PackCtx, value: any) => { pbf.writeBoolean(value) }, + [JsonType.Number]: ({ pbf }: PackCtx, value: any) => { pbf.writeDouble(value) }, + [JsonType.String]: ({ pbf }: PackCtx, value: any) => { pbf.writeString(value) }, + [JsonType.Object]: (ctx: PackCtx, value: any) => { + if (value === null) { + return; + } + const { pbf } = ctx; + const keys = Object.keys(value); + pbf.writeVarint(keys.length); + keys.forEach((key) => { + toPbf(value[key], ctx, indexateString(ctx, key)); + }); + }, + [JsonType.Array]: (ctx: PackCtx, value: any) => { + const { pbf } = ctx; + const len = value.length; + pbf.writeVarint(len); + for (let i = 0; i < len; i++) { + toPbf(value[i], ctx); + } + }, + [JsonType.Null]: (_ctx: PackCtx, _value: any) => {} +}; - break; +const decoders: Record any> = { + [JsonType.Boolean]: ({ pbf }: UnpackCtx) => pbf.readBoolean(), + [JsonType.Number]: ({ pbf }: UnpackCtx) => pbf.readDouble(), + [JsonType.String]: ({ pbf }: UnpackCtx) => pbf.readString(), + [JsonType.Array]: (ctx: UnpackCtx) => { + const { pbf } = ctx; + const len = pbf.readVarint(); + const arr = new Array(len); + for (let i = 0; i < len; i++) { + arr[i] = fromPbf(ctx, pbf.readVarint()); } - default: - writeTag(pbf, key, ValueTypes.undefined); - break; + return arr; + }, + [JsonType.Object]: (ctx: UnpackCtx) => { + const { pbf } = ctx; + const len = pbf.readVarint(); + const obj = {}; + for (let i = 0; i < len; i++) { + var val = pbf.readVarint(); + obj[ctx.keys[val >> 3]] = fromPbf(ctx, val & 0x7); + } + return obj; + }, + [JsonType.Null]: (_ctx: UnpackCtx) => null +} + +const typeToKey: Record = { + boolean: JsonType.Boolean, + number: JsonType.Number, + string: JsonType.String, + object: JsonType.Object, + array: JsonType.Array, + null: JsonType.Null +} + + +function toPbf(value: any, ctx: PackCtx, key = 0) { + const { pbf } = ctx; + let typeOf: string = typeof value; + if (value === null) { + typeOf = 'null'; + } + if (Array.isArray(value)) { + typeOf = 'array'; + } + const jsonType = typeToKey[typeOf]; + const encoder = encoders[jsonType]; + if (!encoder) { + throw new Error(`Type ${typeof value} is not supported`); } + writeTag(pbf, key, jsonType); + encoder(ctx, value); return pbf; } -function jsonFromPBF(ctx: JsonToPBFUnpackContext, type: ValueTypes) { + +function fromPbf(ctx: UnpackCtx, type: JsonType) { const pbf = ctx.pbf; if (pbf.pos < pbf.length) { - switch (type) { - case ValueTypes.array: { - const len = pbf.readVarint(); - const arr = new Array(len); - for (let i = 0; i < len; i++) { - arr[i] = jsonFromPBF(ctx, pbf.readVarint()); - } - return arr; + const decoder = decoders[type]; + if (!decoder) { + throw new Error(`Type ${type} is not supported`); + } + return decoder(ctx); + + // switch (type) { + // case JsonType.Array: { + // const len = pbf.readVarint(); + // const arr = new Array(len); + // for (let i = 0; i < len; i++) { + // arr[i] = fromPbf(ctx, pbf.readVarint()); + // } + // return arr; + // } + // case JsonType.Boolean: { + // return pbf.readBoolean(); + // } + // case JsonType.Null: { + // return null; + // } + // case JsonType.Number: { + // return pbf.readDouble(); + // } + // case JsonType.Object: { + // const len = pbf.readVarint(); + // const obj = {}; + // for (let i = 0; i < len; i++) { + // var val = pbf.readVarint(); + // obj[ctx.keys[val >> 3]] = fromPbf(ctx, val & 0x7); + // } + // return obj; + // } + // case JsonType.String: { + // return pbf.readString(); + // } + // default: + // return undefined; + // } + } +} +export interface PackOptions { + pbf?: Pbf, + method?: PackMethod, + columns?: Record +} +function writeColumns(columns: Record, pbf: Pbf) { + const length = Object.keys(columns).length; + pbf.writeVarint(length); + for (const k in columns) { + pbf.writeString(k); + pbf.writeVarint(columns[k]); + } +} + +function readColumns(pbf: Pbf) { + const length = pbf.readVarint(); + const columns: Array<{ key: string, type: JsonType }> = []; + for (let i = 0; i < length; i++) { + const key = pbf.readString(); + const type = pbf.readVarint(); + columns.push({ key, type }); + } + return columns; +} + +export function pack(val: any, options?: PackOptions): ArrayBuffer { + const pbf = options?.pbf ?? new Pbf(); + const method = options?.method ?? PackMethod.Generic; + const columns = options?.columns; + + pbf.writeFixed32(VERSION << 24 | method << 16); + + const strings = []; + const stringMap = {}; + const ctx: PackCtx = { pbf, strings, stringMap }; + + switch (method) { + case PackMethod.Generic: + const keysPos = pbf.pos; + pbf.writeFixed32(0); + + toPbf(val, ctx); + + const keysOffset = pbf.pos; + pbf.pos = keysPos; + pbf.writeFixed32(keysOffset); + pbf.pos = keysOffset; + + pbf.writeVarint(strings.length); + for (const s of strings) { + pbf.writeString(s); } - case ValueTypes.boolean: { - return pbf.readBoolean(); + break; + case PackMethod.Columnar: + if (!columns) { + throw new Error('No columns'); } - case ValueTypes.null: { - return null; + const fcol = Object.keys(val)[0]; + const clen = val?.[fcol]?.length; + if (!clen) { + throw new Error('Cannot determine columnar data length'); } - case ValueTypes.double: { - return pbf.readDouble(); + pbf.writeFixed32(clen); + writeColumns(columns, pbf); + for (const col in columns) { + const jsonType = columns[col]; + const encoder = encoders[jsonType]; + const data = val[col]; + for (const item of data) { + encoder(ctx, item); + } } - case ValueTypes.varint: { - return pbf.readSVarint(true); + break; + case PackMethod.Row: + if (!columns) { + throw new Error('No columns'); } - case ValueTypes.object: { - const len = pbf.readVarint(); - const obj = {}; - for (let i = 0; i < len; i++) { - var val = pbf.readVarint(); - obj[ctx.keys[val >> 3]] = jsonFromPBF(ctx, val & 0x7); - } - return obj; + const rlen = val?.length; + if (!rlen) { + throw new Error('Cannot determine columnar data length'); } - case ValueTypes.string: { - return pbf.readString(); + pbf.writeFixed32(rlen); + writeColumns(columns, pbf); + for (const col in columns) { + const jsonType = columns[col]; + const encoder = encoders[jsonType]; + for (const item of val) { + encoder(ctx, item[col]); + } } - default: - return undefined; - } - } -} - -export function packJson(val: any, pbf?: Pbf): ArrayBuffer { - if (!pbf) { - pbf = new Pbf(); - } - pbf.pos = 0; - pbf.writeFixed32(0); - const keysArr = []; - const ctx: JsonToPBFPackContext = { pbf, keys: {}, keysArr, index: 0 }; - jsonToPBF(val, ctx); - const keysOffset = pbf.pos; - pbf.pos = 0; - pbf.writeFixed32(keysOffset); - pbf.pos = keysOffset; - - pbf.writeVarint(keysArr.length); - keysArr.forEach((key) => { - pbf.writeString(key); - }); - const pos = pbf.pos; - pbf.pos = 0; - return pbf.buf.slice(0, pos).buffer; + break; + default: + throw new Error(`Method ${method} is not supported`); + } + return pbf.buf.slice(0, pbf.pos).buffer; } -export function unpackJson(arr: ArrayBuffer) { - if (arr.byteLength < 4) { - return; +export function unpack(arr: ArrayBuffer) { + if (!arr || !arr.byteLength || arr.byteLength < 4) { + throw new Error('Bad array or insufficient array length.') } const pbf = new Pbf(arr); - const keysOffset = pbf.readFixed32(); - if (keysOffset >= arr.byteLength) { - return; - } - pbf.pos = keysOffset; - const keysCount = pbf.readVarint(); - const keys = new Array(keysCount); - for (let i = 0; i < keysCount; i++) { - keys[i] = pbf.readString(); + const header = pbf.readFixed32(); + const version = (header >> 24) & 0xff; + if (version > VERSION) { + throw new Error(`Version ${version} is not supported`); + } + const method = (header >> 16) & 0xff; + switch (method) { + case PackMethod.Generic: + const keysOffset = pbf.readFixed32(); + if (keysOffset >= arr.byteLength) { + return; + } + + const dataPos = pbf.pos; + pbf.pos = keysOffset; + const keysCount = pbf.readVarint(); + const keys = new Array(keysCount); + for (let i = 0; i < keysCount; i++) { + keys[i] = pbf.readString(); + } + + pbf.pos = dataPos; + const startType = pbf.readVarint(); + const ctx: UnpackCtx = { keys, pbf }; + return fromPbf(ctx, startType); + case PackMethod.Columnar: { + const ctx: UnpackCtx = { keys: [], pbf }; + const len = pbf.readFixed32(); + const columns = readColumns(pbf); + const result = {}; + for (const { key, type } of columns) { + const decoder = decoders[type]; + const data: any[] = []; + for (let i = 0; i < len; i++) { + data.push(decoder(ctx)); + } + result[key] = data; + } + return result; + } + case PackMethod.Row: { + const ctx: UnpackCtx = { keys: [], pbf }; + const len = pbf.readFixed32(); + const columns = readColumns(pbf); + const result = new Array(len).fill(0).map(() => ({})); + for (const { key, type } of columns) { + const decoder = decoders[type]; + for (let i = 0; i < len; i++) { + result[i][key] = decoder(ctx); + } + } + return result; + } + default: + throw new Error(`Method ${method} is not supported`); + } - pbf.pos = 4; - const startType = pbf.readVarint(); - return jsonFromPBF({ keys, pbf }, startType); } - -// const TIMES = 10000; -// const jsonTemplate = [2, 769, 0.00000011, NaN, NaN, 10000, '#ffffff', true, '#ff0000', null]; - -// function makeTest(title: string, packFn: (json: any) => T, unpackFn: (buf: T) => any) { -// let json: any = jsonTemplate; -// let buf: T; -// let i: number = 0; -// title = `${title} (${TIMES} запусков)`; -// console.group(title); -// console.time('pack'); -// i = TIMES - 1; -// buf = packFn(json); -// while (i--) { -// buf = packFn(json); -// } -// console.timeEnd('pack'); -// console.log(buf); - -// console.time('unpack'); -// i = TIMES; -// while (i--) { -// json = unpackFn(buf); -// } -// console.timeEnd('unpack'); -// console.log(json); -// console.groupEnd(); -// } - -// window.setTimeout(function () { -// console.group('TEMPLATE'); -// console.log(jsonTemplate); -// console.groupEnd(); -// const encoder = new TextEncoder(); -// const decoder = new TextDecoder('utf-8'); -// makeTest( -// 'JSON.stringify + TextEncoder', -// (json: any) => { -// return encoder.encode(JSON.stringify(json)).buffer; -// }, -// (buf: ArrayBuffer) => { -// const arr = JSON.parse(decoder.decode(buf)); -// arr.forEach((v, i) => { -// if (v === null) { -// arr[i] = NaN; -// } -// }); -// return arr; -// }, -// ); - -// const pbf = new Pbf(); - -// makeTest( -// 'PBF', -// (json: any) => { -// return packJson(json, pbf); -// }, -// (buf: ArrayBuffer) => { -// return unpackJson(buf); -// }, -// ); - -// makeTest( -// 'array', -// (json: any[]) => { -// return json.slice(); -// }, -// (buf: any[]) => { -// return buf.slice(); -// }, -// ); -// }, 500); - -// function compact(json) { -// let last = [0, 0]; -// function packPoint(pnt) { -// for (let i = 0; i < 2; i++) { -// const c = Math.round(pnt[i] * 1e6); -// pnt[i] = c - last[i]; -// last[i] = c; -// } -// } - -// json.features.forEach((f) => { -// if (f.geometry.type === 'Polygon') { -// f.geometry.coordinates.forEach((ring) => ring.forEach(packPoint)); -// } else if (f.geometry.type === 'MultiPolygon') { -// f.geometry.coordinates.forEach((part) => -// part.forEach((ring) => ring.forEach(packPoint)), -// ); -// } -// }); -// } - -// function uncompact(json) { -// let last = [0, 0]; -// function packPoint(pnt) { -// for (let i = 0; i < 2; i++) { -// const c = pnt[i] / 1e6; -// pnt[i] = c + last[i]; -// } -// last = pnt; -// } - -// json.features.forEach((f) => { -// if (f.geometry.type === 'Polygon') { -// f.geometry.coordinates.forEach((ring) => ring.forEach(packPoint)); -// } else if (f.geometry.type === 'MultiPolygon') { -// f.geometry.coordinates.forEach((part) => -// part.forEach((ring) => ring.forEach(packPoint)), -// ); -// } -// }); -// } diff --git a/src/worker.ts b/src/worker.ts index 1d46421..8108e00 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -1,5 +1,6 @@ -import { ITERATIONS_COUNT, JSON_TEMPLATE } from './config'; -import { packJson } from './json2pbf'; +import { ITERATIONS_COUNT, JSON_COLUMNS_TEMPLATE, JSON_TEMPLATE } from './config'; +import { JsonType, pack, PackMethod } from './json2pbf'; +import { packBfsm } from './bfsm'; const oldMsgTemplate = new Float64Array(JSON_TEMPLATE as any); @@ -30,7 +31,22 @@ onmessage = function ({ data }) { break; case 4: for (let i = 0; i < ITERATIONS_COUNT; i++) { - results[i] = { data: packJson(JSON_TEMPLATE) }; + results[i] = { data: pack(JSON_TEMPLATE) }; + } + break; + case 5: + for (let i = 0; i < ITERATIONS_COUNT; i++) { + results[i] = { data: pack(JSON_TEMPLATE, { method: PackMethod.Row, columns: { id: JsonType.String, hidden: JsonType.Boolean } }) }; + } + break; + case 6: + for (let i = 0; i < ITERATIONS_COUNT; i++) { + results[i] = { data: pack(JSON_COLUMNS_TEMPLATE, { method: PackMethod.Columnar, columns: { id: JsonType.String, hidden: JsonType.Boolean } }) }; + } + break; + case 7: + for (let i = 0; i < ITERATIONS_COUNT; i++) { + results[i] = packBfsm(JSON_TEMPLATE); } break; default: @@ -38,10 +54,24 @@ onmessage = function ({ data }) { } console.timeEnd('pack'); + if (results[0].data instanceof ArrayBuffer) { + console.log(`Binary size: ${( results[0].data.byteLength / 10**6).toFixed(3)} MB`); + } else if ('transfer' in results[0]) { + console.log(`Binary size: ${( results[0].transfer[0].byteLength / 10**6).toFixed(3)} MB`); + } + console.time('send'); for (let i = 0; i < ITERATIONS_COUNT; i++) { - //@ts-ignore - postMessage(results[i]); + if (data instanceof ArrayBuffer) { + this.postMessage(results[i], { transfer: [data] }); + } else if (data instanceof Float64Array) { + this.postMessage(results[i], { transfer: [data.buffer] }); + } else if ('transfers' in results[i]) { + const { data, transfer } = results[i]; + this.postMessage({ data }, { transfer }); + } else { + this.postMessage(results[i]); + } } console.timeEnd('send'); }; diff --git a/tsconfig.json b/tsconfig.json index 8d77834..b7305f0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,6 +2,7 @@ "extends": "./tsconfig.build.json", "compilerOptions": { "module": "esnext", + "lib": ["es2020", "dom"], "baseUrl": ".", "paths": { "@webmaps/jakarta": ["packages/jakarta/src"],