-
Notifications
You must be signed in to change notification settings - Fork 38
/
package.json
130 lines (130 loc) · 4.67 KB
/
package.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
{
"name": "gpt-tokenizer",
"version": "0.0.0",
"description": "A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models",
"keywords": [
"BPE",
"encoder",
"decoder",
"tokenizer",
"GPT",
"GPT-2",
"GPT-3",
"GPT-3.5",
"GPT-4",
"GPT-4o",
"NLP",
"Natural Language Processing",
"Text Generation",
"OpenAI",
"Machine Learning",
"ml"
],
"homepage": "https://github.com/niieani/gpt-tokenizer#readme",
"bugs": {
"url": "https://github.com/niieani/gpt-tokenizer/issues"
},
"repository": {
"type": "git",
"url": "https://github.com/niieani/gpt-tokenizer.git"
},
"license": "MIT",
"author": "Bazyli Brzoska <[email protected]> (https://github.com/niieani)",
"contributors": [
{
"name": "Bazyli Brzoska",
"email": "[email protected]",
"url": "https://github.com/niieani"
}
],
"exports": {
".": {
"import": "./esm/main.js",
"require": "./cjs/main.js"
},
"./*": {
"import": "./esm/*.js",
"require": "./cjs/*.js"
},
"./cjs": {
"require": "./cjs/main.js"
},
"./cjs/*": {
"require": "./cjs/*.js"
},
"./esm/*": {
"import": "./esm/*.js"
},
"./data/*": {
"import": "./data/*",
"require": "./data/*"
},
"./package.json": "./package.json"
},
"main": "esm/main.js",
"unpkg": "dist/cl100k_base.js",
"module": "esm/main.js",
"source": "src/main.ts",
"files": [
"src",
"cjs",
"esm",
"data",
"dist"
],
"scripts": {
"codegen:models": "rm -rf src/model && yarn tsx src/codegen/generateByModel.ts",
"codegen:bpe": "rm -rf src/bpeRanks && yarn tsx src/codegen/generateJsBpe.ts",
"build": "yarn build:cjs && yarn build:esm && yarn build:umd",
"build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json",
"build:esm": "mkdir -p esm && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json && yarn rrun tsc --outDir esm --target es2022",
"build:umd": "yarn build:umd:cl100k_base && yarn build:umd:p50k_base && yarn build:umd:p50k_edit && yarn build:umd:r50k_base && yarn build:umd:o200k_base",
"build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'filename=cl100k_base.js'",
"build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'filename=p50k_base.js'",
"build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_edit' --env 'filename=p50k_edit.js'",
"build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_r50k_base' --env 'filename=r50k_base.js'",
"build:umd:o200k_base": "beemo webpack --entry='./src/encoding/o200k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_o200k_base' --env 'filename=o200k_base.js'",
"clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs",
"format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
"postinstallDev": "yarn prepare",
"prepare": "rrun husky install .config/husky && beemo create-config",
"release": "beemo run-script release",
"test": "yarn test:format && yarn test:types && yarn test:lint && yarn test:code",
"test:code": "rrun jest",
"test:format": "yarn rrun prettier --check \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
"test:lint": "rrun eslint 'src/*.{js,jsx,ts,tsx}'",
"test:types": "yarn rrun tsc --noEmit"
},
"release": {
"branches": [
"+([0-9])?(.{+([0-9]),x}).x",
"master",
{
"name": "main",
"channel": false
},
"next",
{
"name": "beta",
"prerelease": true
},
{
"name": "alpha",
"prerelease": true
}
],
"tagFormat": "${version}"
},
"devDependencies": {
"@niieani/scaffold": "^1.7.37",
"tsx": "^4.19.2",
"typescript": "^5.6.3"
},
"resolutions": {
"typescript": "5.6.2"
},
"packageManager": "[email protected]",
"publishConfig": {
"access": "public"
}
}