Skip to content

Commit

Permalink
fix: javascript from keyword tokenize
Browse files Browse the repository at this point in the history
  • Loading branch information
blurfx committed Nov 20, 2023
1 parent da7ee9a commit dfd08fb
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 5 deletions.
2 changes: 1 addition & 1 deletion packages/core/src/rules/javascript.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ export const javascriptBaseRule: ParseRule[] = [
{
kind: 'keyword',
pattern:
/\b(await|break|case|catch|class|const|continue|debugger|default|delete|do|else|export|extends|finally|for|function|if|import|in|instanceof|let|new|null|return|static|super|switch|this|throw|try|typeof|var|void|while|with|yield|of)\b/g,
/\b(await|break|case|catch|class|const|continue|debugger|default|delete|do|else|export|extends|finally|for|function|if|import|from(?=\s*(?:['"]|$))|in|instanceof|let|new|null|return|static|super|switch|this|throw|try|typeof|var|void|while|with|yield|of)\b/g,
},
{
kind: 'string',
Expand Down
32 changes: 32 additions & 0 deletions packages/core/tests/tokenizer/javascript.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { describe, it, expect } from 'vitest';
import { tokenize } from '../../src/tokenizer';
import javascriptRules from '../../src/rules/javascript';
import { detectLanguage } from '../../src';

describe('javascript tokenizer', () => {
it('can tokenize comment', () => {
Expand Down Expand Up @@ -199,4 +200,35 @@ describe('javascript tokenizer', () => {
tokens.some((token) => token.value === '1234' && token.kind === 'number'),
).toBe(true);
});
it('import statement', () => {
let code = "import foo from 'bar';";
let tokens = tokenize(code, javascriptRules);
['import', 'from'].forEach((keyword) => {
expect(
tokens.find(
(token) => token.value === keyword && token.kind === 'keyword',
),
).not.toBeFalsy();
});

code = "import * as foo from 'bar';";
tokens = tokenize(code, javascriptRules);
['import', 'from'].forEach((keyword) => {
expect(
tokens.find(
(token) => token.value === keyword && token.kind === 'keyword',
),
).not.toBeFalsy();
});

code = "import { foo } from 'bar';";
tokens = tokenize(code, javascriptRules);
['import', 'from'].forEach((keyword) => {
expect(
tokens.find(
(token) => token.value === keyword && token.kind === 'keyword',
),
).not.toBeFalsy();
});
});
});
4 changes: 2 additions & 2 deletions packages/preview/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"dependencies": {
"react": "^18.2.0",
"react-dom": "^18.2.0",
"@calor/core": "^0.0.2",
"@calor/highlighter": "^0.0.2"
"@calor/core": "workspace:*",
"@calor/highlighter": "workspace:*"
},
"devDependencies": {
"@types/react": "^18.2.15",
Expand Down
4 changes: 2 additions & 2 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit dfd08fb

Please sign in to comment.