Skip to content

Commit

Permalink
fix: apply token adapter to template tokens (#27)
Browse files Browse the repository at this point in the history
* fix: apply token adapter to template tokens

* update
  • Loading branch information
yeonjuan authored Dec 1, 2024
1 parent 54964a6 commit 88fc019
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 8 deletions.
1 change: 1 addition & 0 deletions src/tokenizer/tokenize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ export function tokenize(
templateRanges: templateRanges || [],
decisionBuffer: new CharsBuffer(),
accumulatedContent: new CharsBuffer(),
tokenAdapter,
sourceCode: new SourceCode(source, templateRanges || []),
tokens: {
push(token: AnyToken) {
Expand Down
3 changes: 2 additions & 1 deletion src/types/tokenizer-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { CharsBuffer } from "../tokenizer/chars-buffer";
import { SourceCode } from "../tokenizer/source-code";
import { Range } from "./range";
import { AnyToken } from "./token";
import { TokenAdapter } from "./token-adapter";

type ContextParams = {
[TokenizerContextTypes.AttributeValueWrapped]?: {
Expand All @@ -26,8 +27,8 @@ export type TokenizerState = {
contextParams: ContextParams;
decisionBuffer: CharsBuffer;
accumulatedContent: CharsBuffer;
// pointer: CharPointer;
templateRanges: Range[];
tokenAdapter: TokenAdapter;
sourceCode: SourceCode;
tokens: {
push(token: AnyToken): void;
Expand Down
21 changes: 14 additions & 7 deletions src/utils/create-templates.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,19 @@ export function createTemplates<T extends TokenTypes>(
type: T
): TemplatableToken<T>[] {
return state.mode === "template" && state.accumulatedContent.hasTemplate()
? state.accumulatedContent.getTemplates().map((chars) => ({
type: type,
range: chars.range,
loc: state.sourceCode.getLocationOf(chars.range),
isTemplate: chars.isTemplate,
value: chars.value,
}))
? state.accumulatedContent.getTemplates().map((chars) => {
const token = {
type: type,
range: chars.range,
loc: state.sourceCode.getLocationOf(chars.range),
isTemplate: chars.isTemplate,
value: chars.value,
};
return {
...token,
range: state.tokenAdapter.finalizeRange(token),
loc: state.tokenAdapter.finalizeLocation(token),
};
})
: [];
}

0 comments on commit 88fc019

Please sign in to comment.