diff --git a/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json b/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json new file mode 100644 index 0000000000..b1f017e2fc --- /dev/null +++ b/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@typespec/compiler", + "comment": "**New language feature** **BREAKING** Added string template literal in typespec. Single and multi-line strings can be interpolated with `${` and `}`. Example `\\`Doc for url ${url} is here: ${location}\\``", + "type": "none" + } + ], + "packageName": "@typespec/compiler" +} \ No newline at end of file diff --git a/common/changes/@typespec/json-schema/feature-string-template_2023-11-14-00-00.json b/common/changes/@typespec/json-schema/feature-string-template_2023-11-14-00-00.json new file mode 100644 index 0000000000..8943b687d9 --- /dev/null +++ b/common/changes/@typespec/json-schema/feature-string-template_2023-11-14-00-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@typespec/json-schema", + "comment": "Added support for string template literals", + "type": "none" + } + ], + "packageName": "@typespec/json-schema" +} \ No newline at end of file diff --git a/common/changes/@typespec/openapi3/feature-string-template_2023-11-16-22-33.json b/common/changes/@typespec/openapi3/feature-string-template_2023-11-16-22-33.json new file mode 100644 index 0000000000..efcf1e2d84 --- /dev/null +++ b/common/changes/@typespec/openapi3/feature-string-template_2023-11-16-22-33.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@typespec/openapi3", + "comment": "Added support for string template literals", + "type": "none" + } + ], + "packageName": "@typespec/openapi3" +} \ No newline at end of file diff --git a/docs/extending-typespec/create-decorators.md b/docs/extending-typespec/create-decorators.md index ac0727e597..a8bb3da727 100644 --- a/docs/extending-typespec/create-decorators.md +++ b/docs/extending-typespec/create-decorators.md @@ -124,6 +124,8 @@ For certain TypeSpec types(Literal types) the decorator do not receive the actua for all the other types they are not transformed. +Example: + ```ts export function $tag( context: DecoratorContext, @@ -133,6 +135,37 @@ export function $tag( ) {} ``` +#### String templates and marshalling + +If a decorator parameter type is `valueof string`, a string template passed to it will also be marshalled as a string. +The TypeSpec type system will already validate the string template can be serialized as a string. + +```tsp +extern dec doc(target: unknown, name: valueof string); + + +alias world = "world!"; +@doc("Hello ${world} ") // receive: "Hello world!" +@doc("Hello ${123} ") // receive: "Hello 123" +@doc("Hello ${true} ") // receive: "Hello true" + +model Bar {} +@doc("Hello ${Bar} ") // not called error + ^ String template cannot be serialized as a string. + +``` + +#### Typescript type Reference + +| TypeSpec Parameter Type | TypeScript types | +| ---------------------------- | -------------------------------------------- | +| `valueof string` | `string` | +| `valueof numeric` | `number` | +| `valueof boolean` | `boolean` | +| `string` | `StringLiteral \| TemplateLiteral \| Scalar` | +| `Reflection.StringLiteral` | `StringLiteral` | +| `Reflection.TemplateLiteral` | `TemplateLiteral` | + ### Adding metadata with decorators Decorators can be used to register some metadata. For this you can use the `context.program.stateMap` or `context.program.stateSet` to insert data that will be tied to the current execution. diff --git a/docs/language-basics/type-literals.md b/docs/language-basics/type-literals.md index cf6ee9af22..0a5706ccd3 100644 --- a/docs/language-basics/type-literals.md +++ b/docs/language-basics/type-literals.md @@ -62,6 +62,22 @@ two } ``` +## String template literal + +Single or multi line string literal can be interpolated using `${}` + +```typespec +alias hello = "bonjour"; +alias Single = "${hello} world!"; + +alias Multi = """ + ${hello} + world! + """; +``` + +Any valid expression can be used in the interpolation but only other literals will result in the template literal being assignable to a `valueof string`. Any other value will be dependent on the decorator/emitter receiving it to handle. + ## Numeric literal Numeric literals can be declared by using the raw number diff --git a/packages/compiler/lib/reflection.tsp b/packages/compiler/lib/reflection.tsp index 38876f7798..f1142b2e73 100644 --- a/packages/compiler/lib/reflection.tsp +++ b/packages/compiler/lib/reflection.tsp @@ -10,3 +10,4 @@ model Operation {} model Scalar {} model Union {} model UnionVariant {} +model StringTemplate {} diff --git a/packages/compiler/src/core/checker.ts b/packages/compiler/src/core/checker.ts index a33a73471e..0aad7216e1 100644 --- a/packages/compiler/src/core/checker.ts +++ b/packages/compiler/src/core/checker.ts @@ -3,7 +3,13 @@ import { createSymbol, createSymbolTable } from "./binder.js"; import { getDeprecationDetails, markDeprecated } from "./deprecation.js"; import { ProjectionError, compilerAssert, reportDeprecated } from "./diagnostics.js"; import { validateInheritanceDiscriminatedUnions } from "./helpers/discriminator-utils.js"; -import { TypeNameOptions, getNamespaceFullName, getTypeName } from "./helpers/index.js"; +import { + TypeNameOptions, + getNamespaceFullName, + getTypeName, + stringTemplateToString, +} from "./helpers/index.js"; +import { isStringTemplateSerializable } from "./helpers/string-template-utils.js"; import { createDiagnostic } from "./messages.js"; import { getIdentifierContext, hasParseError, visitChildren } from "./parser.js"; import { Program, ProjectedProgram } from "./program.js"; @@ -102,6 +108,14 @@ import { StdTypes, StringLiteral, StringLiteralNode, + StringTemplate, + StringTemplateExpressionNode, + StringTemplateHeadNode, + StringTemplateMiddleNode, + StringTemplateSpan, + StringTemplateSpanLiteral, + StringTemplateSpanValue, + StringTemplateTailNode, Sym, SymbolFlags, SymbolLinks, @@ -641,6 +655,8 @@ export function createChecker(program: Program): Checker { return checkTupleExpression(node, mapper); case SyntaxKind.StringLiteral: return checkStringLiteral(node); + case SyntaxKind.StringTemplateExpression: + return checkStringTemplateExpresion(node, mapper); case SyntaxKind.ArrayExpression: return checkArrayExpression(node, mapper); case SyntaxKind.UnionExpression: @@ -2382,6 +2398,48 @@ export function createChecker(program: Program): Checker { return getMergedSymbol(aliasType.node!.symbol) ?? aliasSymbol; } } + + function checkStringTemplateExpresion( + node: StringTemplateExpressionNode, + mapper: TypeMapper | undefined + ): StringTemplate { + const spans: StringTemplateSpan[] = [createTemplateSpanLiteral(node.head)]; + for (const span of node.spans) { + spans.push(createTemplateSpanValue(span.expression, mapper)); + spans.push(createTemplateSpanLiteral(span.literal)); + } + const type = createType({ + kind: "StringTemplate", + node, + spans, + }); + + return type; + } + + function createTemplateSpanLiteral( + node: StringTemplateHeadNode | StringTemplateMiddleNode | StringTemplateTailNode + ): StringTemplateSpanLiteral { + return createType({ + kind: "StringTemplateSpan", + node: node, + isInterpolated: false, + type: getLiteralType(node), + }); + } + + function createTemplateSpanValue( + node: Expression, + mapper: TypeMapper | undefined + ): StringTemplateSpanValue { + return createType({ + kind: "StringTemplateSpan", + node: node, + isInterpolated: true, + type: getTypeForNode(node, mapper), + }); + } + function checkStringLiteral(str: StringLiteralNode): StringLiteral { return getLiteralType(str); } @@ -3243,6 +3301,10 @@ export function createChecker(program: Program): Checker { if (type === nullType) { return true; } + if (type.kind === "StringTemplate") { + const [valid] = isStringTemplateSerializable(type); + return valid; + } const valueTypes = new Set(["String", "Number", "Boolean", "EnumMember", "Tuple"]); return valueTypes.has(type.kind); } @@ -3424,6 +3486,8 @@ export function createChecker(program: Program): Checker { if (valueOf) { if (value.kind === "Boolean" || value.kind === "String" || value.kind === "Number") { return literalTypeToValue(value); + } else if (value.kind === "StringTemplate") { + return stringTemplateToString(value)[0]; } } return value; @@ -4058,7 +4122,13 @@ export function createChecker(program: Program): Checker { return finishTypeForProgramAndChecker(program, typePrototype, typeDef); } - function getLiteralType(node: StringLiteralNode): StringLiteral; + function getLiteralType( + node: + | StringLiteralNode + | StringTemplateHeadNode + | StringTemplateMiddleNode + | StringTemplateTailNode + ): StringLiteral; function getLiteralType(node: NumericLiteralNode): NumericLiteral; function getLiteralType(node: BooleanLiteralNode): BooleanLiteral; function getLiteralType(node: LiteralNode): LiteralType; @@ -4870,16 +4940,23 @@ export function createChecker(program: Program): Checker { } as const); } - function createLiteralType(value: string, node?: StringLiteralNode): StringLiteral; + function createLiteralType( + value: string, + node?: + | StringLiteralNode + | StringTemplateHeadNode + | StringTemplateMiddleNode + | StringTemplateTailNode + ): StringLiteral; function createLiteralType(value: number, node?: NumericLiteralNode): NumericLiteral; function createLiteralType(value: boolean, node?: BooleanLiteralNode): BooleanLiteral; function createLiteralType( value: string | number | boolean, - node?: StringLiteralNode | NumericLiteralNode | BooleanLiteralNode + node?: LiteralNode ): StringLiteral | NumericLiteral | BooleanLiteral; function createLiteralType( value: string | number | boolean, - node?: StringLiteralNode | NumericLiteralNode | BooleanLiteralNode + node?: LiteralNode ): StringLiteral | NumericLiteral | BooleanLiteral { if (program.literalTypes.has(value)) { return program.literalTypes.get(value)!; @@ -5267,6 +5344,7 @@ export function createChecker(program: Program): Checker { case "Number": return isNumericLiteralRelatedTo(source, target); case "String": + case "StringTemplate": return areScalarsRelated(target, getStdType("string")); case "Boolean": return areScalarsRelated(target, getStdType("boolean")); @@ -6043,6 +6121,8 @@ function marshalArgumentsForJS(args: T[]): MarshalledValue[] return args.map((arg) => { if (arg.kind === "Boolean" || arg.kind === "String" || arg.kind === "Number") { return literalTypeToValue(arg); + } else if (arg.kind === "StringTemplate") { + return stringTemplateToString(arg)[0]; } return arg as any; }); diff --git a/packages/compiler/src/core/helpers/index.ts b/packages/compiler/src/core/helpers/index.ts index 9745264f05..f0df08a09a 100644 --- a/packages/compiler/src/core/helpers/index.ts +++ b/packages/compiler/src/core/helpers/index.ts @@ -3,5 +3,6 @@ export { getLocationContext } from "./location-context.js"; export * from "./operation-utils.js"; export * from "./path-interpolation.js"; export * from "./projected-names-utils.js"; +export { stringTemplateToString } from "./string-template-utils.js"; export * from "./type-name-utils.js"; export * from "./usage-resolver.js"; diff --git a/packages/compiler/src/core/helpers/string-template-utils.ts b/packages/compiler/src/core/helpers/string-template-utils.ts new file mode 100644 index 0000000000..e466281a23 --- /dev/null +++ b/packages/compiler/src/core/helpers/string-template-utils.ts @@ -0,0 +1,74 @@ +import { createDiagnosticCollector } from "../diagnostics.js"; +import { createDiagnostic } from "../messages.js"; +import { Diagnostic, StringTemplate } from "../types.js"; +import { getTypeName } from "./type-name-utils.js"; + +/** + * Convert a string template to a string value. + * Only literal interpolated can be converted to string. + * Otherwise diagnostics will be reported. + * + * @param stringTemplate String template to convert. + */ +export function stringTemplateToString( + stringTemplate: StringTemplate +): [string, readonly Diagnostic[]] { + const diagnostics = createDiagnosticCollector(); + const result = stringTemplate.spans + .map((x) => { + if (x.isInterpolated) { + switch (x.type.kind) { + case "String": + case "Number": + case "Boolean": + return String(x.type.value); + case "StringTemplate": + return diagnostics.pipe(stringTemplateToString(x.type)); + default: + diagnostics.add( + createDiagnostic({ + code: "non-literal-string-template", + target: x.node, + }) + ); + return getTypeName(x.type); + } + } else { + return x.type.value; + } + }) + .join(""); + return diagnostics.wrap(result); +} + +export function isStringTemplateSerializable( + stringTemplate: StringTemplate +): [boolean, readonly Diagnostic[]] { + const diagnostics = createDiagnosticCollector(); + for (const span of stringTemplate.spans) { + if (span.isInterpolated) { + switch (span.type.kind) { + case "String": + case "Number": + case "Boolean": + break; + case "StringTemplate": + diagnostics.pipe(isStringTemplateSerializable(span.type)); + break; + case "TemplateParameter": + if (span.type.constraint && span.type.constraint.kind === "Value") { + break; // Value types will be serializable in the template instance. + } + // eslint-disable-next-line no-fallthrough + default: + diagnostics.add( + createDiagnostic({ + code: "non-literal-string-template", + target: span.node, + }) + ); + } + } + } + return [diagnostics.diagnostics.length === 0, diagnostics.diagnostics]; +} diff --git a/packages/compiler/src/core/helpers/type-name-utils.ts b/packages/compiler/src/core/helpers/type-name-utils.ts index e87aef8efb..32ea713ed9 100644 --- a/packages/compiler/src/core/helpers/type-name-utils.ts +++ b/packages/compiler/src/core/helpers/type-name-utils.ts @@ -45,6 +45,8 @@ export function getTypeName(type: Type | ValueType, options?: TypeNameOptions): return getTypeName(type.type, options); case "Tuple": return "[" + type.values.map((x) => getTypeName(x, options)).join(", ") + "]"; + case "StringTemplate": + return "string"; case "String": case "Number": case "Boolean": diff --git a/packages/compiler/src/core/messages.ts b/packages/compiler/src/core/messages.ts index 9b37299759..1a73c3b383 100644 --- a/packages/compiler/src/core/messages.ts +++ b/packages/compiler/src/core/messages.ts @@ -586,6 +586,13 @@ const diagnostics = { "Projections are experimental - your code will need to change as this feature evolves.", }, }, + "non-literal-string-template": { + severity: "error", + messages: { + default: + "Value interpolated in this string template cannot be converted to a string. Only literal types can be automatically interpolated.", + }, + }, /** * Binder diff --git a/packages/compiler/src/core/parser.ts b/packages/compiler/src/core/parser.ts index ed700f1d62..a968da5f87 100644 --- a/packages/compiler/src/core/parser.ts +++ b/packages/compiler/src/core/parser.ts @@ -90,6 +90,11 @@ import { SourceFile, Statement, StringLiteralNode, + StringTemplateExpressionNode, + StringTemplateHeadNode, + StringTemplateMiddleNode, + StringTemplateSpanNode, + StringTemplateTailNode, Sym, SyntaxKind, TemplateParameterDeclarationNode, @@ -1346,6 +1351,8 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa return parseReferenceExpression(); case Token.StringLiteral: return parseStringLiteral(); + case Token.StringTemplateHead: + return parseStringTemplateExpression(); case Token.TrueKeyword: case Token.FalseKeyword: return parseBooleanLiteral(); @@ -1452,6 +1459,119 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa }; } + function parseStringTemplateExpression(): StringTemplateExpressionNode { + const pos = tokenPos(); + const head = parseStringTemplateHead(); + const spans = parseStringTemplateSpans(head.tokenFlags); + const last = spans[spans.length - 1]; + + if (head.tokenFlags & TokenFlags.TripleQuoted) { + const [indentationsStart, indentationEnd] = scanner.findTripleQuotedStringIndent( + last.literal.pos, + last.literal.end + ); + mutate(head).value = scanner.unindentAndUnescapeTripleQuotedString( + head.pos, + head.end, + indentationsStart, + indentationEnd, + Token.StringTemplateHead, + head.tokenFlags + ); + for (const span of spans) { + mutate(span.literal).value = scanner.unindentAndUnescapeTripleQuotedString( + span.literal.pos, + span.literal.end, + indentationsStart, + indentationEnd, + span === last ? Token.StringTemplateTail : Token.StringTemplateMiddle, + head.tokenFlags + ); + } + } + return { + kind: SyntaxKind.StringTemplateExpression, + head, + spans, + ...finishNode(pos), + }; + } + + function parseStringTemplateHead(): StringTemplateHeadNode { + const pos = tokenPos(); + const flags = tokenFlags(); + const text = flags & TokenFlags.TripleQuoted ? "" : tokenValue(); + + parseExpected(Token.StringTemplateHead); + + return { + kind: SyntaxKind.StringTemplateHead, + value: text, + tokenFlags: flags, + ...finishNode(pos), + }; + } + + function parseStringTemplateSpans(tokenFlags: TokenFlags): readonly StringTemplateSpanNode[] { + const list: StringTemplateSpanNode[] = []; + let node: StringTemplateSpanNode; + do { + node = parseTemplateTypeSpan(tokenFlags); + list.push(node); + } while (node.literal.kind === SyntaxKind.StringTemplateMiddle); + return list; + } + + function parseTemplateTypeSpan(tokenFlags: TokenFlags): StringTemplateSpanNode { + const pos = tokenPos(); + const expression = parseExpression(); + const literal = parseLiteralOfTemplateSpan(tokenFlags); + return { + kind: SyntaxKind.StringTemplateSpan, + literal, + expression, + ...finishNode(pos), + }; + } + function parseLiteralOfTemplateSpan( + headTokenFlags: TokenFlags + ): StringTemplateMiddleNode | StringTemplateTailNode { + const pos = tokenPos(); + const flags = tokenFlags(); + const text = flags & TokenFlags.TripleQuoted ? "" : tokenValue(); + + if (token() === Token.CloseBrace) { + nextStringTemplateToken(headTokenFlags); + return parseTemplateMiddleOrTemplateTail(); + } else { + parseExpected(Token.StringTemplateTail); + return { + kind: SyntaxKind.StringTemplateTail, + value: text, + tokenFlags: flags, + ...finishNode(pos), + }; + } + } + + function parseTemplateMiddleOrTemplateTail(): StringTemplateMiddleNode | StringTemplateTailNode { + const pos = tokenPos(); + const flags = tokenFlags(); + const text = flags & TokenFlags.TripleQuoted ? "" : tokenValue(); + const kind = + token() === Token.StringTemplateMiddle + ? SyntaxKind.StringTemplateMiddle + : SyntaxKind.StringTemplateTail; + + nextToken(); + return { + kind, + value: text, + tokenFlags: flags, + ...finishNode(pos), + }; + } + function parseNumericLiteral(): NumericLiteralNode { const pos = tokenPos(); const valueAsString = tokenValue(); @@ -2581,6 +2701,10 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa scanner.scanDoc(); } + function nextStringTemplateToken(tokenFlags: TokenFlags) { + scanner.reScanStringTemplate(tokenFlags); + } + function createMissingIdentifier(): IdentifierNode { const pos = tokenPos(); previousTokenEnd = pos; @@ -3168,7 +3292,15 @@ export function visitChildren(node: Node, cb: NodeCallback): T | undefined case SyntaxKind.DocUnknownTag: return visitNode(cb, node.tagName) || visitEach(cb, node.content); + case SyntaxKind.StringTemplateExpression: + return visitNode(cb, node.head) || visitEach(cb, node.spans); + case SyntaxKind.StringTemplateSpan: + return visitNode(cb, node.expression) || visitNode(cb, node.literal); + // no children for the rest of these. + case SyntaxKind.StringTemplateHead: + case SyntaxKind.StringTemplateMiddle: + case SyntaxKind.StringTemplateTail: case SyntaxKind.StringLiteral: case SyntaxKind.NumericLiteral: case SyntaxKind.BooleanLiteral: diff --git a/packages/compiler/src/core/scanner.ts b/packages/compiler/src/core/scanner.ts index ae52583a76..46354e1da1 100644 --- a/packages/compiler/src/core/scanner.ts +++ b/packages/compiler/src/core/scanner.ts @@ -30,6 +30,9 @@ export enum Token { Identifier, NumericLiteral, StringLiteral, + StringTemplateHead, + StringTemplateMiddle, + StringTemplateTail, // Add new tokens above if they don't fit any of the categories below /////////////////////////////////////////////////////////////// @@ -165,6 +168,11 @@ export type DocToken = | Token.DocCodeFenceDelimiter | Token.EndOfFile; +export type StringTemplateToken = + | Token.StringTemplateHead + | Token.StringTemplateMiddle + | Token.StringTemplateTail; + /** @internal */ export const TokenDisplay = getTokenDisplayTable([ [Token.None, "none"], @@ -175,6 +183,9 @@ export const TokenDisplay = getTokenDisplayTable([ [Token.ConflictMarker, "conflict marker"], [Token.NumericLiteral, "numeric literal"], [Token.StringLiteral, "string literal"], + [Token.StringTemplateHead, "string template head"], + [Token.StringTemplateMiddle, "string template middle"], + [Token.StringTemplateTail, "string template tail"], [Token.NewLine, "newline"], [Token.Whitespace, "whitespace"], [Token.DocCodeFenceDelimiter, "doc code fence delimiter"], @@ -298,6 +309,31 @@ export interface Scanner { /** Advance one token inside DocComment. Use inside {@link scanRange} callback over DocComment range. */ scanDoc(): DocToken; + /** + * Unconditionally back up and scan a template expression portion. + * @param tokenFlags Token Flags for head StringTemplateToken + */ + reScanStringTemplate(tokenFlags: TokenFlags): StringTemplateToken; + + /** + * Finds the indent for the given triple quoted string. + * @param start + * @param end + */ + findTripleQuotedStringIndent(start: number, end: number): [number, number]; + + /** + * Unindent and unescape the triple quoted string rawText + */ + unindentAndUnescapeTripleQuotedString( + start: number, + end: number, + indentationStart: number, + indentationEnd: number, + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags + ): string; + /** Reset the scanner to the given start and end positions, invoke the callback, and then restore scanner state. */ scanRange(range: TextRange, callback: () => T): T; @@ -384,6 +420,9 @@ export function createScanner( scan, scanRange, scanDoc, + reScanStringTemplate, + findTripleQuotedStringIndent, + unindentAndUnescapeTripleQuotedString, eof, getTokenText, getTokenValue, @@ -400,7 +439,10 @@ export function createScanner( function getTokenValue() { switch (token) { case Token.StringLiteral: - return getStringTokenValue(); + case Token.StringTemplateHead: + case Token.StringTemplateMiddle: + case Token.StringTemplateTail: + return getStringTokenValue(token, tokenFlags); case Token.Identifier: return getIdentifierTokenValue(); default: @@ -549,8 +591,8 @@ export function createScanner( case CharCode.DoubleQuote: return lookAhead(1) === CharCode.DoubleQuote && lookAhead(2) === CharCode.DoubleQuote - ? scanTripleQuotedString() - : scanString(); + ? scanString(TokenFlags.TripleQuoted) + : scanString(TokenFlags.None); case CharCode.Exclamation: return lookAhead(1) === CharCode.Equals @@ -642,6 +684,12 @@ export function createScanner( return (token = Token.EndOfFile); } + function reScanStringTemplate(lastTokenFlags: TokenFlags): StringTemplateToken { + position = tokenPosition; + tokenFlags = TokenFlags.None; + return scanStringTemplateSpan(lastTokenFlags); + } + function scanRange(range: TextRange, callback: () => T): T { const savedPosition = position; const savedEndPosition = endPosition; @@ -707,10 +755,14 @@ export function createScanner( function error< C extends keyof CompilerDiagnostics, M extends keyof CompilerDiagnostics[C] = "default", - >(report: Omit, "target">) { + >( + report: Omit, "target">, + pos?: number, + end?: number + ) { const diagnostic = createDiagnostic({ ...report, - target: { file, pos: tokenPosition, end: position }, + target: { file, pos: pos ?? tokenPosition, end: end ?? position }, } as any); diagnosticHandler(diagnostic); } @@ -820,9 +872,31 @@ export function createScanner( return unterminated(Token.DocCodeSpan); } - function scanString(): Token.StringLiteral { - position++; // consume '"' + function scanString(tokenFlags: TokenFlags): Token.StringLiteral | Token.StringTemplateHead { + if (tokenFlags & TokenFlags.TripleQuoted) { + position += 3; // consume '"""' + } else { + position++; // consume '"' + } + return scanStringLiteralLike(tokenFlags, Token.StringTemplateHead, Token.StringLiteral); + } + + function scanStringTemplateSpan( + tokenFlags: TokenFlags + ): Token.StringTemplateMiddle | Token.StringTemplateTail { + position++; // consume '{' + + return scanStringLiteralLike(tokenFlags, Token.StringTemplateMiddle, Token.StringTemplateTail); + } + + function scanStringLiteralLike( + requestedTokenFlags: TokenFlags, + template: M, + tail: T + ): M | T { + const multiLine = requestedTokenFlags & TokenFlags.TripleQuoted; + tokenFlags = requestedTokenFlags; loop: for (; !eof(); position++) { const ch = input.charCodeAt(position); switch (ch) { @@ -834,44 +908,88 @@ export function createScanner( } continue; case CharCode.DoubleQuote: - position++; - return (token = Token.StringLiteral); + if (multiLine) { + if (lookAhead(1) === CharCode.DoubleQuote && lookAhead(2) === CharCode.DoubleQuote) { + position += 3; + token = tail; + return tail; + } else { + continue; + } + } else { + position++; + token = tail; + return tail; + } + case CharCode.$: + if (lookAhead(1) === CharCode.OpenBrace) { + position += 2; + token = template; + return template; + } + continue; case CharCode.CarriageReturn: case CharCode.LineFeed: - break loop; + if (multiLine) { + continue; + } else { + break loop; + } } } - return unterminated(Token.StringLiteral); + return unterminated(tail); } - function scanTripleQuotedString(): Token.StringLiteral { - tokenFlags |= TokenFlags.TripleQuoted; - position += 3; // consume '"""' - - for (; !eof(); position++) { - if ( - input.charCodeAt(position) === CharCode.DoubleQuote && - lookAhead(1) === CharCode.DoubleQuote && - lookAhead(2) === CharCode.DoubleQuote - ) { - position += 3; - return (token = Token.StringLiteral); - } + function getStringLiteralOffsetStart( + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags + ) { + switch (token) { + case Token.StringLiteral: + case Token.StringTemplateHead: + return tokenFlags & TokenFlags.TripleQuoted ? 3 : 1; // """ or " + default: + return 1; // { } - - return unterminated(Token.StringLiteral); } - function getStringTokenValue(): string { - const quoteLength = tokenFlags & TokenFlags.TripleQuoted ? 3 : 1; - const start = tokenPosition + quoteLength; - const end = tokenFlags & TokenFlags.Unterminated ? position : position - quoteLength; + function getStringLiteralOffsetEnd( + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags + ) { + switch (token) { + case Token.StringLiteral: + case Token.StringTemplateTail: + return tokenFlags & TokenFlags.TripleQuoted ? 3 : 1; // """ or " + default: + return 2; // ${ + } + } + function getStringTokenValue( + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags + ): string { if (tokenFlags & TokenFlags.TripleQuoted) { - return unindentAndUnescapeTripleQuotedString(start, end); + const start = tokenPosition; + const end = position; + const [indentationStart, indentationEnd] = findTripleQuotedStringIndent(start, end); + return unindentAndUnescapeTripleQuotedString( + start, + end, + indentationStart, + indentationEnd, + token, + tokenFlags + ); } + const startOffset = getStringLiteralOffsetStart(token, tokenFlags); + const endOffset = getStringLiteralOffsetEnd(token, tokenFlags); + const start = tokenPosition + startOffset; + const end = tokenFlags & TokenFlags.Unterminated ? position : position - endOffset; + if (tokenFlags & TokenFlags.Escaped) { return unescapeString(start, end); } @@ -896,22 +1014,8 @@ export function createScanner( return text; } - function unindentAndUnescapeTripleQuotedString(start: number, end: number): string { - // ignore leading whitespace before required initial line break - while (start < end && isWhiteSpaceSingleLine(input.charCodeAt(start))) { - start++; - } - - // remove required initial line break - if (isLineBreak(input.charCodeAt(start))) { - if (isCrlf(start, start, end)) { - start++; - } - start++; - } else { - error({ code: "no-new-line-start-triple-quote" }); - } - + function findTripleQuotedStringIndent(start: number, end: number): [number, number] { + end = end - 3; // Remove the """ // remove whitespace before closing delimiter and record it as required // indentation for all lines const indentationEnd = end; @@ -922,7 +1026,7 @@ export function createScanner( // remove required final line break if (isLineBreak(input.charCodeAt(end - 1))) { - if (isCrlf(end - 2, start, end)) { + if (isCrlf(end - 2, 0, end)) { end--; } end--; @@ -930,13 +1034,70 @@ export function createScanner( error({ code: "no-new-line-end-triple-quote" }); } + return [indentationStart, indentationEnd]; + } + + function unindentAndUnescapeTripleQuotedString( + start: number, + end: number, + indentationStart: number, + indentationEnd: number, + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags + ): string { + const startOffset = getStringLiteralOffsetStart(token, tokenFlags); + const endOffset = getStringLiteralOffsetEnd(token, tokenFlags); + start = start + startOffset; + end = tokenFlags & TokenFlags.Unterminated ? end : end - endOffset; + + if (token === Token.StringLiteral || token === Token.StringTemplateHead) { + // ignore leading whitespace before required initial line break + while (start < end && isWhiteSpaceSingleLine(input.charCodeAt(start))) { + start++; + } + // remove required initial line break + if (isLineBreak(input.charCodeAt(start))) { + if (isCrlf(start, start, end)) { + start++; + } + start++; + } else { + error({ code: "no-new-line-start-triple-quote" }); + } + } + + if (token === Token.StringLiteral || token === Token.StringTemplateTail) { + while (end > start && isWhiteSpaceSingleLine(input.charCodeAt(end - 1))) { + end--; + } + + // remove required final line break + if (isLineBreak(input.charCodeAt(end - 1))) { + if (isCrlf(end - 2, start, end)) { + end--; + } + end--; + } else { + error({ code: "no-new-line-end-triple-quote" }); + } + } + + let skipUnindentOnce = false; + // We are resuming from the middle of a line so we want to keep text as it is from there. + if (token === Token.StringTemplateMiddle || token === Token.StringTemplateTail) { + skipUnindentOnce = true; + } // remove required matching indentation from each line and unescape in the // process of doing so let result = ""; let pos = start; while (pos < end) { - // skip indentation at start of line - start = skipMatchingIndentation(pos, end, indentationStart, indentationEnd); + if (skipUnindentOnce) { + skipUnindentOnce = false; + } else { + // skip indentation at start of line + start = skipMatchingIndentation(pos, end, indentationStart, indentationEnd); + } let ch; while (pos < end && !isLineBreak((ch = input.charCodeAt(pos)))) { @@ -946,7 +1107,7 @@ export function createScanner( } result += input.substring(start, pos); if (pos === end - 1) { - error({ code: "invalid-escape-sequence" }); + error({ code: "invalid-escape-sequence" }, pos, pos); pos++; } else { result += unescapeOne(pos); @@ -954,7 +1115,6 @@ export function createScanner( } start = pos; } - if (pos < end) { if (isCrlf(pos, start, end)) { // CRLF in multi-line string is normalized to LF in string value. @@ -969,7 +1129,6 @@ export function createScanner( start = pos; } } - result += input.substring(start, pos); return result; } @@ -1021,7 +1180,7 @@ export function createScanner( } if (pos === end - 1) { - error({ code: "invalid-escape-sequence" }); + error({ code: "invalid-escape-sequence" }, pos, pos); break; } @@ -1048,10 +1207,12 @@ export function createScanner( return '"'; case CharCode.Backslash: return "\\"; + case CharCode.$: + return "$"; case CharCode.Backtick: return "`"; default: - error({ code: "invalid-escape-sequence" }); + error({ code: "invalid-escape-sequence" }, pos, pos + 2); return String.fromCharCode(ch); } } diff --git a/packages/compiler/src/core/semantic-walker.ts b/packages/compiler/src/core/semantic-walker.ts index 23de52c00d..d6d4a5657c 100644 --- a/packages/compiler/src/core/semantic-walker.ts +++ b/packages/compiler/src/core/semantic-walker.ts @@ -11,6 +11,8 @@ import { Operation, Scalar, SemanticNodeListener, + StringTemplate, + StringTemplateSpan, TemplateParameter, Tuple, Type, @@ -320,6 +322,22 @@ function navigateTupleType(type: Tuple, context: NavigationContext) { navigateTypeInternal(value, context); } } +function navigateStringTemplate(type: StringTemplate, context: NavigationContext) { + if (checkVisited(context.visited, type)) { + return; + } + if (context.emit("stringTemplate", type) === ListenerFlow.NoRecursion) return; + for (const value of type.spans) { + navigateTypeInternal(value, context); + } +} +function navigateStringTemplateSpan(type: StringTemplateSpan, context: NavigationContext) { + if (checkVisited(context.visited, type)) { + return; + } + if (context.emit("stringTemplateSpan", type as any) === ListenerFlow.NoRecursion) return; + navigateTypeInternal(type.type, context); +} function navigateTemplateParameter(type: TemplateParameter, context: NavigationContext) { if (checkVisited(context.visited, type)) { @@ -357,6 +375,10 @@ function navigateTypeInternal(type: Type, context: NavigationContext) { return navigateUnionTypeVariant(type, context); case "Tuple": return navigateTupleType(type, context); + case "StringTemplate": + return navigateStringTemplate(type, context); + case "StringTemplateSpan": + return navigateStringTemplateSpan(type, context); case "TemplateParameter": return navigateTemplateParameter(type, context); case "Decorator": diff --git a/packages/compiler/src/core/types.ts b/packages/compiler/src/core/types.ts index 026fa47a51..0fc115dd15 100644 --- a/packages/compiler/src/core/types.ts +++ b/packages/compiler/src/core/types.ts @@ -4,6 +4,7 @@ import { AssetEmitter } from "../emitter-framework/types.js"; import { YamlScript } from "../yaml/types.js"; import { ModuleResolutionResult } from "./module-resolver.js"; import { Program } from "./program.js"; +import type { TokenFlags } from "./scanner.js"; // prettier-ignore export type MarshalledValue = @@ -98,6 +99,8 @@ export type Type = | StringLiteral | NumericLiteral | BooleanLiteral + | StringTemplate + | StringTemplateSpan | Tuple | Union | UnionVariant @@ -478,6 +481,28 @@ export interface BooleanLiteral extends BaseType { value: boolean; } +export interface StringTemplate extends BaseType { + kind: "StringTemplate"; + node: StringTemplateExpressionNode; + spans: StringTemplateSpan[]; +} + +export type StringTemplateSpan = StringTemplateSpanLiteral | StringTemplateSpanValue; + +export interface StringTemplateSpanLiteral extends BaseType { + kind: "StringTemplateSpan"; + node: StringTemplateHeadNode | StringTemplateMiddleNode | StringTemplateTailNode; + isInterpolated: false; + type: StringLiteral; +} + +export interface StringTemplateSpanValue extends BaseType { + kind: "StringTemplateSpan"; + node: Expression; + isInterpolated: true; + type: Type; +} + export interface Tuple extends BaseType { kind: "Tuple"; node: TupleExpressionNode; @@ -736,6 +761,11 @@ export enum SyntaxKind { StringLiteral, NumericLiteral, BooleanLiteral, + StringTemplateExpression, + StringTemplateHead, + StringTemplateMiddle, + StringTemplateTail, + StringTemplateSpan, ExternKeyword, VoidKeyword, NeverKeyword, @@ -858,6 +888,10 @@ export type Node = | Statement | Expression | FunctionParameterNode + | StringTemplateSpanNode + | StringTemplateHeadNode + | StringTemplateMiddleNode + | StringTemplateTailNode | Modifier | DocNode | DocContent @@ -1040,6 +1074,7 @@ export type Expression = | StringLiteralNode | NumericLiteralNode | BooleanLiteralNode + | StringTemplateExpressionNode | VoidKeywordNode | NeverKeywordNode | AnyKeywordNode; @@ -1221,7 +1256,13 @@ export interface ModelSpreadPropertyNode extends BaseNode { readonly parent?: ModelStatementNode | ModelExpressionNode; } -export type LiteralNode = StringLiteralNode | NumericLiteralNode | BooleanLiteralNode; +export type LiteralNode = + | StringLiteralNode + | NumericLiteralNode + | BooleanLiteralNode + | StringTemplateHeadNode + | StringTemplateMiddleNode + | StringTemplateTailNode; export interface StringLiteralNode extends BaseNode { readonly kind: SyntaxKind.StringLiteral; @@ -1239,6 +1280,39 @@ export interface BooleanLiteralNode extends BaseNode { readonly value: boolean; } +export interface StringTemplateExpressionNode extends BaseNode { + readonly kind: SyntaxKind.StringTemplateExpression; + readonly head: StringTemplateHeadNode; + readonly spans: readonly StringTemplateSpanNode[]; +} + +// Each of these corresponds to a substitution expression and a template literal, in that order. +// The template literal must have kind TemplateMiddleLiteral or TemplateTailLiteral. +export interface StringTemplateSpanNode extends BaseNode { + readonly kind: SyntaxKind.StringTemplateSpan; + readonly expression: Expression; + readonly literal: StringTemplateMiddleNode | StringTemplateTailNode; +} + +export interface StringTemplateLiteralLikeNode extends BaseNode { + readonly value: string; + + /** @internal */ + readonly tokenFlags: TokenFlags; +} + +export interface StringTemplateHeadNode extends StringTemplateLiteralLikeNode { + readonly kind: SyntaxKind.StringTemplateHead; +} + +export interface StringTemplateMiddleNode extends StringTemplateLiteralLikeNode { + readonly kind: SyntaxKind.StringTemplateMiddle; +} + +export interface StringTemplateTailNode extends StringTemplateLiteralLikeNode { + readonly kind: SyntaxKind.StringTemplateTail; +} + export interface ExternKeywordNode extends BaseNode { readonly kind: SyntaxKind.ExternKeyword; } diff --git a/packages/compiler/src/emitter-framework/asset-emitter.ts b/packages/compiler/src/emitter-framework/asset-emitter.ts index bee07f7a60..b86e42ebdf 100644 --- a/packages/compiler/src/emitter-framework/asset-emitter.ts +++ b/packages/compiler/src/emitter-framework/asset-emitter.ts @@ -776,6 +776,8 @@ export function createAssetEmitter( return "namespace"; case "ModelProperty": return "modelPropertyLiteral"; + case "StringTemplate": + return "stringTemplate"; case "Boolean": return "booleanLiteral"; case "String": @@ -903,6 +905,7 @@ function keyHasContext(key: keyof TypeEmitter) { const noReferenceContext = new Set([ ...noContext, "booleanLiteral", + "stringTemplate", "stringLiteral", "numericLiteral", "scalarInstantiation", diff --git a/packages/compiler/src/emitter-framework/type-emitter.ts b/packages/compiler/src/emitter-framework/type-emitter.ts index aeefe4e316..ef2bee27cf 100644 --- a/packages/compiler/src/emitter-framework/type-emitter.ts +++ b/packages/compiler/src/emitter-framework/type-emitter.ts @@ -15,6 +15,7 @@ import { Program, Scalar, StringLiteral, + StringTemplate, Tuple, Type, Union, @@ -464,6 +465,14 @@ export class TypeEmitter> { return this.emitter.result.none(); } + stringTemplateContext(string: StringTemplate): Context { + return {}; + } + + stringTemplate(stringTemplate: StringTemplate): EmitterOutput { + return this.emitter.result.none(); + } + stringLiteralContext(string: StringLiteral): Context { return {}; } diff --git a/packages/compiler/src/formatter/print/printer.ts b/packages/compiler/src/formatter/print/printer.ts index 50ede4052b..dff54328a6 100644 --- a/packages/compiler/src/formatter/print/printer.ts +++ b/packages/compiler/src/formatter/print/printer.ts @@ -60,6 +60,8 @@ import { ScalarStatementNode, Statement, StringLiteralNode, + StringTemplateExpressionNode, + StringTemplateSpanNode, SyntaxKind, TemplateParameterDeclarationNode, TextRange, @@ -357,6 +359,16 @@ export function printNode( return ""; case SyntaxKind.EmptyStatement: return ""; + case SyntaxKind.StringTemplateExpression: + return printStringTemplateExpression( + path as AstPath, + options, + print + ); + case SyntaxKind.StringTemplateSpan: + case SyntaxKind.StringTemplateHead: + case SyntaxKind.StringTemplateMiddle: + case SyntaxKind.StringTemplateTail: case SyntaxKind.JsSourceFile: case SyntaxKind.JsNamespaceDeclaration: case SyntaxKind.InvalidStatement: @@ -1672,6 +1684,22 @@ export function printReturnExpression( return ["return ", path.call(print, "value")]; } +export function printStringTemplateExpression( + path: AstPath, + options: TypeSpecPrettierOptions, + print: PrettierChildPrint +) { + const node = path.node; + const content = [ + getRawText(node.head, options), + path.map((span: AstPath) => { + const expression = span.call(print, "expression"); + return [expression, getRawText(span.node.literal, options)]; + }, "spans"), + ]; + return content; +} + function printItemList( path: AstPath, options: TypeSpecPrettierOptions, diff --git a/packages/compiler/src/server/serverlib.ts b/packages/compiler/src/server/serverlib.ts index 8b25ab671f..14930247c0 100644 --- a/packages/compiler/src/server/serverlib.ts +++ b/packages/compiler/src/server/serverlib.ts @@ -937,17 +937,12 @@ export function createServer(host: ServerHost): Server { function mapTokens() { const tokens = new Map(); const scanner = createScanner(file, () => {}); - + const templateStack: [Token, TokenFlags][] = []; while (scanner.scan() !== Token.EndOfFile) { if (scanner.tokenFlags & TokenFlags.DocComment) { classifyDocComment({ pos: scanner.tokenPosition, end: scanner.position }); } else { - const kind = classifyToken(scanner.token); - if (kind === ignore) { - continue; - } - tokens.set(scanner.tokenPosition, { - kind: kind === defer ? undefined! : kind, + handleToken(scanner.token, scanner.tokenFlags, { pos: scanner.tokenPosition, end: scanner.position, }); @@ -970,6 +965,98 @@ export function createServer(host: ServerHost): Server { } }); } + + function handleToken(token: Token, tokenFlags: TokenFlags, range: TextRange) { + switch (token) { + case Token.StringTemplateHead: + templateStack.push([token, tokenFlags]); + classifyStringTemplate(token, range); + break; + case Token.OpenBrace: + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack.length > 0) { + templateStack.push([token, tokenFlags]); + } + handleSimpleToken(token, range); + break; + case Token.CloseBrace: + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack.length > 0) { + const [lastToken, lastTokenFlags] = templateStack[templateStack.length - 1]; + + if (lastToken === Token.StringTemplateHead) { + token = scanner.reScanStringTemplate(lastTokenFlags); + + // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us. + if (token === Token.StringTemplateTail) { + templateStack.pop(); + classifyStringTemplate(token, { + pos: scanner.tokenPosition, + end: scanner.position, + }); + } else { + compilerAssert( + token === Token.StringTemplateMiddle, + "Should have been a template middle." + ); + classifyStringTemplate(token, { + pos: scanner.tokenPosition, + end: scanner.position, + }); + } + } else { + compilerAssert(lastToken === Token.OpenBrace, "Should have been an open brace"); + templateStack.pop(); + } + break; + } + handleSimpleToken(token, range); + break; + default: + handleSimpleToken(token, range); + } + } + + function handleSimpleToken(token: Token, range: TextRange) { + const kind = classifyToken(scanner.token); + if (kind === ignore) { + return; + } + tokens.set(range.pos, { + kind: kind === defer ? undefined! : kind, + ...range, + }); + } + + function classifyStringTemplate( + token: Token.StringTemplateHead | Token.StringTemplateMiddle | Token.StringTemplateTail, + range: TextRange + ) { + const stringStart = token === Token.StringTemplateHead ? range.pos : range.pos + 1; + const stringEnd = token === Token.StringTemplateTail ? range.end : range.end - 2; + + if (stringStart !== range.pos) { + tokens.set(range.pos, { + kind: SemanticTokenKind.Operator, + pos: range.pos, + end: stringStart, + }); + } + tokens.set(stringStart, { + kind: SemanticTokenKind.String, + pos: stringStart, + end: stringEnd, + }); + if (stringEnd !== range.end) { + tokens.set(stringEnd, { + kind: SemanticTokenKind.Operator, + pos: stringEnd, + end: range.end, + }); + } + } } function classifyToken(token: Token): SemanticTokenKind | typeof defer | typeof ignore { diff --git a/packages/compiler/src/server/tmlanguage.ts b/packages/compiler/src/server/tmlanguage.ts index 0f5620a7c7..d94cc38313 100644 --- a/packages/compiler/src/server/tmlanguage.ts +++ b/packages/compiler/src/server/tmlanguage.ts @@ -44,6 +44,8 @@ export type TypeSpecScope = | "punctuation.terminator.statement.tsp" | "punctuation.definition.typeparameters.begin.tsp" | "punctuation.definition.typeparameters.end.tsp" + | "punctuation.definition.template-expression.begin.tsp" + | "punctuation.definition.template-expression.end.tsp" | "punctuation.squarebracket.open.tsp" | "punctuation.squarebracket.close.tsp" | "punctuation.curlybrace.open.tsp" @@ -109,12 +111,26 @@ const escapeChar: MatchRule = { match: "\\\\.", }; +const templateExpression: BeginEndRule = { + key: "template-expression", + scope: meta, + begin: "\\$\\{", + beginCaptures: { + "0": { scope: "punctuation.definition.template-expression.begin.tsp" }, + }, + end: "\\}", + endCaptures: { + "0": { scope: "punctuation.definition.template-expression.end.tsp" }, + }, + patterns: [expression], +}; + const stringLiteral: BeginEndRule = { key: "string-literal", scope: "string.quoted.double.tsp", begin: '"', end: '"|$', - patterns: [escapeChar], + patterns: [templateExpression, escapeChar], }; const tripleQuotedStringLiteral: BeginEndRule = { @@ -122,7 +138,7 @@ const tripleQuotedStringLiteral: BeginEndRule = { scope: "string.quoted.triple.tsp", begin: '"""', end: '"""', - patterns: [escapeChar], + patterns: [templateExpression, escapeChar], }; const punctuationComma: MatchRule = { diff --git a/packages/compiler/src/server/type-signature.ts b/packages/compiler/src/server/type-signature.ts index 18a9e24007..a2625b3979 100644 --- a/packages/compiler/src/server/type-signature.ts +++ b/packages/compiler/src/server/type-signature.ts @@ -10,6 +10,7 @@ import { FunctionType, ModelProperty, Operation, + StringTemplate, Sym, SyntaxKind, Type, @@ -54,6 +55,10 @@ function getTypeSignature(type: Type | ValueType): string { return `(boolean)\n${fence(type.value ? "true" : "false")}`; case "Number": return `(number)\n${fence(type.value.toString())}`; + case "StringTemplate": + return `(string template)\n${fence(getStringTemplateSignature(type))}`; + case "StringTemplateSpan": + return `(string template span)\n${fence(getTypeName(type.type))}`; case "Intrinsic": return ""; case "FunctionParameter": @@ -105,6 +110,18 @@ function getFunctionParameterSignature(parameter: FunctionParameter) { return `${rest}${printId(parameter.name)}${optional}: ${getTypeName(parameter.type)}`; } +function getStringTemplateSignature(stringTemplate: StringTemplate) { + return ( + "`" + + [ + stringTemplate.spans.map((span) => { + return span.isInterpolated ? "${" + getTypeName(span.type) + "}" : span.type.value; + }), + ].join("") + + "`" + ); +} + function getModelPropertySignature(property: ModelProperty) { const ns = getQualifier(property.model); return `${ns}${printId(property.name)}: ${getPrintableTypeName(property.type)}`; diff --git a/packages/compiler/test/checker/decorators.test.ts b/packages/compiler/test/checker/decorators.test.ts index 147d9cc458..ce3a0c6e24 100644 --- a/packages/compiler/test/checker/decorators.test.ts +++ b/packages/compiler/test/checker/decorators.test.ts @@ -273,6 +273,60 @@ describe("compiler: checker: decorators", () => { }, ]); }); + + describe("value marshalling", () => { + async function testCallDecorator(type: string, value: string): Promise { + await runner.compile(` + extern dec testDec(target: unknown, arg1: ${type}); + + @testDec(${value}) + @test + model Foo {} + `); + return calledArgs[2]; + } + + describe("passing a string literal", () => { + it("`: valueof string` cast the value to a JS string", async () => { + const arg = await testCallDecorator("valueof string", `"one"`); + strictEqual(arg, "one"); + }); + + it("`: string` keeps the StringLiteral type", async () => { + const arg = await testCallDecorator("string", `"one"`); + strictEqual(arg.kind, "String"); + }); + }); + + describe("passing a string template", () => { + it("`: valueof string` cast the value to a JS string", async () => { + const arg = await testCallDecorator( + "valueof string", + '"Start ${"one"} middle ${"two"} end"' + ); + strictEqual(arg, "Start one middle two end"); + }); + + it("`: string` keeps the StringTemplate type", async () => { + const arg = await testCallDecorator("string", '"Start ${"one"} middle ${"two"} end"'); + strictEqual(arg.kind, "StringTemplate"); + }); + }); + + describe("passing a numeric literal", () => { + it("valueof int32 cast the value to a JS number", async () => { + const arg = await testCallDecorator("valueof int32", `123`); + strictEqual(arg, 123); + }); + }); + + describe("passing a boolean literal", () => { + it("valueof boolean cast the value to a JS boolean", async () => { + const arg = await testCallDecorator("valueof boolean", `true`); + strictEqual(arg, true); + }); + }); + }); }); it("can have the same name as types", async () => { diff --git a/packages/compiler/test/checker/relation.test.ts b/packages/compiler/test/checker/relation.test.ts index 33ba19a4fa..0c3aa4d5e6 100644 --- a/packages/compiler/test/checker/relation.test.ts +++ b/packages/compiler/test/checker/relation.test.ts @@ -209,6 +209,10 @@ describe("compiler: checker: type relations", () => { await expectTypeAssignable({ source: `"foo"`, target: "string" }); }); + it("can assign string template with primitives interpolated", async () => { + await expectTypeAssignable({ source: `"foo \${123} bar"`, target: "string" }); + }); + it("can assign string literal union", async () => { await expectTypeAssignable({ source: `"foo" | "bar"`, target: "string" }); }); diff --git a/packages/compiler/test/checker/string-template.test.ts b/packages/compiler/test/checker/string-template.test.ts new file mode 100644 index 0000000000..fb867ac2c3 --- /dev/null +++ b/packages/compiler/test/checker/string-template.test.ts @@ -0,0 +1,73 @@ +import { strictEqual } from "assert"; +import { Model, StringTemplate } from "../../src/index.js"; +import { BasicTestRunner, createTestRunner } from "../../src/testing/index.js"; + +describe("compiler: string templates", () => { + let runner: BasicTestRunner; + + beforeEach(async () => { + runner = await createTestRunner(); + }); + + async function compileStringTemplate( + templateString: string, + other?: string + ): Promise { + const { Test } = (await runner.compile( + ` + @test model Test { + test: ${templateString}; + } + + ${other ?? ""} + ` + )) as { Test: Model }; + + const prop = Test.properties.get("test")!.type; + + strictEqual(prop.kind, "StringTemplate"); + return prop; + } + + it("simple", async () => { + const template = await compileStringTemplate(`"Start \${123} end"`); + strictEqual(template.spans.length, 3); + strictEqual(template.spans[0].isInterpolated, false); + strictEqual(template.spans[0].type.value, "Start "); + + strictEqual(template.spans[1].isInterpolated, true); + strictEqual(template.spans[1].type.kind, "Number"); + strictEqual(template.spans[1].type.value, 123); + + strictEqual(template.spans[2].isInterpolated, false); + strictEqual(template.spans[2].type.value, " end"); + }); + + it("string interpolated are marked with isInterpolated", async () => { + const template = await compileStringTemplate(`"Start \${"interpolate"} end"`); + strictEqual(template.spans.length, 3); + strictEqual(template.spans[0].isInterpolated, false); + strictEqual(template.spans[0].type.value, "Start "); + + strictEqual(template.spans[1].isInterpolated, true); + strictEqual(template.spans[1].type.kind, "String"); + strictEqual(template.spans[1].type.value, "interpolate"); + + strictEqual(template.spans[2].isInterpolated, false); + strictEqual(template.spans[2].type.value, " end"); + }); + + it("can interpolate a model", async () => { + const template = await compileStringTemplate(`"Start \${TestModel} end"`, "model TestModel {}"); + strictEqual(template.spans.length, 3); + strictEqual(template.spans[0].isInterpolated, false); + strictEqual(template.spans[0].type.value, "Start "); + + strictEqual(template.spans[1].isInterpolated, true); + strictEqual(template.spans[1].type.kind, "Model"); + strictEqual(template.spans[1].type.name, "TestModel"); + + strictEqual(template.spans[2].isInterpolated, false); + strictEqual(template.spans[2].type.value, " end"); + }); +}); diff --git a/packages/compiler/test/formatter/formatter.test.ts b/packages/compiler/test/formatter/formatter.test.ts index 4c33378cbb..0bca93e85c 100644 --- a/packages/compiler/test/formatter/formatter.test.ts +++ b/packages/compiler/test/formatter/formatter.test.ts @@ -2728,4 +2728,54 @@ op test(): string; }); }); }); + + describe("string templates", () => { + describe("single line", () => { + it("format simple single line string template", async () => { + await assertFormat({ + code: `alias T = "foo \${ "def" } baz";`, + expected: `alias T = "foo \${"def"} baz";`, + }); + }); + + it("format simple single line string template with multiple interpolation", async () => { + await assertFormat({ + code: `alias T = "foo \${ "one" } bar \${"two" } baz";`, + expected: `alias T = "foo \${"one"} bar \${"two"} baz";`, + }); + }); + + it("format model expression in single line string template", async () => { + await assertFormat({ + code: `alias T = "foo \${ {foo: 1, bar: 2} } baz";`, + expected: ` +alias T = "foo \${{ + foo: 1; + bar: 2; +}} baz"; + `, + }); + }); + }); + describe("triple quoted", () => { + it("format simple single line string template", async () => { + await assertFormat({ + code: ` +alias T = """ + This \${ "one" } goes over + multiple + \${ "two" } + lines + """;`, + expected: ` +alias T = """ + This \${"one"} goes over + multiple + \${"two"} + lines + """;`, + }); + }); + }); + }); }); diff --git a/packages/compiler/test/helpers/string-template-utils.test.ts b/packages/compiler/test/helpers/string-template-utils.test.ts new file mode 100644 index 0000000000..1c2b2ea29e --- /dev/null +++ b/packages/compiler/test/helpers/string-template-utils.test.ts @@ -0,0 +1,50 @@ +import { strictEqual } from "assert"; +import { ModelProperty, stringTemplateToString } from "../../src/index.js"; +import { expectDiagnosticEmpty } from "../../src/testing/expect.js"; +import { createTestRunner } from "../../src/testing/test-host.js"; + +describe("compiler: stringTemplateToString", () => { + async function stringifyTemplate(template: string) { + const runner = await createTestRunner(); + const { value } = (await runner.compile(`model Foo { @test value: ${template}; }`)) as { + value: ModelProperty; + }; + + strictEqual(value.type.kind, "StringTemplate"); + return stringTemplateToString(value.type); + } + + async function expectTemplateToString(template: string, expectation: string) { + const [result, diagnostics] = await stringifyTemplate(template); + expectDiagnosticEmpty(diagnostics); + strictEqual(result, expectation); + } + + describe("interpolate types", () => { + it("string literal", async () => { + await expectTemplateToString('"Start ${"one"} end"', "Start one end"); + }); + + it("numeric literal", async () => { + await expectTemplateToString('"Start ${123} end"', "Start 123 end"); + }); + + it("boolean literal", async () => { + await expectTemplateToString('"Start ${true} end"', "Start true end"); + }); + + it("nested string template", async () => { + await expectTemplateToString( + '"Start ${"Nested-start ${"one"} nested-end"} end"', + "Start Nested-start one nested-end end" + ); + }); + }); + + it("stringify template with multiple spans", async () => { + await expectTemplateToString( + '"Start ${"one"} middle ${"two"} end"', + "Start one middle two end" + ); + }); +}); diff --git a/packages/compiler/test/parser.test.ts b/packages/compiler/test/parser.test.ts index f35c3768bf..2328da829d 100644 --- a/packages/compiler/test/parser.test.ts +++ b/packages/compiler/test/parser.test.ts @@ -8,11 +8,16 @@ import { NodeFlags, ParseOptions, SourceFile, + StringTemplateExpressionNode, SyntaxKind, TypeSpecScriptNode, } from "../src/core/types.js"; import { DecorableNode } from "../src/formatter/print/types.js"; -import { DiagnosticMatch, expectDiagnostics } from "../src/testing/expect.js"; +import { + DiagnosticMatch, + expectDiagnosticEmpty, + expectDiagnostics, +} from "../src/testing/expect.js"; describe("compiler: parser", () => { describe("import statements", () => { @@ -518,6 +523,141 @@ describe("compiler: parser", () => { parseErrorEach(bad.map((e) => [`model ${e[0]} {}`, [e[1]]])); }); + describe("string template expressions", () => { + function getNode(astNode: TypeSpecScriptNode): Node { + const statement = astNode.statements[0]; + strictEqual(statement.kind, SyntaxKind.AliasStatement); + return statement.value; + } + function getStringTemplateNode(astNode: TypeSpecScriptNode): StringTemplateExpressionNode { + const node = getNode(astNode); + strictEqual(node.kind, SyntaxKind.StringTemplateExpression); + return node; + } + + describe("single line", () => { + it("parse a single line template expression", () => { + const astNode = parseSuccessWithLog(`alias T = "Start \${"one"} middle \${23} end";`); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.value, "Start "); + strictEqual(node.spans.length, 2); + + const span0 = node.spans[0]; + strictEqual(span0.literal.value, " middle "); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + + const span1 = node.spans[1]; + strictEqual(span1.literal.value, " end"); + strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); + strictEqual(span1.expression.value, 23); + }); + + it("parse a single line template with a multi line model expression inside", () => { + const astNode = parseSuccessWithLog( + `alias T = "Start \${{ foo: "one",\nbar: "two" }} end";` + ); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.value, "Start "); + strictEqual(node.spans.length, 1); + + const span0 = node.spans[0]; + strictEqual(span0.literal.value, " end"); + strictEqual(span0.expression.kind, SyntaxKind.ModelExpression); + strictEqual(span0.expression.properties.length, 2); + }); + + it("can escape some ${}", () => { + const astNode = parseSuccessWithLog(`alias T = "Start \${"one"} middle \\\${23} end";`); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.value, "Start "); + strictEqual(node.spans.length, 1); + + const span0 = node.spans[0]; + strictEqual(span0.literal.value, " middle ${23} end"); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + }); + + it("can nest string templates", () => { + const astNode = parseSuccessWithLog( + 'alias T = "Start ${"nested-start ${"hi"} nested-end"} end";' + ); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.value, "Start "); + strictEqual(node.spans.length, 1); + + const span0 = node.spans[0]; + strictEqual(span0.literal.value, " end"); + strictEqual(span0.expression.kind, SyntaxKind.StringTemplateExpression); + strictEqual(span0.expression.head.value, "nested-start "); + strictEqual(span0.expression.spans.length, 1); + + const nestedSpan0 = span0.expression.spans[0]; + strictEqual(nestedSpan0.literal.value, " nested-end"); + strictEqual(nestedSpan0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(nestedSpan0.expression.value, "hi"); + }); + + it("string with all ${} escape is still a StringLiteral", () => { + const astNode = parseSuccessWithLog(`alias T = "Start \\\${12} middle \\\${23} end";`); + const node = getNode(astNode); + strictEqual(node.kind, SyntaxKind.StringLiteral); + strictEqual(node.value, "Start ${12} middle ${23} end"); + }); + }); + + describe("multi line", () => { + it("parse a multiple line template expression", () => { + const astNode = parseSuccessWithLog(`alias T = """ + Start \${"one"} + middle \${23} + end + """;`); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.value, "Start "); + strictEqual(node.spans.length, 2); + + const span0 = node.spans[0]; + strictEqual(span0.literal.value, " \nmiddle "); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + + const span1 = node.spans[1]; + strictEqual(span1.literal.value, " \nend"); + strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); + strictEqual(span1.expression.value, 23); + }); + + it("can escape some ${}", () => { + const astNode = parseSuccessWithLog(`alias T = """ + Start \${"one"} + middle \\\${23} + end + """;`); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.value, "Start "); + strictEqual(node.spans.length, 1); + + const span0 = node.spans[0]; + strictEqual(span0.literal.value, " \nmiddle ${23} \nend"); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + }); + + it("escaping all ${} still produce a string literal", () => { + const astNode = parseSuccessWithLog(`alias T = """ + Start \\\${12} + middle \\\${23} + end + """;`); + const node = getNode(astNode); + strictEqual(node.kind, SyntaxKind.StringLiteral); + strictEqual(node.value, "Start ${12} \nmiddle ${23} \nend"); + }); + }); + }); + // smaller repro of previous regen-samples baseline failures describe("sample regressions", () => { parseEach([ @@ -1224,6 +1364,33 @@ function checkPositioning(node: Node, file: SourceFile) { }); } +/** + * Parse the given code and log debug information. + */ +function parseWithLog(code: string, options?: ParseOptions): TypeSpecScriptNode { + logVerboseTestOutput("=== Source ==="); + logVerboseTestOutput(code); + + const astNode = parse(code, options); + logVerboseTestOutput("\n=== Parse Result ==="); + dumpAST(astNode); + return astNode; +} +/** + * Check the given code parse successfully and log debug information. + */ +function parseSuccessWithLog(code: string, options?: ParseOptions): TypeSpecScriptNode { + const astNode = parseWithLog(code, options); + logVerboseTestOutput("\n=== Diagnostics ==="); + logVerboseTestOutput((log) => { + for (const each of astNode.parseDiagnostics) { + log(formatDiagnostic(each)); + } + }); + expectDiagnosticEmpty(astNode.parseDiagnostics); + return astNode; +} + /** * * @param cases Test cases @@ -1237,16 +1404,10 @@ function parseErrorEach( ) { for (const [code, matches, callback] of cases) { it(`doesn't parse '${shorten(code)}'`, () => { - logVerboseTestOutput("=== Source ==="); - logVerboseTestOutput(code); - - const astNode = parse(code, options); + const astNode = parseWithLog(code, options); if (callback) { callback(astNode); } - logVerboseTestOutput("\n=== Parse Result ==="); - dumpAST(astNode); - logVerboseTestOutput("\n=== Diagnostics ==="); logVerboseTestOutput((log) => { for (const each of astNode.parseDiagnostics) { diff --git a/packages/compiler/test/scanner.test.ts b/packages/compiler/test/scanner.test.ts index 5cb7be94fc..8b52471635 100644 --- a/packages/compiler/test/scanner.test.ts +++ b/packages/compiler/test/scanner.test.ts @@ -13,6 +13,8 @@ import { isPunctuation, isStatementKeyword, } from "../src/core/scanner.js"; +import { DiagnosticMatch, expectDiagnostics } from "../src/testing/expect.js"; +import { extractSquiggles } from "../src/testing/test-server-host.js"; type TokenEntry = [ Token, @@ -215,10 +217,18 @@ describe("compiler: scanner", () => { ]); }); - function scanString(text: string, expectedValue: string, expectedDiagnostic?: RegExp) { + function scanString( + text: string, + expectedValue: string, + expectedDiagnostic?: RegExp | DiagnosticMatch + ) { const scanner = createScanner(text, (diagnostic) => { if (expectedDiagnostic) { - assert.match(diagnostic.message, expectedDiagnostic); + if (expectedDiagnostic instanceof RegExp) { + assert.match(diagnostic.message, expectedDiagnostic); + } else { + expectDiagnostics([diagnostic], expectedDiagnostic); + } } else { assert.fail("No diagnostic expected, but got " + formatDiagnostic(diagnostic)); } @@ -240,6 +250,16 @@ describe("compiler: scanner", () => { scanString('"Hello world \\r\\n \\t \\" \\\\ !"', 'Hello world \r\n \t " \\ !'); }); + it("report diagnostic when escaping invalid char", () => { + const { source, pos, end } = extractSquiggles('"Hello world ~~~\\d~~~"'); + scanString(source, "Hello world d", { + code: "invalid-escape-sequence", + message: "Invalid escape sequence.", + pos, + end, + }); + }); + it("does not allow multi-line, non-triple-quoted strings", () => { scanString('"More\r\nthan\r\none\r\nline"', "More", /Unterminated string/); scanString('"More\nthan\none\nline"', "More", /Unterminated string/); diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index ae991dcce0..a7a74d65a6 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -97,12 +97,18 @@ const Token = { begin: createToken("<", "punctuation.definition.typeparameters.begin.tsp"), end: createToken(">", "punctuation.definition.typeparameters.end.tsp"), }, + templateExpression: { + begin: createToken("${", "punctuation.definition.template-expression.begin.tsp"), + end: createToken("}", "punctuation.definition.template-expression.end.tsp"), + }, }, literals: { + escape: (char: string) => createToken(`\\${char}`, "constant.character.escape.tsp"), numeric: (text: string) => createToken(text, "constant.numeric.tsp"), - string: (text: string) => - createToken(text.startsWith('"') ? text : '"' + text + '"', "string.quoted.double.tsp"), + stringQuoted: (text: string) => createToken('"' + text + '"', "string.quoted.double.tsp"), + string: (text: string) => createToken(text, "string.quoted.double.tsp"), + stringTriple: (text: string) => createToken(text, "string.quoted.triple.tsp"), }, comment: { block: (text: string) => createToken(text, "comment.block.tsp"), @@ -114,7 +120,143 @@ testColorization("semantic colorization", tokenizeSemantic); testColorization("tmlanguage", tokenizeTMLanguage); function testColorization(description: string, tokenize: Tokenize) { + function joinTokensInSemantic(tokens: T[], separator: "" | "\n" = ""): T[] { + if (tokenize === tokenizeSemantic) { + return [createToken(tokens.map((x) => x.text).join(separator), tokens[0].scope)] as any; + } + return tokens; + } + describe(`compiler: server: ${description}`, () => { + describe("strings", () => { + function templateTripleOrDouble(text: string): Token { + return tokenize === tokenizeSemantic + ? Token.literals.string(text) + : Token.literals.stringTriple(text); + } + + describe("single line", () => { + it("tokenize template", async () => { + const tokens = await tokenize(`"Start \${123} end"`); + deepStrictEqual(tokens, [ + ...joinTokensInSemantic([Token.literals.string('"'), Token.literals.string("Start ")]), + Token.punctuation.templateExpression.begin, + Token.literals.numeric("123"), + Token.punctuation.templateExpression.end, + ...joinTokensInSemantic([Token.literals.string(" end"), Token.literals.string('"')]), + ]); + }); + + it("tokenize template with multiple interpolation", async () => { + const tokens = await tokenize(`"Start \${123} middle \${456} end"`); + deepStrictEqual(tokens, [ + ...joinTokensInSemantic([Token.literals.string('"'), Token.literals.string("Start ")]), + Token.punctuation.templateExpression.begin, + Token.literals.numeric("123"), + Token.punctuation.templateExpression.end, + Token.literals.string(" middle "), + Token.punctuation.templateExpression.begin, + Token.literals.numeric("456"), + Token.punctuation.templateExpression.end, + ...joinTokensInSemantic([Token.literals.string(" end"), Token.literals.string('"')]), + ]); + }); + + it("tokenize as a string if the template expression are escaped", async () => { + const tokens = await tokenize(`"Start \\\${123} end"`); + deepStrictEqual(tokens, [ + ...joinTokensInSemantic([ + Token.literals.string('"'), + Token.literals.string("Start "), + Token.literals.escape("$"), + Token.literals.string("{123} end"), + Token.literals.string('"'), + ]), + ]); + }); + + it("tokenize as a string if it is a simple string", async () => { + const tokens = await tokenize(`"Start end"`); + deepStrictEqual(tokens, [Token.literals.stringQuoted("Start end")]); + }); + }); + + describe("multi line", () => { + it("tokenize template", async () => { + const tokens = await tokenize(`""" + Start \${123} + end + """`); + deepStrictEqual(tokens, [ + ...joinTokensInSemantic( + [Token.literals.stringTriple('"""'), Token.literals.stringTriple(" Start ")], + "\n" + ), + Token.punctuation.templateExpression.begin, + Token.literals.numeric("123"), + Token.punctuation.templateExpression.end, + ...joinTokensInSemantic( + [ + templateTripleOrDouble(" "), + templateTripleOrDouble(" end"), + ...joinTokensInSemantic([ + templateTripleOrDouble(" "), + templateTripleOrDouble('"""'), + ]), + ], + "\n" + ), + ]); + }); + + it("tokenize as a string if the template expression are escaped", async () => { + const tokens = await tokenize(`""" + Start \\\${123} + end + """`); + deepStrictEqual(tokens, [ + ...joinTokensInSemantic( + [ + Token.literals.stringTriple('"""'), + ...joinTokensInSemantic([ + Token.literals.stringTriple(" Start "), + Token.literals.escape("$"), + Token.literals.stringTriple("{123} "), + ]), + Token.literals.stringTriple(" end"), + ...joinTokensInSemantic([ + Token.literals.stringTriple(" "), + Token.literals.stringTriple('"""'), + ]), + ], + "\n" + ), + ]); + }); + + it("tokenize as a simple string", async () => { + const tokens = await tokenize(`""" + Start + end + """`); + deepStrictEqual(tokens, [ + ...joinTokensInSemantic( + [ + Token.literals.stringTriple(`"""`), + Token.literals.stringTriple(" Start"), + Token.literals.stringTriple(" end"), + ...joinTokensInSemantic([ + Token.literals.stringTriple(" "), + Token.literals.stringTriple(`"""`), + ]), + ], + "\n" + ), + ]); + }); + }); + }); + describe("aliases", () => { it("simple alias", async () => { const tokens = await tokenize("alias Foo = string"); @@ -213,7 +355,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.identifiers.tag("@"), Token.identifiers.tag("foo"), Token.punctuation.openParen, - Token.literals.string("param1"), + Token.literals.stringQuoted("param1"), Token.punctuation.comma, Token.literals.numeric("123"), Token.punctuation.closeParen, @@ -226,7 +368,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.punctuation.openParen, Token.identifiers.type("MyModel"), Token.punctuation.comma, - Token.literals.string("param1"), + Token.literals.stringQuoted("param1"), Token.punctuation.comma, Token.literals.numeric("123"), Token.punctuation.closeParen, @@ -532,7 +674,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.operators.typeAnnotation, Token.identifiers.type("string"), Token.operators.assignment, - Token.literals.string("my-default"), + Token.literals.stringQuoted("my-default"), Token.punctuation.semicolon, Token.punctuation.closeBrace, ]); @@ -714,7 +856,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.operators.typeAnnotation, Token.identifiers.type("string"), Token.operators.assignment, - Token.literals.string("my-default"), + Token.literals.stringQuoted("my-default"), Token.punctuation.closeParen, Token.operators.typeAnnotation, @@ -1099,11 +1241,24 @@ export async function tokenizeSemantic(input: string): Promise { const semanticTokens = await host.server.getSemanticTokens({ textDocument: document }); const tokens = []; + let templateStack = 0; for (const semanticToken of semanticTokens) { const text = file.text.substring(semanticToken.pos, semanticToken.end); - const token = convertSemanticToken(semanticToken, text); - if (token) { - tokens.push(token); + if (text === "${" && semanticToken.kind === SemanticTokenKind.Operator) { + templateStack++; + tokens.push(Token.punctuation.templateExpression.begin); + } else if ( + templateStack > 0 && + text === "}" && + semanticToken.kind === SemanticTokenKind.Operator + ) { + templateStack--; + tokens.push(Token.punctuation.templateExpression.end); + } else { + const token = convertSemanticToken(semanticToken, text); + if (token) { + tokens.push(token); + } } } @@ -1131,7 +1286,9 @@ export async function tokenizeSemantic(input: string): Promise { case SemanticTokenKind.Keyword: return Token.keywords.other(text); case SemanticTokenKind.String: - return Token.literals.string(text); + return text.startsWith(`"""`) + ? Token.literals.stringTriple(text) + : Token.literals.string(text); case SemanticTokenKind.Comment: return Token.comment.block(text); case SemanticTokenKind.Number: @@ -1269,7 +1426,9 @@ function getPunctuationMap(): ReadonlyMap { if ("text" in value) { map.set(value.text, value); } else { - visit(value); + if (value !== Token.punctuation.templateExpression) { + visit(value); + } } } } diff --git a/packages/json-schema/src/json-schema-emitter.ts b/packages/json-schema/src/json-schema-emitter.ts index 1c498f851d..760547b246 100644 --- a/packages/json-schema/src/json-schema-emitter.ts +++ b/packages/json-schema/src/json-schema-emitter.ts @@ -30,6 +30,8 @@ import { Program, Scalar, StringLiteral, + StringTemplate, + stringTemplateToString, Tuple, Type, typespecTypeToJson, @@ -232,6 +234,17 @@ export class JsonSchemaEmitter extends TypeEmitter, JSONSche return { type: "string", const: string.value }; } + stringTemplate(string: StringTemplate): EmitterOutput { + const [value, diagnostics] = stringTemplateToString(string); + if (diagnostics.length > 0) { + this.emitter + .getProgram() + .reportDiagnostics(diagnostics.map((x) => ({ ...x, severity: "warning" }))); + return { type: "string" }; + } + return { type: "string", const: value }; + } + numericLiteral(number: NumericLiteral): EmitterOutput { return { type: "number", const: number.value }; } diff --git a/packages/json-schema/test/string-template.test.ts b/packages/json-schema/test/string-template.test.ts new file mode 100644 index 0000000000..5738116e05 --- /dev/null +++ b/packages/json-schema/test/string-template.test.ts @@ -0,0 +1,66 @@ +import { expectDiagnostics } from "@typespec/compiler/testing"; +import { deepStrictEqual } from "assert"; +import { emitSchema, emitSchemaWithDiagnostics } from "./utils.js"; + +describe("json-schema: string templates", () => { + describe("handle interpolating literals", () => { + it("string", async () => { + const schemas = await emitSchema(` + model Test { + a: "Start \${"abc"} end", + } + `); + + deepStrictEqual(schemas["Test.json"].properties.a, { + type: "string", + const: "Start abc end", + }); + }); + + it("number", async () => { + const schemas = await emitSchema(` + model Test { + a: "Start \${123} end", + } + `); + + deepStrictEqual(schemas["Test.json"].properties.a, { + type: "string", + const: "Start 123 end", + }); + }); + + it("boolean", async () => { + const schemas = await emitSchema(` + model Test { + a: "Start \${true} end", + } + `); + + deepStrictEqual(schemas["Test.json"].properties.a, { + type: "string", + const: "Start true end", + }); + }); + }); + + it("emit diagnostics if interpolation value are not literals", async () => { + const [schemas, diagnostics] = await emitSchemaWithDiagnostics(` + model Test { + a: "Start \${Bar} end", + } + model Bar {} + `); + + deepStrictEqual(schemas["Test.json"].properties.a, { + type: "string", + }); + + expectDiagnostics(diagnostics, { + code: "non-literal-string-template", + severity: "warning", + message: + "Value interpolated in this string template cannot be converted to a string. Only literal types can be automatically interpolated.", + }); + }); +}); diff --git a/packages/openapi3/src/schema-emitter.ts b/packages/openapi3/src/schema-emitter.ts index c6fda7d378..fc30516f2c 100644 --- a/packages/openapi3/src/schema-emitter.ts +++ b/packages/openapi3/src/schema-emitter.ts @@ -37,6 +37,8 @@ import { Program, Scalar, StringLiteral, + StringTemplate, + stringTemplateToString, Tuple, Type, TypeNameOptions, @@ -368,6 +370,17 @@ export class OpenAPI3SchemaEmitter extends TypeEmitter< return { type: "string", enum: [string.value] }; } + stringTemplate(string: StringTemplate): EmitterOutput { + const [value, diagnostics] = stringTemplateToString(string); + if (diagnostics.length > 0) { + this.emitter + .getProgram() + .reportDiagnostics(diagnostics.map((x) => ({ ...x, severity: "warning" }))); + return { type: "string" }; + } + return { type: "string", enum: [value] }; + } + numericLiteral(number: NumericLiteral): EmitterOutput { return { type: "number", enum: [number.value] }; } diff --git a/packages/openapi3/test/string-template.test.ts b/packages/openapi3/test/string-template.test.ts new file mode 100644 index 0000000000..fe9c684f81 --- /dev/null +++ b/packages/openapi3/test/string-template.test.ts @@ -0,0 +1,66 @@ +import { expectDiagnostics } from "@typespec/compiler/testing"; +import { deepStrictEqual } from "assert"; +import { emitOpenApiWithDiagnostics, openApiFor } from "./test-host.js"; + +describe("openapi3: string templates", () => { + describe("handle interpolating literals", () => { + it("string", async () => { + const schemas = await openApiFor(` + model Test { + a: "Start \${"abc"} end", + } + `); + + deepStrictEqual(schemas.components?.schemas?.Test.properties.a, { + type: "string", + enum: ["Start abc end"], + }); + }); + + it("number", async () => { + const schemas = await openApiFor(` + model Test { + a: "Start \${123} end", + } + `); + + deepStrictEqual(schemas.components?.schemas?.Test.properties.a, { + type: "string", + enum: ["Start 123 end"], + }); + }); + + it("boolean", async () => { + const schemas = await openApiFor(` + model Test { + a: "Start \${true} end", + } + `); + + deepStrictEqual(schemas.components?.schemas?.Test.properties.a, { + type: "string", + enum: ["Start true end"], + }); + }); + }); + + it("emit diagnostics if interpolation value are not literals", async () => { + const [schemas, diagnostics] = await emitOpenApiWithDiagnostics(` + model Test { + a: "Start \${Bar} end", + } + model Bar {} + `); + + deepStrictEqual(schemas.components?.schemas?.Test.properties?.a, { + type: "string", + }); + + expectDiagnostics(diagnostics, { + code: "non-literal-string-template", + severity: "warning", + message: + "Value interpolated in this string template cannot be converted to a string. Only literal types can be automatically interpolated.", + }); + }); +}); diff --git a/packages/openapi3/test/test-host.ts b/packages/openapi3/test/test-host.ts index c238e3695a..b880319513 100644 --- a/packages/openapi3/test/test-host.ts +++ b/packages/openapi3/test/test-host.ts @@ -1,4 +1,4 @@ -import { interpolatePath } from "@typespec/compiler"; +import { Diagnostic, interpolatePath } from "@typespec/compiler"; import { createTestHost, createTestWrapper, @@ -9,8 +9,10 @@ import { HttpTestLibrary } from "@typespec/http/testing"; import { OpenAPITestLibrary } from "@typespec/openapi/testing"; import { RestTestLibrary } from "@typespec/rest/testing"; import { VersioningTestLibrary } from "@typespec/versioning/testing"; +import { ok } from "assert"; import { OpenAPI3EmitterOptions } from "../src/lib.js"; import { OpenAPI3TestLibrary } from "../src/testing/index.js"; +import { OpenAPI3Document } from "../src/types.js"; export async function createOpenAPITestHost() { return createTestHost({ @@ -47,6 +49,25 @@ export async function createOpenAPITestRunner({ }); } +export async function emitOpenApiWithDiagnostics( + code: string, + options: OpenAPI3EmitterOptions = {} +): Promise<[OpenAPI3Document, readonly Diagnostic[]]> { + const runner = await createOpenAPITestRunner(); + const outputFile = resolveVirtualPath("openapi.json"); + const diagnostics = await runner.diagnose(code, { + noEmit: false, + emit: ["@typespec/openapi3"], + options: { + "@typespec/openapi3": { ...options, "output-file": outputFile }, + }, + }); + const content = runner.fs.get(outputFile); + ok(content, "Expected to have found openapi output"); + const doc = JSON.parse(content); + return [doc, diagnostics]; +} + export async function diagnoseOpenApiFor(code: string, options: OpenAPI3EmitterOptions = {}) { const runner = await createOpenAPITestRunner(); const diagnostics = await runner.diagnose(code, { diff --git a/packages/samples/specs/string-template/main.tsp b/packages/samples/specs/string-template/main.tsp new file mode 100644 index 0000000000..686031b287 --- /dev/null +++ b/packages/samples/specs/string-template/main.tsp @@ -0,0 +1,23 @@ +alias myconst = "foobar"; + +model Person { + simple: "Simple ${123} end"; + multiline: """ + Multi + ${123} + ${true} + line + """; + ref: "Ref this alias ${myconst} end"; + template: Template<"custom">; +} + +alias Template = "Foo ${T} bar"; + +/** Example of string template with template parameters */ +@doc("Animal named: ${T}") +model Animal { + kind: T; +} + +model Cat is Animal<"Cat">; diff --git a/packages/samples/test/output/string-template/@typespec/openapi3/openapi.yaml b/packages/samples/test/output/string-template/@typespec/openapi3/openapi.yaml new file mode 100644 index 0000000000..2291a05bc9 --- /dev/null +++ b/packages/samples/test/output/string-template/@typespec/openapi3/openapi.yaml @@ -0,0 +1,46 @@ +openapi: 3.0.0 +info: + title: (title) + version: 0000-00-00 +tags: [] +paths: {} +components: + schemas: + Cat: + type: object + required: + - kind + properties: + kind: + type: string + enum: + - Cat + description: 'Animal named: Cat' + Person: + type: object + required: + - simple + - multiline + - ref + - template + properties: + simple: + type: string + enum: + - Simple 123 end + multiline: + type: string + enum: + - |- + Multi + 123 + true + line + ref: + type: string + enum: + - Ref this alias foobar end + template: + type: string + enum: + - Foo custom bar diff --git a/packages/spec/src/spec.emu.html b/packages/spec/src/spec.emu.html index 144a433acc..46840457fc 100644 --- a/packages/spec/src/spec.emu.html +++ b/packages/spec/src/spec.emu.html @@ -154,14 +154,39 @@

Lexical Grammar

StringLiteral : `"` StringCharacters? `"` `"""` TripleQuotedStringCharacters? `"""` + StringTemplate + +StringTemplate : + TemplateHead Expression TemplateSpans + +TemplateSpans : + TemplateTail + TemplateMiddleList TemplateTail + +TemplateMiddleList : + StringTemplateMiddle Expression + StringTemplateMiddle Expression TemplateMiddleList + +StringTemplateHead : + `"` StringCharacters? `${` + `"""` TripleQuotedStringCharacters? `${` + +StringTemplateMiddle : + `}` TemplateCharacters? `${` + + +StringTemplateTail : + `}` TemplateCharacters? `"` + `}` TemplateCharacters? `"""` StringCharacters : StringCharacter StringCharacters? StringCharacter : + `$` [lookahead != `{`] SourceCharacter but not one of `"` or `\` or LineTerminator `\` EscapeCharacter - + /// // BUG: This does not specify the extra rules about `"""`s going // on their own lines and having consistent indentation. diff --git a/packages/tspd/src/ref-doc/utils/type-signature.ts b/packages/tspd/src/ref-doc/utils/type-signature.ts index 7edeea1b67..15b56f8d96 100644 --- a/packages/tspd/src/ref-doc/utils/type-signature.ts +++ b/packages/tspd/src/ref-doc/utils/type-signature.ts @@ -9,6 +9,7 @@ import { Model, ModelProperty, Operation, + StringTemplate, TemplateParameterDeclarationNode, Type, UnionVariant, @@ -46,9 +47,12 @@ export function getTypeSignature(type: Type | ValueType): string { return `(number) ${type.value.toString()}`; case "Intrinsic": return `(intrinsic) ${type.name}`; - case "FunctionParameter": return getFunctionParameterSignature(type); + case "StringTemplate": + return `(string template)\n${getStringTemplateSignature(type)}`; + case "StringTemplateSpan": + return `(string template span)\n${getTypeName(type.type)}`; case "ModelProperty": return `(model property) ${`${type.name}: ${getTypeName(type.type)}`}`; case "EnumMember": @@ -119,6 +123,18 @@ function getFunctionParameterSignature(parameter: FunctionParameter) { return `${rest}${parameter.name}${optional}: ${getTypeName(parameter.type)}`; } +function getStringTemplateSignature(stringTemplate: StringTemplate) { + return ( + "`" + + [ + stringTemplate.spans.map((span) => { + return span.isInterpolated ? "${" + getTypeName(span.type) + "}" : span.type.value; + }), + ].join("") + + "`" + ); +} + function getModelPropertySignature(property: ModelProperty) { const ns = getQualifier(property.model); return `${ns}${property.name}: ${getTypeName(property.type)}`; diff --git a/packages/website/docusaurus.config.ts b/packages/website/docusaurus.config.ts index 12b3d70dbe..366219b331 100644 --- a/packages/website/docusaurus.config.ts +++ b/packages/website/docusaurus.config.ts @@ -240,7 +240,7 @@ const config: Config = { }, prism: { theme: themes.oneLight, - darkTheme: themes.dracula, + darkTheme: themes.oneDark, additionalLanguages: ["http"], }, mermaid: {}, diff --git a/packages/website/src/theme/typespec-lang-prism.ts b/packages/website/src/theme/typespec-lang-prism.ts index 9142a0c139..f47d7acfbf 100644 --- a/packages/website/src/theme/typespec-lang-prism.ts +++ b/packages/website/src/theme/typespec-lang-prism.ts @@ -1,4 +1,4 @@ -export default { +const lang = { comment: [ { // multiline comments eg /* ASDF */ @@ -34,14 +34,33 @@ export default { }, string: [ + // https://docs.swift.org/swift-book/LanguageGuide/StringsAndCharacters.html { - pattern: /"""[^"][\s\S]*?"""/, - greedy: true, - }, - { - pattern: /(^|[^\\"])"(?:\\.|\$(?!\{)|[^"\\\r\n$])*"/, + pattern: new RegExp( + /(^|[^"#])/.source + + "(?:" + + // multi-line string + /"""(?:\\(?:\$\{(?:[^{}]|\$\{[^{}]*\})*\}|[^(])|[^\\"]|"(?!""))*"""/.source + + "|" + + // single-line string + /"(?:\\(?:\$\{(?:[^{}]|\$\{[^{}]*\})*\}|\r\n|[^(])|[^\\\r\n"])*"/.source + + ")" + ), lookbehind: true, greedy: true, + inside: { + interpolation: { + pattern: /(\$\{)(?:[^{}]|\$\{[^{}]*\})*(?=\})/, + lookbehind: true, + inside: null, // see below + }, + "interpolation-punctuation": { + pattern: /^\}|\$\{$/, + alias: "punctuation", + }, + punctuation: /\\(?=[\r\n])/, + string: /[\s\S]+/, + }, }, ], @@ -50,9 +69,13 @@ export default { /\b(?:import|model|scalar|namespace|op|interface|union|using|is|extends|enum|alias|return|void|never|if|else|projection|dec|extern|fn)\b/, function: /\b[a-z_]\w*(?=[ \t]*\()/i, + variable: /\b(?:[A-Z_\d]*[a-z]\w*)?\b/, number: /(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:E[+-]?\d+)?/i, operator: /--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/, punctuation: /[{}[\];(),.:]/, }; + +lang.string[0].inside.interpolation.inside = lang; +export default lang;