From ef77107371d2eb72de054ee4b528b6451d081eb7 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Wed, 25 Oct 2023 19:44:10 +0200 Subject: [PATCH] refactor: turn Python generator into a service Co-authored-by: WinPlay02 --- package-lock.json | 15 +- package.json | 3 +- src/cli/cli-util.ts | 24 +- src/cli/generator.ts | 616 +----------------- .../generation/safe-ds-python-generator.ts | 615 +++++++++++++++++ src/language/safe-ds-module.ts | 7 + src/language/validation/names.ts | 2 +- .../generation/testGeneration.test.ts | 30 +- 8 files changed, 664 insertions(+), 648 deletions(-) create mode 100644 src/language/generation/safe-ds-python-generator.ts diff --git a/package-lock.json b/package-lock.json index 8afd0cb52..b87c13880 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,8 @@ "langium": "^2.0.2", "true-myth": "^7.1.0", "vscode-languageclient": "^9.0.1", - "vscode-languageserver": "^9.0.1" + "vscode-languageserver": "^9.0.1", + "vscode-languageserver-textdocument": "^1.0.11" }, "bin": { "safe-ds-cli": "bin/cli" @@ -13169,9 +13170,9 @@ } }, "node_modules/vscode-languageserver-textdocument": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.8.tgz", - "integrity": "sha512-1bonkGqQs5/fxGT5UchTgjGVnfysL0O8v1AYMBjqTbWQTFn721zaPGDYFkOKtfDgFiSgXM3KwaG3FMGfW4Ed9Q==" + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.11.tgz", + "integrity": "sha512-X+8T3GoiwTVlJbicx/sIAF+yuJAqz8VvwJyoMVhwEMoEKE/fkDmrqUgDMyBECcM2A2frVZIUj5HI/ErRXCfOeA==" }, "node_modules/vscode-languageserver-types": { "version": "3.17.5", @@ -22556,9 +22557,9 @@ } }, "vscode-languageserver-textdocument": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.8.tgz", - "integrity": "sha512-1bonkGqQs5/fxGT5UchTgjGVnfysL0O8v1AYMBjqTbWQTFn721zaPGDYFkOKtfDgFiSgXM3KwaG3FMGfW4Ed9Q==" + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.11.tgz", + "integrity": "sha512-X+8T3GoiwTVlJbicx/sIAF+yuJAqz8VvwJyoMVhwEMoEKE/fkDmrqUgDMyBECcM2A2frVZIUj5HI/ErRXCfOeA==" }, "vscode-languageserver-types": { "version": "3.17.5", diff --git a/package.json b/package.json index 9ea3d856a..667254bb3 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,8 @@ "langium": "^2.0.2", "true-myth": "^7.1.0", "vscode-languageclient": "^9.0.1", - "vscode-languageserver": "^9.0.1" + "vscode-languageserver": "^9.0.1", + "vscode-languageserver-textdocument": "^1.0.11" }, "devDependencies": { "@lars-reimann/eslint-config": "^5.1.4", diff --git a/src/cli/cli-util.ts b/src/cli/cli-util.ts index 19f0b723c..5f9ebf8ce 100644 --- a/src/cli/cli-util.ts +++ b/src/cli/cli-util.ts @@ -1,16 +1,9 @@ import chalk from 'chalk'; import path from 'path'; import fs from 'fs'; -import { AstNode, LangiumDocument, LangiumServices, URI } from 'langium'; +import { LangiumDocument, LangiumServices, URI } from 'langium'; /* c8 ignore start */ -export const extractAstNode = async function ( - fileName: string, - services: LangiumServices, -): Promise { - return (await extractDocument(fileName, services)).parseResult?.value as T; -}; - export const extractDocument = async function (fileName: string, services: LangiumServices): Promise { const extensions = services.LanguageMetaData.fileExtensions; if (!extensions.includes(path.extname(fileName))) { @@ -45,19 +38,6 @@ export const extractDocument = async function (fileName: string, services: Langi process.exit(1); } - return document; + return document as LangiumDocument; }; /* c8 ignore stop */ - -interface FilePathData { - destination: string; - name: string; -} - -export const extractDestinationAndName = function (filePath: string, destination: string | undefined): FilePathData { - const baseFilePath = path.basename(filePath, path.extname(filePath)).replace(/[.-]/gu, ''); - return { - destination: destination ?? path.join(path.dirname(baseFilePath), 'generated'), - name: path.basename(baseFilePath), - }; -}; diff --git a/src/cli/generator.ts b/src/cli/generator.ts index 5284ffb39..d6587f0c1 100644 --- a/src/cli/generator.ts +++ b/src/cli/generator.ts @@ -1,617 +1,33 @@ -import fs from 'fs'; -import { - expandToString, - expandToStringWithNL, - findRootNode, - getContainerOfType, - getDocument, - streamAllContents, -} from 'langium'; -import path from 'path'; -import { - isSdsAbstractResult, - isSdsAssignment, - isSdsBlockLambda, - isSdsBlockLambdaResult, - isSdsCall, - isSdsCallable, - isSdsEnumVariant, - isSdsExpressionLambda, - isSdsExpressionStatement, - isSdsFunction, - isSdsIndexedAccess, - isSdsInfixOperation, - isSdsList, - isSdsMap, - isSdsMemberAccess, - isSdsParenthesizedExpression, - isSdsPipeline, - isSdsPlaceholder, - isSdsPrefixOperation, - isSdsQualifiedImport, - isSdsReference, - isSdsSegment, - isSdsTemplateString, - isSdsTemplateStringEnd, - isSdsTemplateStringInner, - isSdsTemplateStringPart, - isSdsTemplateStringStart, - isSdsWildcard, - isSdsWildcardImport, - isSdsYield, - SdsArgument, - SdsAssignee, - SdsAssignment, - SdsBlock, - SdsBlockLambda, - SdsDeclaration, - SdsExpression, - SdsModule, - SdsParameter, - SdsParameterList, - SdsPipeline, - SdsSegment, - SdsStatement, -} from '../language/generated/ast.js'; -import { extractAstNode, extractDestinationAndName } from './cli-util.js'; import chalk from 'chalk'; -import { createSafeDsServices, SafeDsServices } from '../language/safe-ds-module.js'; +import { createSafeDsServices } from '../language/safe-ds-module.js'; import { NodeFileSystem } from 'langium/node'; -import { - getAbstractResults, - getAssignees, - getImportedDeclarations, - getImports, - getModuleMembers, - getStatements, - isRequiredParameter, - streamBlockLambdaResults, -} from '../language/helpers/nodeProperties.js'; -import { IdManager } from '../language/helpers/idManager.js'; -import { isInStubFile } from '../language/helpers/fileExtensions.js'; -import { - BooleanConstant, - FloatConstant, - IntConstant, - NullConstant, - StringConstant, -} from '../language/partialEvaluation/model.js'; -import { groupBy } from '../helpers/collectionUtils.js'; - -export const CODEGEN_PREFIX = '__gen_'; -const BLOCK_LAMBDA_PREFIX = `${CODEGEN_PREFIX}block_lambda_`; -const BLOCK_LAMBDA_RESULT_PREFIX = `${CODEGEN_PREFIX}block_lambda_result_`; -const YIELD_PREFIX = `${CODEGEN_PREFIX}yield_`; - -const RUNNER_CODEGEN_PACKAGE = 'safeds_runner.codegen'; -const PYTHON_INDENT = ' '; +import fs from 'node:fs'; +import { URI } from 'langium'; +import path from 'node:path'; +import { extractDocument } from './cli-util.js'; /* c8 ignore start */ export const generateAction = async (fileName: string, opts: GenerateOptions): Promise => { const services = createSafeDsServices(NodeFileSystem).SafeDs; - const module = await extractAstNode(fileName, services); - const generatedFilePath = generatePython(services, module, fileName, opts.destination); - // eslint-disable-next-line no-console - console.log(chalk.green(`Python code generated successfully: ${generatedFilePath}`)); -}; -/* c8 ignore stop */ - -export const generatePython = function ( - services: SafeDsServices, - module: SdsModule, - filePath: string, - destination: string | undefined, -): string[] { - // Do not generate stub files - if (isInStubFile(module)) { - return []; - } - const data = extractDestinationAndName(filePath, destination); - const pythonModuleName = services.builtins.Annotations.getPythonModule(module); - const packagePath = pythonModuleName === undefined ? module.name.split('.') : [pythonModuleName]; - const parentDirectoryPath = path.join(data.destination, ...packagePath); - - const generatedFiles = new Map(); - generatedFiles.set( - `${path.join(parentDirectoryPath, formatGeneratedFileName(data.name))}.py`, - generateModule(services, module), - ); - for (const pipeline of streamAllContents(module).filter(isSdsPipeline)) { - const entryPointFilename = `${path.join( - parentDirectoryPath, - `${formatGeneratedFileName(data.name)}_${getPythonNameOrDefault(services, pipeline)}`, - )}.py`; - const entryPointContent = expandToStringWithNL`from ${formatGeneratedFileName( - data.name, - )} import ${getPythonNameOrDefault( - services, - pipeline, - )}\n\nif __name__ == '__main__':\n${PYTHON_INDENT}${getPythonNameOrDefault(services, pipeline)}()`; - generatedFiles.set(entryPointFilename, entryPointContent); - } - if (!fs.existsSync(parentDirectoryPath)) { - fs.mkdirSync(parentDirectoryPath, { recursive: true }); - } - for (const [generatedFilePath, generatedFileContent] of generatedFiles.entries()) { - fs.writeFileSync(generatedFilePath, generatedFileContent); - } - return [...generatedFiles.keys()]; -}; - -const getPythonNameOrDefault = function ( - services: SafeDsServices, - object: SdsPipeline | SdsSegment | SdsParameter | SdsDeclaration, -) { - return services.builtins.Annotations.getPythonName(object) || object.name; -}; - -const formatGeneratedFileName = function (baseName: string): string { - return `gen_${baseName.replaceAll('%2520', '_').replaceAll(/[ .-]/gu, '_').replaceAll(/\\W/gu, '')}`; -}; - -const generateModule = function (services: SafeDsServices, module: SdsModule): string { - const importSet = new Map(); - const segments = getModuleMembers(module) - .filter(isSdsSegment) - .map((segment) => generateSegment(services, segment, importSet)); - const pipelines = getModuleMembers(module) - .filter(isSdsPipeline) - .map((pipeline) => generatePipeline(services, pipeline, importSet)); - const imports = generateImports(Array.from(importSet.values())); - const output: string[] = []; - if (imports.length > 0) { - output.push( - expandToStringWithNL`# Imports ----------------------------------------------------------------------\n\n${imports.join( - '\n', - )}`, - ); - } - if (segments.length > 0) { - output.push( - expandToStringWithNL`# Segments ---------------------------------------------------------------------\n\n${segments.join( - '\n\n', - )}`, - ); - } - if (pipelines.length > 0) { - output.push( - expandToStringWithNL`# Pipelines --------------------------------------------------------------------\n\n${pipelines.join( - '\n\n', - )}`, - ); - } - return expandToStringWithNL`${output.join('\n')}`; -}; - -const generateSegment = function ( - services: SafeDsServices, - segment: SdsSegment, - importSet: Map, -): string { - const infoFrame = new GenerationInfoFrame(services, importSet); - const segmentResult = segment.resultList?.results || []; - let segmentBlock = generateBlock(segment.body, infoFrame); - if (segmentResult.length !== 0) { - segmentBlock += `\nreturn ${segmentResult.map((result) => `${YIELD_PREFIX}${result.name}`).join(', ')}`; - } - return expandToString`def ${getPythonNameOrDefault(services, segment)}(${generateParameters( - segment.parameterList, - infoFrame, - )}):\n${PYTHON_INDENT}${segmentBlock}`; -}; - -const generateParameters = function (parameters: SdsParameterList | undefined, frame: GenerationInfoFrame): string { - const result = (parameters?.parameters || []).map((param) => generateParameter(param, frame)); - return result.join(', '); -}; - -const generateParameter = function ( - parameter: SdsParameter, - frame: GenerationInfoFrame, - defaultValue: boolean = true, -): string { - return expandToString`${getPythonNameOrDefault(frame.services, parameter)}${ - defaultValue && parameter.defaultValue !== undefined - ? '=' + generateExpression(parameter.defaultValue, frame) - : '' - }`; -}; - -const generatePipeline = function ( - services: SafeDsServices, - pipeline: SdsPipeline, - importSet: Map, -): string { - const infoFrame = new GenerationInfoFrame(services, importSet); - return expandToString`def ${getPythonNameOrDefault(services, pipeline)}():\n${PYTHON_INDENT}${generateBlock( - pipeline.body, - infoFrame, - )}`; -}; - -const generateImports = function (importSet: ImportData[]): string[] { - const qualifiedImports = importSet - .filter((importStmt) => importStmt.declarationName === undefined) - .sort((a, b) => a.importPath.localeCompare(b.importPath)) - .map(generateQualifiedImport); - const groupedImports = groupBy( - importSet.filter((importStmt) => importStmt.declarationName !== undefined), - (importStmt) => importStmt.importPath, - ) - .toArray() - .sort(([key1, _value1], [key2, _value2]) => key1.localeCompare(key2)); - const declaredImports: string[] = []; - for (const [key, value] of groupedImports) { - const importedDecls = - value - ?.filter((importData) => importData !== undefined) - .sort((a, b) => a.declarationName!.localeCompare(b.declarationName!)) - .map((localValue) => - localValue.alias !== undefined - ? `${localValue.declarationName} as ${localValue.alias}` - : localValue.declarationName!, - ) || []; - declaredImports.push(`from ${key} import ${[...new Set(importedDecls)].join(', ')}`); - } - return [...new Set(qualifiedImports), ...new Set(declaredImports)]; -}; - -const generateQualifiedImport = function (importStmt: ImportData): string { - if (importStmt.alias === undefined) { - return `import ${importStmt.importPath}`; - } else { - /* c8 ignore next 2 */ - return `import ${importStmt.importPath} as ${importStmt.alias}`; - } -}; - -const generateBlock = function (block: SdsBlock, frame: GenerationInfoFrame): string { - // TODO filter withEffect - let statements = getStatements(block); - if (statements.length === 0) { - return 'pass'; - } - return expandToString`${statements.map((stmt) => generateStatement(stmt, frame)).join('\n')}`; -}; - -const generateStatement = function (statement: SdsStatement, frame: GenerationInfoFrame): string { - if (isSdsAssignment(statement)) { - return generateAssignment(statement, frame); - } else if (isSdsExpressionStatement(statement)) { - const expressionStatement = statement; - const blockLambdaCode: string[] = []; - for (const lambda of streamAllContents(expressionStatement.expression).filter(isSdsBlockLambda)) { - blockLambdaCode.push(generateBlockLambda(lambda, frame)); - } - blockLambdaCode.push(generateExpression(expressionStatement.expression, frame)); - return expandToString`${blockLambdaCode.join('\n')}`; - } - /* c8 ignore next 2 */ - throw new Error(`Unknown SdsStatement: ${statement}`); -}; - -const generateAssignment = function (assignment: SdsAssignment, frame: GenerationInfoFrame): string { - const requiredAssignees = isSdsCall(assignment.expression) - ? getAbstractResults(frame.services.helpers.NodeMapper.callToCallable(assignment.expression)).length - : /* c8 ignore next */ - 1; - const assignees = getAssignees(assignment); - if (assignees.some((value) => !isSdsWildcard(value))) { - const actualAssignees = assignees.map(generateAssignee); - if (requiredAssignees === actualAssignees.length) { - return `${actualAssignees.join(', ')} = ${generateExpression(assignment.expression!, frame)}`; - } else { - // Add wildcards to match given results - return `${actualAssignees - .concat(Array(requiredAssignees - actualAssignees.length).fill('_')) - .join(', ')} = ${generateExpression(assignment.expression!, frame)}`; - } - } else { - return generateExpression(assignment.expression!, frame); - } -}; + const document = await extractDocument(fileName, services); + const generatedFiles = services.generation.PythonGenerator.generate(document, opts.destination); -const generateAssignee = function (assignee: SdsAssignee): string { - if (isSdsBlockLambdaResult(assignee)) { - return `${BLOCK_LAMBDA_RESULT_PREFIX}${assignee.name}`; - } else if (isSdsPlaceholder(assignee)) { - return assignee.name; - } else if (isSdsWildcard(assignee)) { - return '_'; - } else if (isSdsYield(assignee)) { - return `${YIELD_PREFIX}${assignee.result?.ref?.name!}`; - } - /* c8 ignore next 2 */ - throw new Error(`Unknown SdsAssignment: ${assignee.$type}`); -}; - -const generateBlockLambda = function (blockLambda: SdsBlockLambda, frame: GenerationInfoFrame): string { - const results = streamBlockLambdaResults(blockLambda); - let lambdaBlock = generateBlock(blockLambda.body, frame); - if (!results.isEmpty()) { - lambdaBlock += `\nreturn ${results.map((result) => `${BLOCK_LAMBDA_RESULT_PREFIX}${result.name}`).join(', ')}`; - } - return expandToString`def ${frame.getUniqueLambdaBlockName(blockLambda)}(${generateParameters( - blockLambda.parameterList, - frame, - )}):\n${PYTHON_INDENT}${lambdaBlock}`; -}; - -const generateExpression = function (expression: SdsExpression, frame: GenerationInfoFrame): string { - if (isSdsTemplateStringPart(expression)) { - if (isSdsTemplateStringStart(expression)) { - return `${formatStringSingleLine(expression.value)}{ `; - } else if (isSdsTemplateStringInner(expression)) { - return ` }${formatStringSingleLine(expression.value)}{ `; - } else if (isSdsTemplateStringEnd(expression)) { - return ` }${formatStringSingleLine(expression.value)}`; - } - } - - const partiallyEvaluatedNode = frame.services.evaluation.PartialEvaluator.evaluate(expression); - if (partiallyEvaluatedNode instanceof BooleanConstant) { - return partiallyEvaluatedNode.value ? 'True' : 'False'; - } else if (partiallyEvaluatedNode instanceof IntConstant) { - return String(partiallyEvaluatedNode.value); - } else if (partiallyEvaluatedNode instanceof FloatConstant) { - const floatValue = partiallyEvaluatedNode.value; - return Number.isInteger(floatValue) ? `${floatValue}.0` : String(floatValue); - } else if (partiallyEvaluatedNode === NullConstant) { - return 'None'; - } else if (partiallyEvaluatedNode instanceof StringConstant) { - return `'${formatStringSingleLine(partiallyEvaluatedNode.value)}'`; - } - - // Handled after constant expressions: EnumVariant, List, Map - else if (isSdsTemplateString(expression)) { - return `f'${expression.expressions.map((expr) => generateExpression(expr, frame)).join('')}'`; - } else if (isSdsMap(expression)) { - const mapContent = expression.entries.map( - (entry) => `${generateExpression(entry.key, frame)}: ${generateExpression(entry.value, frame)}`, - ); - return `{${mapContent.join(', ')}}`; - } else if (isSdsList(expression)) { - const listContent = expression.elements.map((value) => generateExpression(value, frame)); - return `[${listContent.join(', ')}]`; - } else if (isSdsBlockLambda(expression)) { - return frame.getUniqueLambdaBlockName(expression); - } else if (isSdsCall(expression)) { - const callable = frame.services.helpers.NodeMapper.callToCallable(expression); - if (isSdsFunction(callable)) { - const pythonCall = frame.services.builtins.Annotations.getPythonCall(callable); - if (pythonCall) { - let thisParam: string | undefined = undefined; - if (isSdsMemberAccess(expression.receiver)) { - thisParam = generateExpression(expression.receiver.receiver, frame); - } - const argumentsMap = getArgumentsMap(expression.argumentList.arguments, frame); - return generatePythonCall(pythonCall, argumentsMap, thisParam); - } - } + for (const file of generatedFiles) { + const fsPath = URI.parse(file.uri).fsPath; + const parentDirectoryPath = path.dirname(fsPath); - const sortedArgs = sortArguments(frame.services, expression.argumentList.arguments); - return expandToString`${generateExpression(expression.receiver, frame)}(${sortedArgs - .map((arg) => generateArgument(arg, frame)) - .join(', ')})`; - } else if (isSdsExpressionLambda(expression)) { - return `lambda ${generateParameters(expression.parameterList, frame)}: ${generateExpression( - expression.result, - frame, - )}`; - } else if (isSdsInfixOperation(expression)) { - const leftOperand = generateExpression(expression.leftOperand, frame); - const rightOperand = generateExpression(expression.rightOperand, frame); - switch (expression.operator) { - case 'or': - frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE }); - return `${RUNNER_CODEGEN_PACKAGE}.eager_or(${leftOperand}, ${rightOperand})`; - case 'and': - frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE }); - return `${RUNNER_CODEGEN_PACKAGE}.eager_and(${leftOperand}, ${rightOperand})`; - case '?:': - frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE }); - return `${RUNNER_CODEGEN_PACKAGE}.eager_elvis(${leftOperand}, ${rightOperand})`; - case '===': - return `(${leftOperand}) is (${rightOperand})`; - case '!==': - return `(${leftOperand}) is not (${rightOperand})`; - default: - return `(${leftOperand}) ${expression.operator} (${rightOperand})`; + if (!fs.existsSync(parentDirectoryPath)) { + fs.mkdirSync(parentDirectoryPath, { recursive: true }); } - } else if (isSdsIndexedAccess(expression)) { - return expandToString`${generateExpression(expression.receiver, frame)}[${generateExpression( - expression.index, - frame, - )}]`; - } else if (isSdsMemberAccess(expression)) { - const member = expression.member?.target.ref!; - const receiver = generateExpression(expression.receiver, frame); - if (isSdsEnumVariant(member)) { - const enumMember = generateExpression(expression.member!, frame); - const suffix = isSdsCall(expression.$container) ? '' : '()'; - return `${receiver}.${enumMember}${suffix}`; - } else if (isSdsAbstractResult(member)) { - const resultList = getAbstractResults(getContainerOfType(member, isSdsCallable)); - if (resultList.length === 1) { - return receiver; - } - const currentIndex = resultList.indexOf(member); - return `${receiver}[${currentIndex}]`; - } else { - const memberExpression = generateExpression(expression.member!, frame); - if (expression.isNullSafe) { - frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE }); - return `${RUNNER_CODEGEN_PACKAGE}.safe_access(${receiver}, '${memberExpression}')`; - } else { - return `${receiver}.${memberExpression}`; - } - } - } else if (isSdsParenthesizedExpression(expression)) { - return expandToString`${generateExpression(expression.expression, frame)}`; - } else if (isSdsPrefixOperation(expression)) { - const operand = generateExpression(expression.operand, frame); - switch (expression.operator) { - case 'not': - return expandToString`not (${operand})`; - case '-': - return expandToString`-(${operand})`; - } - } else if (isSdsReference(expression)) { - const declaration = expression.target.ref!; - const referenceImport = - getExternalReferenceNeededImport(frame.services, expression, declaration) || - getInternalReferenceNeededImport(frame.services, expression, declaration); - frame.addImport(referenceImport); - return referenceImport?.alias || getPythonNameOrDefault(frame.services, declaration); - } - /* c8 ignore next 2 */ - throw new Error(`Unknown expression type: ${expression.$type}`); -}; -const generatePythonCall = function ( - pythonCall: string, - argumentsMap: Map, - thisParam: string | undefined = undefined, -): string { - if (thisParam) { - argumentsMap.set('this', thisParam); + fs.writeFileSync(fsPath, file.getText()); } - return pythonCall.replace(/\$[_a-zA-Z][_a-zA-Z0-9]*/gu, (value) => argumentsMap.get(value.substring(1))!); -}; - -const getArgumentsMap = function (argumentList: SdsArgument[], frame: GenerationInfoFrame): Map { - const argumentsMap = new Map(); - argumentList.reduce((map, value) => { - map.set(frame.services.helpers.NodeMapper.argumentToParameter(value)?.name!, generateArgument(value, frame)); - return map; - }, argumentsMap); - return argumentsMap; -}; - -const sortArguments = function (services: SafeDsServices, argumentList: SdsArgument[]): SdsArgument[] { - // $containerIndex contains the index of the parameter in the receivers parameter list - const parameters = argumentList.map((argument) => { - return { par: services.helpers.NodeMapper.argumentToParameter(argument), arg: argument }; - }); - return parameters - .slice() - .filter((value) => value.par !== undefined) - .sort((a, b) => - a.par !== undefined && b.par !== undefined ? a.par.$containerIndex! - b.par.$containerIndex! : 0, - ) - .map((value) => value.arg); -}; - -const generateArgument = function (argument: SdsArgument, frame: GenerationInfoFrame) { - const parameter = frame.services.helpers.NodeMapper.argumentToParameter(argument); - return expandToString`${ - parameter !== undefined && !isRequiredParameter(parameter) - ? generateParameter(parameter, frame, false) + '=' - : '' - }${generateExpression(argument.value, frame)}`; -}; - -const getExternalReferenceNeededImport = function ( - services: SafeDsServices, - expression: SdsExpression, - declaration: SdsDeclaration, -): ImportData | undefined { - // Root Node is always a module. - const currentModule = findRootNode(expression); - const targetModule = findRootNode(declaration); - for (const value of getImports(currentModule)) { - // Verify same package - if (value.package !== targetModule.name) { - continue; - } - if (isSdsQualifiedImport(value)) { - const importedDeclarations = getImportedDeclarations(value); - for (const importedDeclaration of importedDeclarations) { - if (declaration === importedDeclaration.declaration?.ref) { - if (importedDeclaration.alias !== undefined) { - return { - importPath: services.builtins.Annotations.getPythonModule(targetModule) || value.package, - declarationName: importedDeclaration.declaration?.ref?.name, - alias: importedDeclaration.alias.alias, - }; - } else { - return { - importPath: services.builtins.Annotations.getPythonModule(targetModule) || value.package, - declarationName: importedDeclaration.declaration?.ref?.name, - }; - } - } - } - } - if (isSdsWildcardImport(value)) { - return { - importPath: services.builtins.Annotations.getPythonModule(targetModule) || value.package, - declarationName: declaration.name, - }; - } - } - return undefined; -}; - -const getInternalReferenceNeededImport = function ( - services: SafeDsServices, - expression: SdsExpression, - declaration: SdsDeclaration, -): ImportData | undefined { - // Root Node is always a module. - const currentModule = findRootNode(expression); - const targetModule = findRootNode(declaration); - if (currentModule !== targetModule && !isInStubFile(targetModule)) { - return { - importPath: `${ - services.builtins.Annotations.getPythonModule(targetModule) || targetModule.name - }.${formatGeneratedFileName(getModuleFileBaseName(targetModule))}`, - declarationName: getPythonNameOrDefault(services, declaration), - }; - } - return undefined; -}; - -const getModuleFileBaseName = function (module: SdsModule): string { - const filePath = getDocument(module).uri.fsPath; - return path.basename(filePath, path.extname(filePath)); -}; - -const formatStringSingleLine = function (value: string): string { - return value.replaceAll('\r\n', '\\n').replaceAll('\n', '\\n'); + // eslint-disable-next-line no-console + console.log(chalk.green(`Python code generated successfully.`)); }; -interface ImportData { - readonly importPath: string; - readonly declarationName?: string; - readonly alias?: string; -} - -class GenerationInfoFrame { - readonly services: SafeDsServices; - private readonly blockLambdaManager: IdManager; - private readonly importSet: Map; - - constructor(services: SafeDsServices, importSet: Map = new Map()) { - this.services = services; - this.blockLambdaManager = new IdManager(); - this.importSet = importSet; - } - - addImport(importData: ImportData | undefined) { - if (importData) { - const hashKey = JSON.stringify(importData); - if (!this.importSet.has(hashKey)) { - this.importSet.set(hashKey, importData); - } - } - } - - getUniqueLambdaBlockName(lambda: SdsBlockLambda): string { - return `${BLOCK_LAMBDA_PREFIX}${this.blockLambdaManager.assignId(lambda)}`; - } -} +/* c8 ignore stop */ export interface GenerateOptions { destination?: string; diff --git a/src/language/generation/safe-ds-python-generator.ts b/src/language/generation/safe-ds-python-generator.ts new file mode 100644 index 000000000..bbdc3f674 --- /dev/null +++ b/src/language/generation/safe-ds-python-generator.ts @@ -0,0 +1,615 @@ +import { SafeDsServices } from '../safe-ds-module.js'; +import { + isSdsAbstractResult, + isSdsAssignment, + isSdsBlockLambda, + isSdsBlockLambdaResult, + isSdsCall, + isSdsCallable, + isSdsEnumVariant, + isSdsExpressionLambda, + isSdsExpressionStatement, + isSdsFunction, + isSdsIndexedAccess, + isSdsInfixOperation, + isSdsList, + isSdsMap, + isSdsMemberAccess, + isSdsModule, + isSdsParenthesizedExpression, + isSdsPipeline, + isSdsPlaceholder, + isSdsPrefixOperation, + isSdsQualifiedImport, + isSdsReference, + isSdsSegment, + isSdsTemplateString, + isSdsTemplateStringEnd, + isSdsTemplateStringInner, + isSdsTemplateStringPart, + isSdsTemplateStringStart, + isSdsWildcard, + isSdsWildcardImport, + isSdsYield, + SdsArgument, + SdsAssignee, + SdsAssignment, + SdsBlock, + SdsBlockLambda, + SdsDeclaration, + SdsExpression, + SdsModule, + SdsParameter, + SdsParameterList, + SdsPipeline, + SdsSegment, + SdsStatement, +} from '../generated/ast.js'; +import { isInStubFile, isStubFile } from '../helpers/fileExtensions.js'; +import path from 'path'; +import { + expandToString, + expandToStringWithNL, + findRootNode, + getContainerOfType, + getDocument, + LangiumDocument, + streamAllContents, + URI, +} from 'langium'; +import { + getAbstractResults, + getAssignees, + getImportedDeclarations, + getImports, + getModuleMembers, + getStatements, + isRequiredParameter, + streamBlockLambdaResults, +} from '../helpers/nodeProperties.js'; +import { groupBy } from '../../helpers/collectionUtils.js'; +import { + BooleanConstant, + FloatConstant, + IntConstant, + NullConstant, + StringConstant, +} from '../partialEvaluation/model.js'; +import { IdManager } from '../helpers/idManager.js'; +import { TextDocument } from 'vscode-languageserver-textdocument'; +import { SafeDsAnnotations } from '../builtins/safe-ds-annotations.js'; +import { SafeDsNodeMapper } from '../helpers/safe-ds-node-mapper.js'; +import { SafeDsPartialEvaluator } from '../partialEvaluation/safe-ds-partial-evaluator.js'; + +export const CODEGEN_PREFIX = '__gen_'; +const BLOCK_LAMBDA_PREFIX = `${CODEGEN_PREFIX}block_lambda_`; +const BLOCK_LAMBDA_RESULT_PREFIX = `${CODEGEN_PREFIX}block_lambda_result_`; +const YIELD_PREFIX = `${CODEGEN_PREFIX}yield_`; + +const RUNNER_CODEGEN_PACKAGE = 'safeds_runner.codegen'; +const PYTHON_INDENT = ' '; + +export class SafeDsPythonGenerator { + private readonly builtinAnnotations: SafeDsAnnotations; + private readonly nodeMapper: SafeDsNodeMapper; + private readonly partialEvaluator: SafeDsPartialEvaluator; + + constructor(services: SafeDsServices) { + this.builtinAnnotations = services.builtins.Annotations; + this.nodeMapper = services.helpers.NodeMapper; + this.partialEvaluator = services.evaluation.PartialEvaluator; + } + + generate(document: LangiumDocument, destination: string | undefined): TextDocument[] { + const node = document.parseResult.value; + + // Do not generate stub files + if (isStubFile(document) || !isSdsModule(node)) { + return []; + } + + const filePath = document.uri.fsPath; + const data = extractDestinationAndName(filePath, destination); + const pythonModuleName = this.builtinAnnotations.getPythonModule(node); + const packagePath = pythonModuleName === undefined ? node.name.split('.') : [pythonModuleName]; + const parentDirectoryPath = path.join(data.destination, ...packagePath); + + const generatedFiles = new Map(); + generatedFiles.set( + `${path.join(parentDirectoryPath, this.formatGeneratedFileName(data.name))}.py`, + this.generateModule(node), + ); + for (const pipeline of streamAllContents(node).filter(isSdsPipeline)) { + const entryPointFilename = `${path.join( + parentDirectoryPath, + `${this.formatGeneratedFileName(data.name)}_${this.getPythonNameOrDefault(pipeline)}`, + )}.py`; + const entryPointContent = expandToStringWithNL`from ${this.formatGeneratedFileName( + data.name, + )} import ${this.getPythonNameOrDefault( + pipeline, + )}\n\nif __name__ == '__main__':\n${PYTHON_INDENT}${this.getPythonNameOrDefault(pipeline)}()`; + generatedFiles.set(entryPointFilename, entryPointContent); + } + + return Array.from(generatedFiles.entries()).map(([fsPath, content]) => + TextDocument.create(URI.file(fsPath).toString(), 'py', 0, content), + ); + } + + private getPythonNameOrDefault(object: SdsDeclaration) { + return this.builtinAnnotations.getPythonName(object) || object.name; + } + + private formatGeneratedFileName(baseName: string): string { + return `gen_${baseName.replaceAll('%2520', '_').replaceAll(/[ .-]/gu, '_').replaceAll(/\\W/gu, '')}`; + } + + private generateModule(module: SdsModule): string { + const importSet = new Map(); + const segments = getModuleMembers(module) + .filter(isSdsSegment) + .map((segment) => this.generateSegment(segment, importSet)); + const pipelines = getModuleMembers(module) + .filter(isSdsPipeline) + .map((pipeline) => this.generatePipeline(pipeline, importSet)); + const imports = this.generateImports(Array.from(importSet.values())); + const output: string[] = []; + if (imports.length > 0) { + output.push( + expandToStringWithNL`# Imports ----------------------------------------------------------------------\n\n${imports.join( + '\n', + )}`, + ); + } + if (segments.length > 0) { + output.push( + expandToStringWithNL`# Segments ---------------------------------------------------------------------\n\n${segments.join( + '\n\n', + )}`, + ); + } + if (pipelines.length > 0) { + output.push( + expandToStringWithNL`# Pipelines --------------------------------------------------------------------\n\n${pipelines.join( + '\n\n', + )}`, + ); + } + return expandToStringWithNL`${output.join('\n')}`; + } + + private generateSegment(segment: SdsSegment, importSet: Map): string { + const infoFrame = new GenerationInfoFrame(importSet); + const segmentResult = segment.resultList?.results || []; + let segmentBlock = this.generateBlock(segment.body, infoFrame); + if (segmentResult.length !== 0) { + segmentBlock += `\nreturn ${segmentResult.map((result) => `${YIELD_PREFIX}${result.name}`).join(', ')}`; + } + return expandToString`def ${this.getPythonNameOrDefault(segment)}(${this.generateParameters( + segment.parameterList, + infoFrame, + )}):\n${PYTHON_INDENT}${segmentBlock}`; + } + + private generateParameters(parameters: SdsParameterList | undefined, frame: GenerationInfoFrame): string { + const result = (parameters?.parameters || []).map((param) => this.generateParameter(param, frame)); + return result.join(', '); + } + + private generateParameter( + parameter: SdsParameter, + frame: GenerationInfoFrame, + defaultValue: boolean = true, + ): string { + return expandToString`${this.getPythonNameOrDefault(parameter)}${ + defaultValue && parameter.defaultValue !== undefined + ? '=' + this.generateExpression(parameter.defaultValue, frame) + : '' + }`; + } + + private generatePipeline(pipeline: SdsPipeline, importSet: Map): string { + const infoFrame = new GenerationInfoFrame(importSet); + return expandToString`def ${this.getPythonNameOrDefault(pipeline)}():\n${PYTHON_INDENT}${this.generateBlock( + pipeline.body, + infoFrame, + )}`; + } + + private generateImports(importSet: ImportData[]): string[] { + const qualifiedImports = importSet + .filter((importStmt) => importStmt.declarationName === undefined) + .sort((a, b) => a.importPath.localeCompare(b.importPath)) + .map(this.generateQualifiedImport); + const groupedImports = groupBy( + importSet.filter((importStmt) => importStmt.declarationName !== undefined), + (importStmt) => importStmt.importPath, + ) + .toArray() + .sort(([key1, _value1], [key2, _value2]) => key1.localeCompare(key2)); + const declaredImports: string[] = []; + for (const [key, value] of groupedImports) { + const importedDecls = + value + ?.filter((importData) => importData !== undefined) + .sort((a, b) => a.declarationName!.localeCompare(b.declarationName!)) + .map((localValue) => + localValue.alias !== undefined + ? `${localValue.declarationName} as ${localValue.alias}` + : localValue.declarationName!, + ) || []; + declaredImports.push(`from ${key} import ${[...new Set(importedDecls)].join(', ')}`); + } + return [...new Set(qualifiedImports), ...new Set(declaredImports)]; + } + + private generateQualifiedImport(importStmt: ImportData): string { + if (importStmt.alias === undefined) { + return `import ${importStmt.importPath}`; + } else { + /* c8 ignore next 2 */ + return `import ${importStmt.importPath} as ${importStmt.alias}`; + } + } + + private generateBlock(block: SdsBlock, frame: GenerationInfoFrame): string { + // TODO filter withEffect + let statements = getStatements(block); + if (statements.length === 0) { + return 'pass'; + } + return expandToString`${statements.map((stmt) => this.generateStatement(stmt, frame)).join('\n')}`; + } + + private generateStatement(statement: SdsStatement, frame: GenerationInfoFrame): string { + if (isSdsAssignment(statement)) { + return this.generateAssignment(statement, frame); + } else if (isSdsExpressionStatement(statement)) { + const expressionStatement = statement; + const blockLambdaCode: string[] = []; + for (const lambda of streamAllContents(expressionStatement.expression).filter(isSdsBlockLambda)) { + blockLambdaCode.push(this.generateBlockLambda(lambda, frame)); + } + blockLambdaCode.push(this.generateExpression(expressionStatement.expression, frame)); + return expandToString`${blockLambdaCode.join('\n')}`; + } + /* c8 ignore next 2 */ + throw new Error(`Unknown SdsStatement: ${statement}`); + } + + private generateAssignment(assignment: SdsAssignment, frame: GenerationInfoFrame): string { + const requiredAssignees = isSdsCall(assignment.expression) + ? getAbstractResults(this.nodeMapper.callToCallable(assignment.expression)).length + : /* c8 ignore next */ + 1; + const assignees = getAssignees(assignment); + if (assignees.some((value) => !isSdsWildcard(value))) { + const actualAssignees = assignees.map(this.generateAssignee); + if (requiredAssignees === actualAssignees.length) { + return `${actualAssignees.join(', ')} = ${this.generateExpression(assignment.expression!, frame)}`; + } else { + // Add wildcards to match given results + return `${actualAssignees + .concat(Array(requiredAssignees - actualAssignees.length).fill('_')) + .join(', ')} = ${this.generateExpression(assignment.expression!, frame)}`; + } + } else { + return this.generateExpression(assignment.expression!, frame); + } + } + + private generateAssignee(assignee: SdsAssignee): string { + if (isSdsBlockLambdaResult(assignee)) { + return `${BLOCK_LAMBDA_RESULT_PREFIX}${assignee.name}`; + } else if (isSdsPlaceholder(assignee)) { + return assignee.name; + } else if (isSdsWildcard(assignee)) { + return '_'; + } else if (isSdsYield(assignee)) { + return `${YIELD_PREFIX}${assignee.result?.ref?.name!}`; + } + /* c8 ignore next 2 */ + throw new Error(`Unknown SdsAssignment: ${assignee.$type}`); + } + + private generateBlockLambda(blockLambda: SdsBlockLambda, frame: GenerationInfoFrame): string { + const results = streamBlockLambdaResults(blockLambda); + let lambdaBlock = this.generateBlock(blockLambda.body, frame); + if (!results.isEmpty()) { + lambdaBlock += `\nreturn ${results + .map((result) => `${BLOCK_LAMBDA_RESULT_PREFIX}${result.name}`) + .join(', ')}`; + } + return expandToString`def ${frame.getUniqueLambdaBlockName(blockLambda)}(${this.generateParameters( + blockLambda.parameterList, + frame, + )}):\n${PYTHON_INDENT}${lambdaBlock}`; + } + + private generateExpression(expression: SdsExpression, frame: GenerationInfoFrame): string { + if (isSdsTemplateStringPart(expression)) { + if (isSdsTemplateStringStart(expression)) { + return `${this.formatStringSingleLine(expression.value)}{ `; + } else if (isSdsTemplateStringInner(expression)) { + return ` }${this.formatStringSingleLine(expression.value)}{ `; + } else if (isSdsTemplateStringEnd(expression)) { + return ` }${this.formatStringSingleLine(expression.value)}`; + } + } + + const partiallyEvaluatedNode = this.partialEvaluator.evaluate(expression); + if (partiallyEvaluatedNode instanceof BooleanConstant) { + return partiallyEvaluatedNode.value ? 'True' : 'False'; + } else if (partiallyEvaluatedNode instanceof IntConstant) { + return String(partiallyEvaluatedNode.value); + } else if (partiallyEvaluatedNode instanceof FloatConstant) { + const floatValue = partiallyEvaluatedNode.value; + return Number.isInteger(floatValue) ? `${floatValue}.0` : String(floatValue); + } else if (partiallyEvaluatedNode === NullConstant) { + return 'None'; + } else if (partiallyEvaluatedNode instanceof StringConstant) { + return `'${this.formatStringSingleLine(partiallyEvaluatedNode.value)}'`; + } + + // Handled after constant expressions: EnumVariant, List, Map + else if (isSdsTemplateString(expression)) { + return `f'${expression.expressions.map((expr) => this.generateExpression(expr, frame)).join('')}'`; + } else if (isSdsMap(expression)) { + const mapContent = expression.entries.map( + (entry) => + `${this.generateExpression(entry.key, frame)}: ${this.generateExpression(entry.value, frame)}`, + ); + return `{${mapContent.join(', ')}}`; + } else if (isSdsList(expression)) { + const listContent = expression.elements.map((value) => this.generateExpression(value, frame)); + return `[${listContent.join(', ')}]`; + } else if (isSdsBlockLambda(expression)) { + return frame.getUniqueLambdaBlockName(expression); + } else if (isSdsCall(expression)) { + const callable = this.nodeMapper.callToCallable(expression); + if (isSdsFunction(callable)) { + const pythonCall = this.builtinAnnotations.getPythonCall(callable); + if (pythonCall) { + let thisParam: string | undefined = undefined; + if (isSdsMemberAccess(expression.receiver)) { + thisParam = this.generateExpression(expression.receiver.receiver, frame); + } + const argumentsMap = this.getArgumentsMap(expression.argumentList.arguments, frame); + return this.generatePythonCall(pythonCall, argumentsMap, thisParam); + } + } + + const sortedArgs = this.sortArguments(expression.argumentList.arguments); + return expandToString`${this.generateExpression(expression.receiver, frame)}(${sortedArgs + .map((arg) => this.generateArgument(arg, frame)) + .join(', ')})`; + } else if (isSdsExpressionLambda(expression)) { + return `lambda ${this.generateParameters(expression.parameterList, frame)}: ${this.generateExpression( + expression.result, + frame, + )}`; + } else if (isSdsInfixOperation(expression)) { + const leftOperand = this.generateExpression(expression.leftOperand, frame); + const rightOperand = this.generateExpression(expression.rightOperand, frame); + switch (expression.operator) { + case 'or': + frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE }); + return `${RUNNER_CODEGEN_PACKAGE}.eager_or(${leftOperand}, ${rightOperand})`; + case 'and': + frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE }); + return `${RUNNER_CODEGEN_PACKAGE}.eager_and(${leftOperand}, ${rightOperand})`; + case '?:': + frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE }); + return `${RUNNER_CODEGEN_PACKAGE}.eager_elvis(${leftOperand}, ${rightOperand})`; + case '===': + return `(${leftOperand}) is (${rightOperand})`; + case '!==': + return `(${leftOperand}) is not (${rightOperand})`; + default: + return `(${leftOperand}) ${expression.operator} (${rightOperand})`; + } + } else if (isSdsIndexedAccess(expression)) { + return expandToString`${this.generateExpression(expression.receiver, frame)}[${this.generateExpression( + expression.index, + frame, + )}]`; + } else if (isSdsMemberAccess(expression)) { + const member = expression.member?.target.ref!; + const receiver = this.generateExpression(expression.receiver, frame); + if (isSdsEnumVariant(member)) { + const enumMember = this.generateExpression(expression.member!, frame); + const suffix = isSdsCall(expression.$container) ? '' : '()'; + return `${receiver}.${enumMember}${suffix}`; + } else if (isSdsAbstractResult(member)) { + const resultList = getAbstractResults(getContainerOfType(member, isSdsCallable)); + if (resultList.length === 1) { + return receiver; + } + const currentIndex = resultList.indexOf(member); + return `${receiver}[${currentIndex}]`; + } else { + const memberExpression = this.generateExpression(expression.member!, frame); + if (expression.isNullSafe) { + frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE }); + return `${RUNNER_CODEGEN_PACKAGE}.safe_access(${receiver}, '${memberExpression}')`; + } else { + return `${receiver}.${memberExpression}`; + } + } + } else if (isSdsParenthesizedExpression(expression)) { + return expandToString`${this.generateExpression(expression.expression, frame)}`; + } else if (isSdsPrefixOperation(expression)) { + const operand = this.generateExpression(expression.operand, frame); + switch (expression.operator) { + case 'not': + return expandToString`not (${operand})`; + case '-': + return expandToString`-(${operand})`; + } + } else if (isSdsReference(expression)) { + const declaration = expression.target.ref!; + const referenceImport = + this.getExternalReferenceNeededImport(expression, declaration) || + this.getInternalReferenceNeededImport(expression, declaration); + frame.addImport(referenceImport); + return referenceImport?.alias || this.getPythonNameOrDefault(declaration); + } + /* c8 ignore next 2 */ + throw new Error(`Unknown expression type: ${expression.$type}`); + } + + private generatePythonCall( + pythonCall: string, + argumentsMap: Map, + thisParam: string | undefined = undefined, + ): string { + if (thisParam) { + argumentsMap.set('this', thisParam); + } + + return pythonCall.replace(/\$[_a-zA-Z][_a-zA-Z0-9]*/gu, (value) => argumentsMap.get(value.substring(1))!); + } + + private getArgumentsMap(argumentList: SdsArgument[], frame: GenerationInfoFrame): Map { + const argumentsMap = new Map(); + argumentList.reduce((map, value) => { + map.set(this.nodeMapper.argumentToParameter(value)?.name!, this.generateArgument(value, frame)); + return map; + }, argumentsMap); + return argumentsMap; + } + + private sortArguments(argumentList: SdsArgument[]): SdsArgument[] { + // $containerIndex contains the index of the parameter in the receivers parameter list + const parameters = argumentList.map((argument) => { + return { par: this.nodeMapper.argumentToParameter(argument), arg: argument }; + }); + return parameters + .slice() + .filter((value) => value.par !== undefined) + .sort((a, b) => + a.par !== undefined && b.par !== undefined ? a.par.$containerIndex! - b.par.$containerIndex! : 0, + ) + .map((value) => value.arg); + } + + private generateArgument(argument: SdsArgument, frame: GenerationInfoFrame) { + const parameter = this.nodeMapper.argumentToParameter(argument); + return expandToString`${ + parameter !== undefined && !isRequiredParameter(parameter) + ? this.generateParameter(parameter, frame, false) + '=' + : '' + }${this.generateExpression(argument.value, frame)}`; + } + + private getExternalReferenceNeededImport( + expression: SdsExpression, + declaration: SdsDeclaration, + ): ImportData | undefined { + // Root Node is always a module. + const currentModule = findRootNode(expression); + const targetModule = findRootNode(declaration); + for (const value of getImports(currentModule)) { + // Verify same package + if (value.package !== targetModule.name) { + continue; + } + if (isSdsQualifiedImport(value)) { + const importedDeclarations = getImportedDeclarations(value); + for (const importedDeclaration of importedDeclarations) { + if (declaration === importedDeclaration.declaration?.ref) { + if (importedDeclaration.alias !== undefined) { + return { + importPath: this.builtinAnnotations.getPythonModule(targetModule) || value.package, + declarationName: importedDeclaration.declaration?.ref?.name, + alias: importedDeclaration.alias.alias, + }; + } else { + return { + importPath: this.builtinAnnotations.getPythonModule(targetModule) || value.package, + declarationName: importedDeclaration.declaration?.ref?.name, + }; + } + } + } + } + if (isSdsWildcardImport(value)) { + return { + importPath: this.builtinAnnotations.getPythonModule(targetModule) || value.package, + declarationName: declaration.name, + }; + } + } + return undefined; + } + + private getInternalReferenceNeededImport( + expression: SdsExpression, + declaration: SdsDeclaration, + ): ImportData | undefined { + // Root Node is always a module. + const currentModule = findRootNode(expression); + const targetModule = findRootNode(declaration); + if (currentModule !== targetModule && !isInStubFile(targetModule)) { + return { + importPath: `${ + this.builtinAnnotations.getPythonModule(targetModule) || targetModule.name + }.${this.formatGeneratedFileName(this.getModuleFileBaseName(targetModule))}`, + declarationName: this.getPythonNameOrDefault(declaration), + }; + } + return undefined; + } + + private getModuleFileBaseName(module: SdsModule): string { + const filePath = getDocument(module).uri.fsPath; + return path.basename(filePath, path.extname(filePath)); + } + + private formatStringSingleLine(value: string): string { + return value.replaceAll('\r\n', '\\n').replaceAll('\n', '\\n'); + } +} + +interface ImportData { + readonly importPath: string; + readonly declarationName?: string; + readonly alias?: string; +} + +class GenerationInfoFrame { + private readonly blockLambdaManager: IdManager; + private readonly importSet: Map; + + constructor(importSet: Map = new Map()) { + this.blockLambdaManager = new IdManager(); + this.importSet = importSet; + } + + addImport(importData: ImportData | undefined) { + if (importData) { + const hashKey = JSON.stringify(importData); + if (!this.importSet.has(hashKey)) { + this.importSet.set(hashKey, importData); + } + } + } + + getUniqueLambdaBlockName(lambda: SdsBlockLambda): string { + return `${BLOCK_LAMBDA_PREFIX}${this.blockLambdaManager.assignId(lambda)}`; + } +} + +interface FilePathData { + destination: string; + name: string; +} + +const extractDestinationAndName = function (filePath: string, destination: string | undefined): FilePathData { + const baseFilePath = path.basename(filePath, path.extname(filePath)).replace(/[.-]/gu, ''); + return { + destination: destination ?? path.join(path.dirname(baseFilePath), 'generated'), + name: path.basename(baseFilePath), + }; +}; diff --git a/src/language/safe-ds-module.ts b/src/language/safe-ds-module.ts index 58a156fc7..b37c41e9f 100644 --- a/src/language/safe-ds-module.ts +++ b/src/language/safe-ds-module.ts @@ -33,6 +33,7 @@ import { SafeDsEnums } from './builtins/safe-ds-enums.js'; import { SafeDsInlayHintProvider } from './lsp/safe-ds-inlay-hint-provider.js'; import { SafeDsCommentProvider } from './documentation/safe-ds-comment-provider.js'; import { SafeDsDocumentationProvider } from './documentation/safe-ds-documentation-provider.js'; +import { SafeDsPythonGenerator } from './generation/safe-ds-python-generator.js'; /** * Declaration of custom services - add your own service classes here. @@ -46,6 +47,9 @@ export type SafeDsAddedServices = { evaluation: { PartialEvaluator: SafeDsPartialEvaluator; }; + generation: { + PythonGenerator: SafeDsPythonGenerator; + }; helpers: { NodeMapper: SafeDsNodeMapper; }; @@ -84,6 +88,9 @@ export const SafeDsModule: Module new SafeDsPartialEvaluator(services), }, + generation: { + PythonGenerator: (services) => new SafeDsPythonGenerator(services), + }, helpers: { NodeMapper: (services) => new SafeDsNodeMapper(services), }, diff --git a/src/language/validation/names.ts b/src/language/validation/names.ts index d6873ee45..5e373f27b 100644 --- a/src/language/validation/names.ts +++ b/src/language/validation/names.ts @@ -42,8 +42,8 @@ import { isInPipelineFile, isInStubFile, isInTestFile } from '../helpers/fileExt import { declarationIsAllowedInPipelineFile, declarationIsAllowedInStubFile } from './other/modules.js'; import { SafeDsServices } from '../safe-ds-module.js'; import { listBuiltinFiles } from '../builtins/fileFinder.js'; -import { CODEGEN_PREFIX } from '../../cli/generator.js'; import { BUILTINS_ROOT_PACKAGE } from '../builtins/packageNames.js'; +import { CODEGEN_PREFIX } from '../generation/safe-ds-python-generator.js'; export const CODE_NAME_CODEGEN_PREFIX = 'name/codegen-prefix'; export const CODE_NAME_CASING = 'name/casing'; diff --git a/tests/language/generation/testGeneration.test.ts b/tests/language/generation/testGeneration.test.ts index 79ba59b58..2b6e6393e 100644 --- a/tests/language/generation/testGeneration.test.ts +++ b/tests/language/generation/testGeneration.test.ts @@ -3,12 +3,11 @@ import { clearDocuments } from 'langium/test'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { NodeFileSystem } from 'langium/node'; import { createGenerationTests } from './creator.js'; -import { SdsModule } from '../../../src/language/generated/ast.js'; -import { generatePython } from '../../../src/cli/generator.js'; -import fs from 'fs'; import { loadDocuments } from '../../helpers/testResources.js'; +import { stream } from 'langium'; const services = createSafeDsServices(NodeFileSystem).SafeDs; +const pythonGenerator = services.generation.PythonGenerator; const generationTests = createGenerationTests(); describe('generation', async () => { @@ -31,26 +30,23 @@ describe('generation', async () => { const documents = await loadDocuments(services, test.inputUris); // Generate code for all documents - const actualOutputPaths: string[] = []; - - for (const document of documents) { - const module = document.parseResult.value as SdsModule; - const fileName = document.uri.fsPath; - const generatedFilePaths = generatePython(services, module, fileName, test.actualOutputRoot.fsPath); - actualOutputPaths.push(...generatedFilePaths); - } + const actualOutputs = stream(documents) + .flatMap((document) => pythonGenerator.generate(document, test.actualOutputRoot.fsPath)) + .map((textDocument) => [textDocument.uri, textDocument.getText()]) + .toMap( + (entry) => entry[0], + (entry) => entry[1], + ); // File paths must match - const expectedOutputPaths = test.expectedOutputFiles.map((file) => file.uri.fsPath).sort(); - expect(actualOutputPaths.sort()).toStrictEqual(expectedOutputPaths); + const actualOutputPaths = Array.from(actualOutputs.keys()).sort(); + const expectedOutputPaths = test.expectedOutputFiles.map((file) => file.uri.toString()).sort(); + expect(actualOutputPaths).toStrictEqual(expectedOutputPaths); // File contents must match for (const expectedOutputFile of test.expectedOutputFiles) { - const actualCode = fs.readFileSync(expectedOutputFile.uri.fsPath).toString(); + const actualCode = actualOutputs.get(expectedOutputFile.uri.toString()); expect(actualCode).toBe(expectedOutputFile.code); } - - // Remove generated files (if the test fails, the files are kept for debugging) - fs.rmSync(test.actualOutputRoot.fsPath, { recursive: true, force: true }); }); });