diff --git a/README.md b/README.md index d7f51d8..14f6830 100644 --- a/README.md +++ b/README.md @@ -61,15 +61,18 @@ type ParserOptions = { // like undefined functions and variables. If `failOnWarn` is set to true, // warnings will still cause the parser to raise an error. Defaults to false. quiet: boolean; + // An optional string representing the origin of the GLSL, for debugging and // error messages. For example, "main.js". If the parser raises an error, the // grammarSource shows up in the error.source field. If you format the error // (see the errors section), the grammarSource shows up in the formatted error // string. Defaults to undefined. grammarSource: string; + // If true, sets location information on each AST node, in the form of // { column: number, line: number, offset: number }. Defaults to false. includeLocation: boolean; + // If true, causes the parser to raise an error instead of log a warning. // The parser does limited type checking, and things like undeclared variables // are treated as warnings. Defaults to false. @@ -427,19 +430,86 @@ visitors. ### Utility Functions -Rename all the variables in a program: +#### Rename variables / identifiers in a program + +You can rename bindings (aka variables), functions, and types (aka structs) with `renameBindings`, `renameFunctions`, and `renameTypes` respectively. + +The signature for these methods: + +```ts +const renameBindings = ( + // The scope to rename the bindings in. ast.scopes[0] is the global scope. + // Passing this ast.scopes[0] renames all global variables + bindings: ScopeIndex, + + // The rename function. This is called once per scope entry with the original + // name in the scope, to generate the renamed variable. + mangle: (name: string) => string +): ScopeIndex +``` + +These scope renaming functions, `renameBindings`, `renameFunctions`, and `renameTypes`, do two things: +1. Each function *mutates* the AST to rename identifiers in place. +2. They *return* an *immutable* new ScopeIndex where the scope references + themselves are renamed. + +If you want your ast.scopes array to stay in sync with your AST, you need to +re-assign it to the output of the functions! Examples: ```typescript import { renameBindings, renameFunctions, renameTypes } from '@shaderfrog/glsl-parser/utils'; -// ... parse an ast... +// Suffix top level variables with _x, and update the scope +ast.scopes[0].bindings = renameBindings(ast.scopes[0].bindings, (name) => `${name}_x`); -// Suffix top level variables with _x -renameBindings(ast.scopes[0], (name, node) => `${name}_x`); // Suffix function names with _x -renameFunctions(ast.scopes[0], (name, node) => `${name}_x`); +ast.scopes[0].functions = renameFunctions(ast.scopes[0].functions, (name) => `${name}_x`); + // Suffix struct names and usages (including constructors) with _x -renameTypes(ast.scopes[0], (name, node) => `${name}_x`); +ast.scopes[0].types = renameTypes(ast.scopes[0].types, (name) => `${name}_x`); +``` + +There are also functions to rename only one variable/identifier in a given +scope. Use these if you know specifically which variable you want to rename. + +```typescript +import { renameBinding, renameFunction, renameType } from '@shaderfrog/glsl-parser/utils'; + +// Replace all instances of "oldVar" with "newVar" in the global scope, and +// creates a new global scope entry named "newVar" +ast.scopes[0].bindings.newVar = renameBinding( + ast.scopes[0].bindings.oldVar, + 'newVar', +); +// You need to manually delete the old scope entry if you want the scope to stay +// in sync with your program AST +delete ast.scopes[0].bindings.oldVar; + +// Rename a specific function +ast.scopes[0].functions.newFn = renameFunction( + ast.scopes[0].functions.oldFn, + 'newFn', +); +delete ast.scopes[0].functions.oldFn; + +// Rename a specific type/struct +ast.scopes[0].functions.newType = renametype( + ast.scopes[0].functions.oldType, + 'newType', +); +delete ast.scopes[0].functions.oldType; +``` + +#### Debugging utility functions + +The parser also exports a debugging function, useful for logging information +about the AST. + +```ts +import { debugScopes } from '@shaderfrog/glsl-parser/parser/utils'; + +// Print a condensed representation of the AST scopes to the console +debugScopes(ast); ``` ## What are "parsing" and "preprocessing"? diff --git a/package-lock.json b/package-lock.json index 9bd114c..06580cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@shaderfrog/glsl-parser", - "version": "3.0.0", + "version": "5.0.0-beta.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@shaderfrog/glsl-parser", - "version": "3.0.0", + "version": "5.0.0-beta.4", "license": "ISC", "devDependencies": { "@babel/core": "^7.15.5", @@ -21,7 +21,7 @@ "prettier": "^2.1.2", "ts-jest": "^29.1.2", "ts-jest-resolver": "^2.0.1", - "typescript": "^5.3.3" + "typescript": "^5.5.3" }, "engines": { "node": ">=16" @@ -6308,9 +6308,9 @@ } }, "node_modules/typescript": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", - "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", + "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -11132,9 +11132,9 @@ "dev": true }, "typescript": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", - "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", + "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", "dev": true }, "unicode-canonical-property-names-ecmascript": { diff --git a/package.json b/package.json index d051124..d8febdc 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "engines": { "node": ">=16" }, - "version": "4.1.1", + "version": "5.0.0", "type": "module", "description": "A GLSL ES 1.0 and 3.0 parser and preprocessor that can preserve whitespace and comments", "scripts": { @@ -49,6 +49,6 @@ "prettier": "^2.1.2", "ts-jest": "^29.1.2", "ts-jest-resolver": "^2.0.1", - "typescript": "^5.3.3" + "typescript": "^5.5.3" } } diff --git a/src/parser/scope.test.ts b/src/parser/scope.test.ts index ed0cc56..6499050 100644 --- a/src/parser/scope.test.ts +++ b/src/parser/scope.test.ts @@ -176,7 +176,10 @@ float fn() { `); expect(ast.scopes[0].functions.noise); - renameFunctions(ast.scopes[0], (name) => `${name}_FUNCTION`); + ast.scopes[0].functions = renameFunctions( + ast.scopes[0].functions, + (name) => `${name}_FUNCTION` + ); expect(generate(ast)).toBe(` float noise_FUNCTION() {} float fn_FUNCTION() { @@ -233,8 +236,14 @@ vec4 linearToOutputTexel( vec4 value ) { return LinearToLinear( value ); } { quiet: true } ); - renameBindings(ast.scopes[0], (name) => `${name}_VARIABLE`); - renameFunctions(ast.scopes[0], (name) => `${name}_FUNCTION`); + ast.scopes[0].bindings = renameBindings( + ast.scopes[0].bindings, + (name) => `${name}_VARIABLE` + ); + ast.scopes[0].functions = renameFunctions( + ast.scopes[0].functions, + (name) => `${name}_FUNCTION` + ); expect(generate(ast)).toBe(` float selfref_VARIABLE, b_VARIABLE = 1.0, c_VARIABLE = selfref_VARIABLE; @@ -306,7 +315,8 @@ StructName main(in StructName x, StructName[3] y) { float a2 = 1.0 + StructName(1.0).color.x; } `); - renameTypes(ast.scopes[0], (name) => `${name}_x`); + ast.scopes[0].types = renameTypes(ast.scopes[0].types, (name) => `${name}_x`); + expect(generate(ast)).toBe(` struct StructName_x { vec3 color; @@ -342,10 +352,10 @@ StructName_x main(in StructName_x x, StructName_x[3] y) { ]); expect(Object.keys(ast.scopes[0].bindings)).toEqual(['reflectedLight']); expect(Object.keys(ast.scopes[0].types)).toEqual([ - 'StructName', - 'OtherStruct', + 'StructName_x', + 'OtherStruct_x', ]); - expect(ast.scopes[0].types.StructName.references).toHaveLength(16); + expect(ast.scopes[0].types.StructName_x.references).toHaveLength(16); // Inner struct definition should be found in inner fn scope expect(Object.keys(ast.scopes[2].types)).toEqual(['StructName']); @@ -357,7 +367,7 @@ attribute vec3 position; vec3 func(vec3 position) { return position; }`); - renameBindings(ast.scopes[0], (name) => + ast.scopes[0].bindings = renameBindings(ast.scopes[0].bindings, (name) => name === 'position' ? 'renamed' : name ); // The func arg "position" shadows the global binding, it should be untouched @@ -378,7 +388,10 @@ uniform vec2 vProp; };`); // This shouldn't crash - see the comment block in renameBindings() - renameBindings(ast.scopes[0], (name) => `${name}_x`); + ast.scopes[0].bindings = renameBindings( + ast.scopes[0].bindings, + (name) => `${name}_x` + ); expect(generate(ast)).toBe(` layout(std140,column_major) uniform; float a_x; diff --git a/src/parser/scope.ts b/src/parser/scope.ts index 15b0bd5..87b9b87 100644 --- a/src/parser/scope.ts +++ b/src/parser/scope.ts @@ -118,7 +118,6 @@ export const functionDeclarationSignature = ( const quantifiers = specifier.quantifier || []; const parameterTypes = proto?.parameters?.map(({ specifier, quantifier }) => { - // todo: saving place on putting quantifiers here const quantifiers = // vec4[1][2] param specifier.quantifier || diff --git a/src/parser/test-helpers.ts b/src/parser/test-helpers.ts index a3e2309..5a8615a 100644 --- a/src/parser/test-helpers.ts +++ b/src/parser/test-helpers.ts @@ -67,37 +67,6 @@ export const buildParser = () => { // } // }; -export const debugEntry = (bindings: ScopeIndex) => { - return Object.entries(bindings).map( - ([k, v]) => - `${k}: (${v.references.length} references, ${ - v.declaration ? '' : 'un' - }declared): ${v.references.map((r) => r.type).join(', ')}` - ); -}; -export const debugFunctionEntry = (bindings: FunctionScopeIndex) => - Object.entries(bindings).flatMap(([name, overloads]) => - Object.entries(overloads).map( - ([signature, overload]) => - `${name} (${signature}): (${overload.references.length} references, ${ - overload.declaration ? '' : 'un' - }declared): ${overload.references.map((r) => r.type).join(', ')}` - ) - ); - -export const debugScopes = (astOrScopes: Program | Scope[]) => - console.log( - 'Scopes:', - 'scopes' in astOrScopes - ? astOrScopes.scopes - : astOrScopes.map((s) => ({ - name: s.name, - types: debugEntry(s.types), - bindings: debugEntry(s.bindings), - functions: debugFunctionEntry(s.functions), - })) - ); - const middle = /\/\* start \*\/((.|[\r\n])+)(\/\* end \*\/)?/m; type ParseSrc = (src: string, options?: ParserOptions) => Program; diff --git a/src/parser/utils.ts b/src/parser/utils.ts index 78438b7..86a9c62 100644 --- a/src/parser/utils.ts +++ b/src/parser/utils.ts @@ -1,96 +1,158 @@ -import type { AstNode } from '../ast/index.js'; -import { Scope } from './scope.js'; +import { Program } from '../ast/ast-types.js'; +import { + FunctionOverloadIndex, + FunctionScopeIndex, + Scope, + ScopeEntry, + ScopeIndex, + TypeScopeEntry, + TypeScopeIndex, +} from './scope.js'; -export const renameBindings = ( - scope: Scope, - mangle: (name: string, node: AstNode) => string -) => { - Object.entries(scope.bindings).forEach(([name, binding]) => { - binding.references.forEach((node) => { - if (node.type === 'declaration') { - node.identifier.identifier = mangle(node.identifier.identifier, node); - } else if (node.type === 'identifier') { - node.identifier = mangle(node.identifier, node); - } else if (node.type === 'parameter_declaration' && node.identifier) { - node.identifier.identifier = mangle(node.identifier.identifier, node); - /* Ignore case of: +export const renameBinding = (binding: ScopeEntry, newName: string) => { + binding.references.forEach((node) => { + if (node.type === 'declaration') { + node.identifier.identifier = newName; + } else if (node.type === 'identifier') { + node.identifier = newName; + } else if (node.type === 'parameter_declaration' && node.identifier) { + node.identifier.identifier = newName; + /* Ignore case of: layout(std140,column_major) uniform; - uniform Material - { - uniform vec2 prop; - } - */ - } else if (node.type !== 'interface_declarator') { - console.warn('Unknown binding node', node); - throw new Error(`Binding for type ${node.type} not recognized`); - } - }); + uniform Material { + uniform vec2 prop; + } + */ + } else if (node.type !== 'interface_declarator') { + console.warn('Unknown binding node', node); + throw new Error(`Binding for type ${node.type} not recognized`); + } }); + return binding; }; -export const renameTypes = ( - scope: Scope, - mangle: (name: string, node: AstNode) => string -) => { - Object.entries(scope.types).forEach(([name, type]) => { - type.references.forEach((node) => { - if (node.type === 'type_name') { - node.identifier = mangle(node.identifier, node); - } else { - console.warn('Unknown type node', node); - throw new Error(`Type ${node.type} not recognized`); - } - }); +export const renameBindings = ( + bindings: ScopeIndex, + mangle: (name: string) => string +) => + Object.entries(bindings).reduce((acc, [name, binding]) => { + const mangled = mangle(name); + return { + ...acc, + [mangled]: renameBinding(binding, mangled), + }; + }, {}); + +export const renameType = (type: TypeScopeEntry, newName: string) => { + type.references.forEach((node) => { + if (node.type === 'type_name') { + node.identifier = newName; + } else { + console.warn('Unknown type node', node); + throw new Error(`Type ${node.type} not recognized`); + } }); + return type; }; -export const renameFunctions = ( - scope: Scope, - mangle: (name: string, node: AstNode) => string +export const renameTypes = ( + types: TypeScopeIndex, + mangle: (name: string) => string +) => + Object.entries(types).reduce((acc, [name, type]) => { + const mangled = mangle(name); + return { + ...acc, + [mangled]: renameType(type, mangled), + }; + }, {}); + +export const renameFunction = ( + overloadIndex: FunctionOverloadIndex, + newName: string ) => { - Object.entries(scope.functions).forEach(([fnName, overloads]) => { - Object.entries(overloads).forEach(([signature, overload]) => { - overload.references.forEach((node) => { - if (node.type === 'function') { - node['prototype'].header.name.identifier = mangle( - node['prototype'].header.name.identifier, - node - ); - } else if ( - node.type === 'function_call' && - node.identifier.type === 'postfix' - ) { - // @ts-ignore - const specifier = node.identifier.expression.identifier.specifier; - if (specifier) { - specifier.identifier = mangle(specifier.identifier, node); - } else { - console.warn('Unknown function node to rename', node); - throw new Error( - `Function specifier type ${node.type} not recognized` - ); - } - } else if ( - node.type === 'function_call' && - 'specifier' in node.identifier && - 'identifier' in node.identifier.specifier - ) { - node.identifier.specifier.identifier = mangle( - node.identifier.specifier.identifier, - node - ); - } else if ( - node.type === 'function_call' && - node.identifier.type === 'identifier' - ) { - node.identifier.identifier = mangle(node.identifier.identifier, node); + Object.entries(overloadIndex).forEach(([signature, overload]) => { + overload.references.forEach((node) => { + if (node.type === 'function') { + node['prototype'].header.name.identifier = newName; + } else if ( + node.type === 'function_call' && + node.identifier.type === 'postfix' + ) { + // @ts-ignore + const specifier = node.identifier.expression.identifier.specifier; + if (specifier) { + specifier.identifier = newName; } else { console.warn('Unknown function node to rename', node); - throw new Error(`Function for type ${node.type} not recognized`); + throw new Error( + `Function specifier type ${node.type} not recognized` + ); } - }); + } else if ( + node.type === 'function_call' && + 'specifier' in node.identifier && + 'identifier' in node.identifier.specifier + ) { + node.identifier.specifier.identifier = newName; + } else if ( + node.type === 'function_call' && + node.identifier.type === 'identifier' + ) { + node.identifier.identifier = newName; + } else { + console.warn('Unknown function node to rename', node); + throw new Error(`Function for type ${node.type} not recognized`); + } }); }); + return overloadIndex; }; +export const renameFunctions = ( + functions: FunctionScopeIndex, + mangle: (name: string) => string +) => + Object.entries(functions).reduce( + (acc, [fnName, overloads]) => { + const mangled = mangle(fnName); + return { + ...acc, + [mangled]: renameFunction(overloads, mangled), + }; + }, + {} + ); + export const xor = (a: any, b: any): boolean => (a || b) && !(a && b); + +export const debugEntry = (bindings: ScopeIndex) => { + return Object.entries(bindings).map( + ([k, v]) => + `${k}: (${v.references.length} references, ${ + v.declaration ? '' : 'un' + }declared): ${v.references.map((r) => r.type).join(', ')}` + ); +}; +export const debugFunctionEntry = (bindings: FunctionScopeIndex) => + Object.entries(bindings).flatMap(([name, overloads]) => + Object.entries(overloads).map( + ([signature, overload]) => + `${name} (${signature}): (${overload.references.length} references, ${ + overload.declaration ? '' : 'un' + }declared): ${overload.references.map((r) => r.type).join(', ')}` + ) + ); + +export const debugScopes = (astOrScopes: Program | Scope[]) => + console.log( + 'Scopes:', + 'scopes' in astOrScopes + ? astOrScopes.scopes + : astOrScopes.map((s) => ({ + name: s.name, + types: debugEntry(s.types), + bindings: debugEntry(s.bindings), + functions: debugFunctionEntry(s.functions), + })) + );