Add in parse tree testing for every grammar, every input file

.gitattributes

@@ -6,7 +6,6 @@
 # Explicitly declare text files you want to always be normalized and converted
 # to native line endings on checkout.
 *.abnf text
-*.agc text
 *.alg text
 *.asm text
 *.asn text
@@ -87,33 +86,129 @@ json/examples/*.json text
 *.yml text
 access_log text
 
-# Declare files that will always have LF line endings on checkout.
-# Unicode input files end with *.txt, but should not be converted on Windows
-unicode/graphemes/examples/*.txt eol=lf
-# T-SQL tests with trees can fail with multiline strings
-sql/tsql/examples/*.sql eol=lf
-# Rexx places EOL on default channel, so the .tree's contain EOL.
-# Make sure EOL's are consistent regardless of OS.
-rexx/examples/* eol=lf
 # All grammars that contain parser rules for token NEWLINE must have
 # standard Unix endings.
+# vb6/examples/**/*.vb eol=lf
 abb/examples/*.sys eol=lf
+agc/examples/**/* eol=lf
+antlr/antlr2/examples/**/*.g2 eol=lf
+antlr/antlr3/examples/**/*.g eol=lf
+antlr/antlr4/examples/**/*.g4 eol=lf
+arden/examples/**/*.mlm eol=lf
 asl/examples/* eol=lf
+asm/asm6502/examples/* eol=lf
+asm/asm8080/examples/* eol=lf
+asm/asm8086/examples/* eol=lf
+asm/asmMASM/examples/**/*.asm eol=lf
+asm/asmRISCV/examples/**/*.S eol=lf
+asm/asmZ80/examples/*.z80 eol=lf
+asm/nasm/examples/**/*.asm eol=lf
+asm/pdp7/examples/**/*.s eol=lf
+asm/ptx/ptx-isa-2.1/examples/**/*.ptx eol=lf
+asm/ptx/ptx-isa-2.1/examples-small/**/*.ptx eol=lf
+awk/examples/**/*.awk eol=lf
+bcpl/examples/**/*.b eol=lf
+bcpl/examples/**/*.bcpl eol=lf
+bicep/examples/**/*.txt eol=lf
+bison/examples/**/*.y eol=lf
+bnf/examples/**/*.bnf eol=lf
+c/examples/**/*.c eol=lf
+callable/examples/**/*.txt eol=lf
+caql/examples/**/*.txt eol=lf
+clf/examples/**/* eol=lf
+cmake/examples/**/*.txt eol=lf
+cobol85/examples/**/*.txt eol=lf
+cpp/examples/**/*.cc eol=lf
+cpp/examples/**/*.cpp eol=lf
+cql3/examples/**/*.cql eol=lf
+creole/examples/**/*.txt eol=lf
+csharp/examples/**/*.cs eol=lf
 css3/examples/*.css eol=lf
+csv/examples/*.csv eol=lf
+databank/examples/*.db eol=lf
+elixir/examples/*.ex eol=lf
+esolang/sickbay/examples/**/* eol=lf
+fasta/examples/**/* eol=lf
+fol/examples/**/* eol=lf
+fortran/fortran77/examples/**/* eol=lf
 gdscript/examples/*.gd eol=lf
+gedcom/examples/**/* eol=lf
+gff3/examples/**/* eol=lf
+golang/examples/**/* eol=lf
+graphstream-dgs/examples/**/* eol=lf
 haskell/examples/*.hs eol=lf
-hypertalk/example-scripts/*.txt eol=lf
 hypertalk/example-scriptlets/*.txt eol=lf
+hypertalk/example-scripts/*.txt eol=lf
+icalendar/examples/**/* eol=lf
+inf/examples/**/* eol=lf
+informix/examples/**/* eol=lf
+java/java/examples/**/* eol=lf
 javadoc/examples/**/*.java eol=lf
+javascript/javascript/examples/**/* eol=lf
+javascript/typescript/examples/**/* eol=lf
+json5/examples/**/* eol=lf
+kotlin/kotlin-formal/examples/**/* eol=lf
 kuka/examples/*.txt eol=lf
+logo/logo/examples/**/* eol=lf
+lua/examples/**/* eol=lf
+matlab/examples/**/* eol=lf
 mckeeman-form/examples/*.txt eol=lf
+modelica/examples/**/* eol=lf
+moo/examples/**/* eol=lf
+muddb/examples/**/* eol=lf
+mumps/examples/**/* eol=lf
 objc/examples/* eol=lf
+pcre/examples/**/* eol=lf
+pegen/examples/**/* eol=lf
+php/examples/**/* eol=lf
+ply/examples/**/* eol=lf
 properties/examples/* eol=lf
+python/python/examples/**/* eol=lf
+python/python2_7_18/examples/**/* eol=lf
+python/python3/large/**/* eol=lf
+python/python3/small/**/* eol=lf
+python/python3_12_1/examples/**/* eol=lf
 r/examples/*.txt eol=lf
-# vb6/examples/**/*.vb eol=lf
+recfile/examples/**/* eol=lf
+redcode/examples/**/* eol=lf
+rexx/examples/* eol=lf
+robotwars/examples/**/* eol=lf
+ruby/examples/**/* eol=lf
+rust/examples/**/* eol=lf
+scala/examples/**/* eol=lf
+sgf/examples/**/* eol=lf
+smalltalk/examples/**/* eol=lf
+smtlibv2/examples/**/* eol=lf
+snobol/examples/**/* eol=lf
+spass/examples/**/* eol=lf
+sql/informix-sql/examples/**/* eol=lf
+sql/mariadb/examples/**/* eol=lf
+sql/mysql/Positive-Technologies/examples/**/* eol=lf
+sql/plsql/examples/**/* eol=lf
+sql/postgresql/examples/**/* eol=lf
+sql/snowflake/examples/**/* eol=lf
+sql/tsql/examples/*.sql eol=lf
+swift-fin/examples/**/* eol=lf
+terraform/examples/**/* eol=lf
+tinybasic/examples/**/* eol=lf
+tinymud/examples/**/* eol=lf
+toml/examples/**/* eol=lf
+tsv/examples/**/* eol=lf
+turtle/examples/**/* eol=lf
+unicode/graphemes/examples/*.txt eol=lf
+url/examples/**/* eol=lf
+vb6/examples/**/* eol=lf
 vba/examples/*.bas eol=lf
+vba/vba6/examples/**/* eol=lf
+vba/vba_cc/examples/**/* eol=lf
+vba/vba_like/examples/**/* eol=lf
+wavefront/examples/**/* eol=lf
+xml/examples/**/* eol=lf
+xsd-regex/examples/**/* eol=lf
+xyz/examples/**/* eol=lf
 z/examples/*.utf8 eol=lf
 
+
 # Declare files that will always have CRLF line endings on checkout.
 # Currently the /vb6/**/*.cls and *.frm file need this, otherwise the grammar fails. 
 *.cls text eol=crlf
@@ -139,3 +234,5 @@ sql/postgresql/examples.errors/** linguist-vendored
 v/failed_examples/** linguist-vendored
 
 lua/examples/* eol=lf
+
+*.ipt eol=lf

_scripts/templates/Antlr4ng/st.Test.ts

@@ -1,21 +1,29 @@
 // Generated from trgen <version>
 
 import { ATNSimulator } from 'antlr4ng';
+import { BaseErrorListener } from 'antlr4ng';
 import { CharStream } from 'antlr4ng';
 import { CommonTokenStream } from 'antlr4ng';
 import { ConsoleErrorListener } from 'antlr4ng';
-import { BaseErrorListener } from 'antlr4ng';
+import { ErrorNode } from 'antlr4ng';
 //import { InputStream } from 'antlr4ng';
+import { Parser } from 'antlr4ng';
+import { ParserRuleContext } from 'antlr4ng';
+import { ParseTree } from 'antlr4ng';
 import { Recognizer } from 'antlr4ng';
 import { RecognitionException } from 'antlr4ng';
+import { TerminalNode } from 'antlr4ng';
 import { Token } from 'antlr4ng';
+import { Trees } from 'antlr4ng';
+import { escapeWhitespace } from 'antlr4ng';
 import { readFileSync } from 'fs';
 import { writeFileSync } from 'fs';
 import { openSync } from 'fs';
 import { readSync } from 'fs';
 import { writeSync } from 'fs';
 import { closeSync } from 'fs';
 import { readFile } from 'fs/promises'
+import { isToken } from 'antlr4ng';
 
 <tool_grammar_tuples: {x | import { <x.GrammarAutomName> \} from './<x.GrammarAutomName>.js';
 } >
@@ -208,9 +216,9 @@ function DoParse(str: CharStream, input_name: string, row_number: number) {
     var t = timer.time().m * 60 + timer.time().s + timer.time().ms / 1000;
     if (show_tree) {
         if (tee) {
-            writeFileSync(input_name + ".tree", tree.toStringTree(parser.ruleNames, parser));
+            writeFileSync(input_name + ".ipt", toStringTree(tree, parser));
         } else {
-            console.error(tree.toStringTree(parser.ruleNames, parser));
+            console.error(toStringTree(tree, parser));
         }
     }
     if (!quiet) {
@@ -221,5 +229,55 @@ function DoParse(str: CharStream, input_name: string, row_number: number) {
     }
 }
 
+function toStringTree(tree: ParseTree, recog?: Parser | null): string {
+    var sb = new StringBuilder();
+	let ruleNames = recog.ruleNames;
+	toStringTree2(sb, tree, 0, ruleNames);
+	return sb.toString();
+}
+
+function toStringTree2(sb: StringBuilder, tree: ParseTree, indent: number, ruleNames: string[] | null): void {
+	let s = getNodeText(tree, ruleNames);
+	s = escapeWhitespace(s!, false);
+	const c = tree.getChildCount();
+	if (c === 0) {
+	    for (let i = 0; i \< indent; ++i) sb.Append(" ");
+		sb.Append(s); sb.AppendLine("");
+		return;
+	}
+    for (let i = 0; i \< indent; ++i) sb.Append(" ");
+	sb.Append(s); sb.AppendLine("");
+	for (let i = 0; i \< c; i++) {
+		toStringTree2(sb, tree.getChild(i)!, indent+1, ruleNames);
+	}
+}
+
+function getNodeText(t: ParseTree, ruleNames: string[] | null, recog?: Parser | null): string | undefined {
+	if (ruleNames !== null) {
+		if (t instanceof ParserRuleContext) {
+			const context = t.ruleContext;
+			const altNumber = context.getAltNumber();
+			// use const value of ATN.INVALID_ALT_NUMBER to avoid circular dependency
+			if (altNumber !== 0) {
+				return ruleNames[t.ruleIndex] + ":" + altNumber;
+			}
+
+			return ruleNames[t.ruleIndex];
+		} else if (t instanceof ErrorNode) {
+			return t.toString();
+		} else if (t instanceof TerminalNode) {
+			if (t.symbol !== null) {
+				return t.symbol.text;
+			}
+		}
+	}
+	const payload = t.getPayload();
+	if (isToken(payload)) {
+		return payload.text;
+	}
+
+	return String(t.getPayload());
+}
+
 
 main()

_scripts/templates/Antlr4ng/st.test.ps1

@@ -4,8 +4,8 @@ $Tests = "<if(os_win)>../<example_files_win><else>../<example_files_unix><endif>
 Write-Host "Test cases here: $Tests"
 
 # Get a list of test files from the test directory. Do not include any
-# .errors or .tree files. Pay close attention to remove only file names
-# that end with the suffix .errors or .tree.
+# .errors or .ipt files. Pay close attention to remove only file names
+# that end with the suffix .errors or .ipt.
 if (Test-Path -Path "tests.txt" -PathType Leaf) {
     Remove-Item "tests.txt"
 }
@@ -17,7 +17,7 @@ foreach ($file in $allFiles) {
         continue
     } elseif ($ext -eq ".errors") {
         continue
-    } elseif ($ext -eq ".tree") {
+    } elseif ($ext -eq ".ipt") {
         continue
     } else {
         $(& dotnet triconv -- -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null
@@ -66,10 +66,28 @@ if ( $size -eq 0 ) {
     exit 1
 }
 
+function func {
+    param (
+        [string]$p
+    )
+
+    # Find the index of the first asterisk (*)
+    $index = $p.IndexOf('*')
+
+    if ($index -eq -1) {
+        # No asterisk found, return the entire path
+        return $p
+    } else {
+        # Asterisk found, return the path up to but not including the asterisk
+        $path = $p.Substring(0, $index)
+        return $path
+    }
+}
+
 $old = Get-Location
-Set-Location ..
+Set-Location (func("<if(os_win)>../<example_files_win><else>../<example_files_unix><endif>"))
 
-# Check if any .errors/.tree files have changed. That's not good.
+# Check if any .errors/.ipt files have changed. That's not good.
 git config --global pager.diff false
 Remove-Item -Force -Path $old/updated.txt -errorAction ignore 2>&1 | Out-Null
 $updated = 0
@@ -87,7 +105,8 @@ foreach ($item in Get-ChildItem . -Recurse) {
 foreach ($item in Get-ChildItem . -Recurse) {
     $file = $item.fullname
     $ext = $item.Extension
-    if ($ext -eq ".tree") {
+    if ($ext -eq ".ipt") {
+        [IO.File]::WriteAllText($file, $([IO.File]::ReadAllText($file) -replace "`r`n", "`n"))
         git diff --exit-code $file *>> $old/updated.txt
 	$st = $LASTEXITCODE
         if ($st -ne 0) {