Grammar compiler tweaks and improvements (github-linguist#4444)

* Add git-config > gitconfig grammar alias

* Add list of files to ignore and grammars in nonstd path

* Validate grammars with files in nonstd path

* Ignore known duds

* Easy rebuild and run grammar-compiler locally

* Alias source.smarty to text.html.smarty

* Alias text.slm to text.slim

slm is a javascript port of Slim as per slim-template/slim#526

* Alias text.pug to text.jade

Jade was renamed to pug but the grammar still refers to it as Jade

* Ignore files produced by the grammar compiler

* Use a map[string]bool instead of a loop

* Don't ignore err

.gitignore

@@ -11,3 +11,4 @@ test/fixtures/ace_modes.json
 /tmp
 *.bundle
 *.so
+linguist-grammars*

script/grammar-compiler

@@ -6,9 +6,15 @@ cd "$(dirname "$0")/.."
 image="linguist/grammar-compiler:latest"
 mkdir -p grammars
 
-docker pull $image
+if [ -z "$REBUILD" ]; then
+    docker pull $image
+else
+    cd tools/grammars
+    image=$(docker build -q .)
+    cd "../.."
+fi
 
 exec docker run --rm \
-    -u $(id -u $USER):$(id -g $USER) \
-    -v $PWD:/src/linguist \
-    -w /src/linguist $image "$@"
+    -u "$(id -u "$USER")":"$(id -g "$USER")" \
+    -v "$PWD:/src/linguist" \
+    -w /src/linguist "$image" "$@"

tools/grammars/compiler/data.go

@@ -11,6 +11,10 @@ var GrammarAliases = map[string]string{
 	"source.asciidoc":    "text.html.asciidoc",
 	"source.perl6":       "source.perl6fe",
 	"source.css.scss":    "source.scss",
+	"source.git-config":  "source.gitconfig",
+	"source.smarty":      "text.html.smarty",
+	"text.slm":           "text.slim",
+	"text.pug":           "text.jade",
 }
 
 var KnownFields = map[string]bool{
@@ -34,3 +38,15 @@ var KnownFields = map[string]bool{
 	"backgroundColor":       true,
 	"increaseIndentPattern": true,
 }
+
+// GrammarsInNonStdPath is a list of grammars known to have their syntax .cson or .json files in non-standard directories.
+var GrammarsInNonStdPath = map[string]bool{
+	"conllu-linguist-grammar": true,
+	"hy.tmLanguage":           true,
+}
+
+// IgnoredFiles is a list of files that look like syntax files but aren't, or are known to be broken and never likely to be fixed.
+var IgnoredFiles = map[string]bool{
+	"ballerina-grammar/syntaxes/ballerina.monarch.json": true,
+	"oz-tmbundle/Originals/Oz.tmLanguage":               true,
+}

tools/grammars/compiler/loader.go

@@ -122,7 +122,7 @@ func isValidGrammar(path string, info os.FileInfo) bool {
 	case ".tmlanguage", ".yaml-tmlanguage":
 		return true
 	case ".cson", ".json":
-		return strings.HasSuffix(dir, "/grammars") || strings.HasSuffix(dir, "/syntaxes")
+		return strings.HasSuffix(dir, "/grammars") || strings.HasSuffix(dir, "/syntaxes") || GrammarsInNonStdPath[filepath.Base(dir)]
 	default:
 		return false
 	}

tools/grammars/compiler/loader_fs.go

@@ -63,6 +63,16 @@ func (l *fsLoader) load() {
 	}
 
 	for _, path := range grammars {
+		rel, err := filepath.Rel("/src/linguist/vendor/grammars/", path)
+		if err != nil {
+			l.Fail(err)
+			return
+		}
+
+		if IgnoredFiles[rel] {
+			continue
+		}
+
 		data, err := ioutil.ReadFile(path)
 		if err != nil {
 			l.Fail(err)