New Grammars Compiler (github-linguist#3915)

* grammars: Update several grammars with compat issues

* [WIP] Add new grammar conversion tools

* Wrap in a Docker script

* Proper Dockerfile support

* Add Javadoc grammar

* Remove NPM package.json

* Remove superfluous test

This is now always checked by the grammars compiler

* Update JSyntax grammar to new submodule

* Approve Javadoc license

* grammars: Remove checked-in dependencies

* grammars: Add regex checks to the compiler

* grammars: Point Oz to its actual submodule

* grammars: Refactor compiler to group errors by repo

* grammars: Cleanups to error reporting

.gitmodules

@@ -439,9 +439,6 @@
 [submodule "vendor/grammars/sublime-golo"]
 	path = vendor/grammars/sublime-golo
 	url = https://github.com/TypeUnsafe/sublime-golo
-[submodule "vendor/grammars/JSyntax"]
-	path = vendor/grammars/JSyntax
-	url = https://github.com/bcj/JSyntax
 [submodule "vendor/grammars/TXL"]
 	path = vendor/grammars/TXL
 	url = https://github.com/MikeHoffert/Sublime-Text-TXL-syntax
@@ -892,3 +889,9 @@
 [submodule "vendor/grammars/Syntax-highlighting-for-PostCSS"]
 	path = vendor/grammars/Syntax-highlighting-for-PostCSS
 	url = https://github.com/hudochenkov/Syntax-highlighting-for-PostCSS
+[submodule "vendor/grammars/javadoc.tmbundle"]
+	path = vendor/grammars/javadoc.tmbundle
+	url = https://github.com/textmate/javadoc.tmbundle
+[submodule "vendor/grammars/JSyntax"]
+	path = vendor/grammars/JSyntax
+	url = https://github.com/tikkanz/JSyntax

grammars.yml

@@ -1,4 +1,3 @@
----
 https://bitbucket.org/Clams/sublimesystemverilog/get/default.tar.gz:
 - source.systemverilog
 - source.ucfconstraints
@@ -344,6 +343,8 @@ vendor/grammars/java.tmbundle:
 - source.java-properties
 - text.html.jsp
 - text.junit-test-report
+vendor/grammars/javadoc.tmbundle:
+- text.html.javadoc
 vendor/grammars/javascript-objective-j.tmbundle:
 - source.js.objj
 vendor/grammars/jflex.tmbundle:
@@ -574,7 +575,7 @@ vendor/grammars/opa.tmbundle:
 - source.opa
 vendor/grammars/openscad.tmbundle:
 - source.scad
-vendor/grammars/oz-tmbundle/Syntaxes/Oz.tmLanguage:
+vendor/grammars/oz-tmbundle:
 - source.oz
 vendor/grammars/parrot:
 - source.parrot.pir

script/add-grammar

@@ -84,13 +84,13 @@ if repo_old
   log "Deregistering: #{repo_old}"
   `git submodule deinit #{repo_old}`
   `git rm -rf #{repo_old}`
-  `script/convert-grammars`
+  `script/grammar-compiler -update`
 end
 
 log "Registering new submodule: #{repo_new}"
 `git submodule add -f #{https} #{repo_new}`
 exit 1 if $?.exitstatus > 0
-`script/convert-grammars --add #{repo_new}`
+`script/grammar-compiler -add #{repo_new}`
 
 log "Confirming license"
 if repo_old

script/grammar-compiler

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -e
+cd "$(dirname "$0")/.."
+
+image="linguist/grammar-compiler:latest"
+mkdir -p grammars
+
+exec docker run --rm \
+    -u $(id -u $USER):$(id -g $USER) \
+    -v $PWD:/src/linguist \
+    -w /src/linguist -ti $image "$@"