lsif typed syntax highlighter v0.1 (sourcegraph#30016)

* (syntax-highlighter): tree-sitter,lsif-typed * lsif_typed -> lsiftyped * doc updates * update docker images to match .tool-versions * fixup: Remove submodules when running backcompat tests
bobheadlabs · Feb 23, 2022 · e2c2857 · e2c2857
1 parent c34c01b
commit e2c2857
Show file tree

Hide file tree

Showing 68 changed files with 7,503 additions and 1,548 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,8 @@
+[submodule "tree-sitter-go"]
+	path = docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-go
+	url = https://github.com/tree-sitter/tree-sitter-go
+	shallow = true
+[submodule "tree-sitter-sql"]
+	path = docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-sql
+	url = https://github.com/m-novikov/tree-sitter-sql
+	shallow = true
diff --git a/.golangci.yml b/.golangci.yml
@@ -63,6 +63,7 @@ run:
     - ui
     - vendor
     - node_modules
+    - docker-images/syntax-highlighter
 
   skip-files:
     # Auto-generated with depguard and forbidigo failures

diff --git a/.prettierignore b/.prettierignore
@@ -26,6 +26,7 @@ doc/
 **/__fixtures__
 GH2SG.bookmarklet.js
 docker-images/grafana/config/provisioning/dashboards/sourcegraph/
+docker-images/syntax-highlighter/crates/
 storybook-static/
 browser/code-intel-extensions/
 .buildkite-cache/

diff --git a/.tool-versions b/.tool-versions
@@ -11,3 +11,4 @@ trivy 0.20.0
 kustomize 4.0.5
 awscli 2.4.7
 python system
+rust 1.58.0
diff --git a/buf.gen.yaml b/buf.gen.yaml
@@ -12,3 +12,6 @@ plugins:
     path: ./.bin/protoc-gen-doc
     out: ./doc/code_intelligence/references
     opt: doc/code_intelligence/references/lsif.sprig,lsif.md
+  - name: rust
+    path: ./.bin/bin/protoc-gen-rust
+    out: ./docker-images/syntax-highlighter/crates/sg-lsif/src/
diff --git a/client/branded/src/global-styles/highlight.scss b/client/branded/src/global-styles/highlight.scss
@@ -94,6 +94,13 @@
     --hl-purple: #6c71c4;
     --hl-pink: #d33682;
 
+    // TODO: Followup for https://github.com/sourcegraph/sourcegraph/issues/30995
+    // Define these in a way that are good. These are just the same values as before.
+    --hl-dark-green: #859900;
+    --hl-dark-blue-1: #268bd2;
+    --hl-dark-blue-2: #268bd2;
+    --hl-dark-blue-3: #268bd2;
+
     // These grays are not ordered at all,
     // they're just in the order they were defined
     --hl-gray-0: #767676;
@@ -471,6 +478,120 @@
             color: var(--hl-orange);
         }
     }
+
+    // The following are syntax highlighting constructs for highlight
+    // groups returned by lsif-typed compatible indexers.
+    //
+    // Each group should exactly match the name of the spec.
+    // .hl-typed-$SYNTAX_KIND
+
+    .hl-typed-Comment {
+        color: var(--hl-dark-green);
+    }
+    .hl-typed-PunctuationDelimiter {
+        color: var(--hl-orange);
+    }
+    .hl-typed-PunctuationBracket {
+        color: var(--hl-orange);
+    }
+    .hl-typed-IdentifierKeyword {
+        color: var(--hl-dark-blue-3);
+    }
+    .hl-typed-IdentifierOperator {
+        color: var(--hl-dark-blue-1);
+    }
+    .hl-typed-Identifier {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierBuiltin {
+        color: var(--hl-blue);
+        // filter: brightness(105%);
+    }
+    .hl-typed-IdentifierNull {
+        color: var(--hl-orange);
+        // filter: brightness(130%);
+    }
+    .hl-typed-BooleanLiteral {
+        color: var(--hl-orange);
+    }
+    .hl-typed-IdentifierConstant {
+        color: var(--hl-dark-blue-1);
+    }
+    .hl-typed-IdentifierMutableGlobal {
+        color: var(--hl-dark-blue-1);
+    }
+    .hl-typed-IdentifierParameter {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierLocal {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierShadowed {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierModule {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierFunction {
+        color: var(--hl-yellow);
+    }
+    .hl-typed-IdentifierFunctionDefinition {
+        color: var(--hl-yellow);
+    }
+    .hl-typed-IdentifierMacro {
+        color: var(--hl-yellow);
+    }
+    .hl-typed-IdentifierMacroDefinition {
+        color: var(--hl-yellow);
+    }
+    .hl-typed-IdentifierType {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierBuiltinType {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierAttribute {
+        color: var(--hl-purple);
+    }
+
+    // TODO: Followup for https://github.com/sourcegraph/sourcegraph/issues/30995
+    // .hl-typed-RegexEscape {
+    // }
+    // .hl-typed-RegexRepeated {
+    // }
+    // .hl-typed-RegexWildcard {
+    // }
+    // .hl-typed-RegexDelimiter {
+    // }
+    // .hl-typed-RegexJoin {
+    // }
+
+    .hl-typed-StringLiteral {
+        color: var(--hl-red);
+    }
+    .hl-typed-StringLiteralEscape {
+        color: var(--hl-red);
+    }
+    .hl-typed-StringLiteralSpecial {
+        color: var(--hl-red);
+    }
+    .hl-typed-StringLiteralKey {
+        color: var(--hl-red);
+    }
+    .hl-typed-CharacterLiteral {
+        color: var(--hl-red);
+    }
+    .hl-typed-NumericLiteral {
+        color: var(--hl-green-1);
+    }
+
+    // TODO: Followup for https://github.com/sourcegraph/sourcegraph/issues/30995
+    // .hl-typed-Tag {
+    // }
+    // .hl-typed-TagAttribute {
+    // }
+    // .hl-typed-TagDelimiter {
+    // }
 }
 
 .theme-dark {
@@ -774,4 +895,118 @@
             }
         }
     }
+
+    // The following are syntax highlighting constructs for highlight
+    // groups returned by lsif-typed compatible indexers.
+    //
+    // Each group should exactly match the name of the spec.
+    // .hl-typed-$SYNTAX_KIND
+
+    .hl-typed-Comment {
+        color: var(--hl-dark-green);
+    }
+    .hl-typed-PunctuationDelimiter {
+        color: var(--hl-orange);
+    }
+    .hl-typed-PunctuationBracket {
+        color: var(--hl-orange);
+    }
+    .hl-typed-IdentifierKeyword {
+        color: var(--hl-dark-blue-3);
+    }
+    .hl-typed-IdentifierOperator {
+        color: var(--hl-dark-blue-1);
+    }
+    .hl-typed-Identifier {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierBuiltin {
+        color: var(--hl-blue);
+        // filter: brightness(105%);
+    }
+    .hl-typed-IdentifierNull {
+        color: var(--hl-orange);
+        // filter: brightness(130%);
+    }
+    .hl-typed-BooleanLiteral {
+        color: var(--hl-orange);
+    }
+    .hl-typed-IdentifierConstant {
+        color: var(--hl-dark-blue-1);
+    }
+    .hl-typed-IdentifierMutableGlobal {
+        color: var(--hl-dark-blue-1);
+    }
+    .hl-typed-IdentifierParameter {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierLocal {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierShadowed {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierModule {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierFunction {
+        color: var(--hl-yellow);
+    }
+    .hl-typed-IdentifierFunctionDefinition {
+        color: var(--hl-yellow);
+    }
+    .hl-typed-IdentifierMacro {
+        color: var(--hl-yellow);
+    }
+    .hl-typed-IdentifierMacroDefinition {
+        color: var(--hl-yellow);
+    }
+    .hl-typed-IdentifierType {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierBuiltinType {
+        color: var(--hl-blue);
+    }
+    .hl-typed-IdentifierAttribute {
+        color: var(--hl-purple);
+    }
+
+    // TODO: Followup for https://github.com/sourcegraph/sourcegraph/issues/30995
+    // .hl-typed-RegexEscape {
+    // }
+    // .hl-typed-RegexRepeated {
+    // }
+    // .hl-typed-RegexWildcard {
+    // }
+    // .hl-typed-RegexDelimiter {
+    // }
+    // .hl-typed-RegexJoin {
+    // }
+
+    .hl-typed-StringLiteral {
+        color: var(--hl-red);
+    }
+    .hl-typed-StringLiteralEscape {
+        color: var(--hl-red);
+    }
+    .hl-typed-StringLiteralSpecial {
+        color: var(--hl-red);
+    }
+    .hl-typed-StringLiteralKey {
+        color: var(--hl-red);
+    }
+    .hl-typed-CharacterLiteral {
+        color: var(--hl-red);
+    }
+    .hl-typed-NumericLiteral {
+        color: var(--hl-green-1);
+    }
+
+    // TODO: Followup for https://github.com/sourcegraph/sourcegraph/issues/30995
+    // .hl-typed-Tag {
+    // }
+    // .hl-typed-TagAttribute {
+    // }
+    // .hl-typed-TagDelimiter {
+    // }
 }
diff --git a/client/web/src/integration/blob-viewer.test.ts b/client/web/src/integration/blob-viewer.test.ts
@@ -133,6 +133,7 @@ describe('Blob viewer', () => {
                                         '<span class="hl-meta hl-function-call hl-method hl-js"></span>' +
                                         '<span class="hl-comment hl-line hl-double-slash hl-js"><span class="hl-punctuation hl-definition hl-comment hl-js">//</span> ' +
                                         'Third line\n</span></span></td></tr></tbody></table>',
+                                    lsif: '',
                                 },
                             },
                         },
@@ -214,6 +215,7 @@ describe('Blob viewer', () => {
                                         'Hello world' +
                                         '<span class="hl-punctuation hl-definition hl-string hl-end hl-js">&quot;</span></span></span>' +
                                         '<span class="hl-punctuation hl-section hl-group hl-end hl-js">)</span></span>\n</span></span></td></tr></tbody></table>',
+                                    lsif: '',
                                 },
                             },
                         },
@@ -482,6 +484,7 @@ describe('Blob viewer', () => {
                                         '<span class="hl-meta hl-function-call hl-method hl-js"></span>' +
                                         '<span class="hl-comment hl-line hl-double-slash hl-js"><span class="hl-punctuation hl-definition hl-comment hl-js">//</span> ' +
                                         'Third line\n</span></span></td></tr></tbody></table>',
+                                    lsif: '',
                                 },
                             },
                         },
@@ -729,6 +732,7 @@ describe('Blob viewer', () => {
                                           '<table><tbody><tr><td class="line" data-line="1"></td><td class="code"><div><span style="color: gray">&sol;&sol; First word line\n' +
                                           '</span></div></td></tr><tr><td class="line" data-line="2"></td><td class="code"><div><span style="color: gray">&sol;&sol; Second line</span></td></tr>\n' +
                                           '<tr><td class="line" data-line="3"></td><td class="code"><div><span style="color: gray">&sol;&sol; Third word line</span></td></tr></tbody></table>',
+                                      lsif: '',
                                   },
                               }
                             : {
@@ -741,6 +745,7 @@ describe('Blob viewer', () => {
                                           '<table><tbody><tr><td class="line" data-line="1"></td><td class="code"><div><span style="color: gray">&sol;&sol; First line\n' +
                                           '</span></div></td></tr><tr><td class="line" data-line="2"></td><td class="code"><div><span style="color: gray">&sol;&sol; Second word line</span></td></tr>\n' +
                                           '<tr><td class="line" data-line="3"></td><td class="code"><div><span style="color: gray">&sol;&sol; Third line</span></td></tr></tbody></table>',
+                                      lsif: '',
                                   },
                               }
 
@@ -935,6 +940,7 @@ describe('Blob viewer', () => {
                                         '<span class="hl-punctuation hl-definition hl-string hl-end hl-js">&quot;</span></span></span>' +
                                         '<span class="hl-punctuation hl-section hl-group hl-end hl-js">)</span></span>' +
                                         '\n</span></span></td></tr></tbody></table>',
+                                    lsif: '',
                                     lineRanges: [],
                                 },
                             },

diff --git a/client/web/src/integration/graphQlResponseHelpers.ts b/client/web/src/integration/graphQlResponseHelpers.ts
@@ -41,6 +41,7 @@ export const createBlobContentResult = (
                 highlight: {
                     aborted: false,
                     html,
+                    lsif: '',
                 },
             },
         },
-Original file line number
+Diff line change
@@ Expand Up / @@ -41,6 +41,7 @@ export const createBlobContentResult = ( @@
                     highlight: {
                         aborted: false,
                         html,
+                        lsif: '',
                     },
                 },
             },
@@ Expand Down @@