codemirror file view: Use new /scip highlighting endpoint

client/web/src/lsif/html.ts

@@ -70,6 +70,10 @@ function highlightSlice(html: HtmlBuilder, kind: SyntaxKind | undefined, slice:
 
 // Currently assumes that no ranges overlap in the occurrences.
 export function render(lsif_json: string, content: string): string {
+    if (!lsif_json.trim()) {
+        return ''
+    }
+
     const occurrences = (JSON.parse(lsif_json) as JsonDocument).occurrences?.map(occ => new Occurrence(occ))
     if (!occurrences) {
         return ''

client/web/src/repo/blob/BlobPage.tsx

@@ -13,7 +13,7 @@ import { ErrorLike, isErrorLike, asError } from '@sourcegraph/common'
 import { SearchContextProps } from '@sourcegraph/search'
 import { StreamingSearchResultsListProps } from '@sourcegraph/search-ui'
 import { ExtensionsControllerProps } from '@sourcegraph/shared/src/extensions/controller'
-import { Scalars } from '@sourcegraph/shared/src/graphql-operations'
+import { HighlightResponseFormat, Scalars } from '@sourcegraph/shared/src/graphql-operations'
 import { PlatformContextProps } from '@sourcegraph/shared/src/platform/context'
 import { SettingsCascadeProps } from '@sourcegraph/shared/src/settings/settings'
 import { TelemetryProps } from '@sourcegraph/shared/src/telemetry/telemetryService'
@@ -158,15 +158,15 @@ export const BlobPage: React.FunctionComponent<React.PropsWithChildren<Props>> =
                 return of(undefined)
             }
 
-            return fetchBlob({ repoName, commitID, filePath, formatOnly: true }).pipe(
+            return fetchBlob({ repoName, commitID, filePath, format: HighlightResponseFormat.HTML_PLAINTEXT }).pipe(
                 map(blob => {
                     if (blob === null) {
                         return blob
                     }
 
                     const blobInfo: BlobPageInfo = {
                         content: blob.content,
-                        html: blob.highlight.html,
+                        html: blob.highlight.html ?? '',
                         repoName,
                         revision,
                         commitID,
@@ -202,6 +202,9 @@ export const BlobPage: React.FunctionComponent<React.PropsWithChildren<Props>> =
                             commitID,
                             filePath,
                             disableTimeout,
+                            format: enableCodeMirror
+                                ? HighlightResponseFormat.JSON_SCIP
+                                : HighlightResponseFormat.HTML_HIGHLIGHT,
                         })
                     ),
                     map(blob => {
@@ -219,8 +222,8 @@ export const BlobPage: React.FunctionComponent<React.PropsWithChildren<Props>> =
 
                         const blobInfo: BlobPageInfo = {
                             content: blob.content,
-                            html: blob.highlight.html,
-                            lsif: blob.highlight.lsif,
+                            html: blob.highlight.html ?? '',
+                            lsif: blob.highlight.lsif ?? '',
                             repoName,
                             revision,
                             commitID,

client/web/src/repo/blob/backend.ts

@@ -6,18 +6,17 @@ import { dataOrThrowErrors, gql } from '@sourcegraph/http-client'
 import { ParsedRepoURI, makeRepoURI } from '@sourcegraph/shared/src/util/url'
 
 import { requestGraphQL } from '../../backend/graphql'
-import { BlobFileFields, BlobResult, BlobVariables } from '../../graphql-operations'
+import { BlobFileFields, BlobResult, BlobVariables, HighlightResponseFormat } from '../../graphql-operations'
 
-function fetchBlobCacheKey(parsed: ParsedRepoURI & { disableTimeout?: boolean; formatOnly?: boolean }): string {
-    return `${makeRepoURI(parsed)}?disableTimeout=${parsed.disableTimeout}&formatOnly=${parsed.formatOnly}`
+function fetchBlobCacheKey(parsed: ParsedRepoURI & { disableTimeout?: boolean; format?: string }): string {
+    return `${makeRepoURI(parsed)}?disableTimeout=${parsed.disableTimeout}&=${parsed.format}`
 }
-
 interface FetchBlobArguments {
     repoName: string
     commitID: string
     filePath: string
     disableTimeout?: boolean
-    formatOnly?: boolean
+    format?: HighlightResponseFormat
 }
 
 export const fetchBlob = memoizeObservable(
@@ -26,16 +25,24 @@ export const fetchBlob = memoizeObservable(
         commitID,
         filePath,
         disableTimeout = false,
-        formatOnly = false,
-    }: FetchBlobArguments): Observable<BlobFileFields | null> =>
-        requestGraphQL<BlobResult, BlobVariables>(
+        format = HighlightResponseFormat.HTML_HIGHLIGHT,
+    }: FetchBlobArguments): Observable<BlobFileFields | null> => {
+        // We only want to include HTML data if explicitly requested. We always
+        // include LSIF because this is used for languages that are configured
+        // to be processed with tree sitter (and is used when explicitly
+        // requested via JSON_SCIP).
+        const html =
+            format === HighlightResponseFormat.HTML_PLAINTEXT || format === HighlightResponseFormat.HTML_HIGHLIGHT
+
+        return requestGraphQL<BlobResult, BlobVariables>(
             gql`
                 query Blob(
                     $repoName: String!
                     $commitID: String!
                     $filePath: String!
                     $disableTimeout: Boolean!
-                    $formatOnly: Boolean!
+                    $format: HighlightResponseFormat!
+                    $html: Boolean!
                 ) {
                     repository(name: $repoName) {
                         commit(rev: $commitID) {
@@ -49,14 +56,14 @@ export const fetchBlob = memoizeObservable(
                 fragment BlobFileFields on File2 {
                     content
                     richHTML
-                    highlight(disableTimeout: $disableTimeout, formatOnly: $formatOnly) {
+                    highlight(disableTimeout: $disableTimeout, format: $format) {
                         aborted
-                        html
+                        html @include(if: $html)
                         lsif
                     }
                 }
             `,
-            { repoName, commitID, filePath, disableTimeout, formatOnly }
+            { repoName, commitID, filePath, disableTimeout, format, html }
         ).pipe(
             map(dataOrThrowErrors),
             map(data => {
@@ -65,6 +72,7 @@ export const fetchBlob = memoizeObservable(
                 }
                 return data.repository.commit.file
             })
-        ),
+        )
+    },
     fetchBlobCacheKey
 )

client/web/src/repo/tree/HomeTab.tsx

@@ -106,7 +106,7 @@ export const HomeTab: React.FunctionComponent<React.PropsWithChildren<Props>> =
 
                         const blobInfo: BlobInfo & { richHTML: string; aborted: boolean } = {
                             content: blob.content,
-                            html: blob.highlight.html,
+                            html: blob.highlight.html ?? '',
                             repoName: repo.name,
                             revision,
                             commitID,

cmd/frontend/graphqlbackend/highlight.go

@@ -7,6 +7,7 @@ import (
 	"github.com/gogo/protobuf/jsonpb"
 
 	"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/highlight"
+	"github.com/sourcegraph/sourcegraph/internal/gosyntect"
 	"github.com/sourcegraph/sourcegraph/internal/search/result"
 )
 
@@ -35,7 +36,7 @@ type HighlightArgs struct {
 	DisableTimeout     bool
 	IsLightTheme       *bool
 	HighlightLongLines bool
-	FormatOnly         bool
+	Format             string
 }
 
 type highlightedFileResolver struct {
@@ -92,7 +93,7 @@ func highlightContent(ctx context.Context, args *HighlightArgs, content, path st
 		HighlightLongLines: args.HighlightLongLines,
 		SimulateTimeout:    simulateTimeout,
 		Metadata:           metadata,
-		FormatOnly:         args.FormatOnly,
+		Format:             gosyntect.GetResponseFormat(args.Format),
 	})
 
 	result.aborted = aborted

cmd/frontend/graphqlbackend/schema.graphql

@@ -4326,6 +4326,24 @@ type GitTree implements TreeEntry {
     ): Boolean!
 }
 
+"""
+The format and highlighting to use when requesting highlighting information for a file.
+"""
+enum HighlightResponseFormat {
+    """
+    HTML formatted file content without syntax highlighting.
+    """
+    HTML_PLAINTEXT
+    """
+    HTML formatted file content with syntax highlighting.
+    """
+    HTML_HIGHLIGHT
+    """
+    SCIP highlighting information as JSON.
+    """
+    JSON_SCIP
+}
+
 """
 A file.
 In a future version of Sourcegraph, a repository's files may be distinct from a repository's blobs
@@ -4393,9 +4411,9 @@ interface File2 {
         """
         highlightLongLines: Boolean = false
         """
-        Return un-highlighted blob contents that are only formatted as a table for use in our blob views.
+        Specifies which format/highlighting technique to use.
         """
-        formatOnly: Boolean = false
+        format: HighlightResponseFormat = HTML_HIGHLIGHT
     ): HighlightedFile!
 }
 
@@ -4460,9 +4478,9 @@ type VirtualFile implements File2 {
         """
         highlightLongLines: Boolean = false
         """
-        Return un-highlighted blob contents that are only formatted as a table for use in our blob views.
+        Specifies which format/highlighting technique to use.
         """
-        formatOnly: Boolean = false
+        format: HighlightResponseFormat = HTML_HIGHLIGHT
     ): HighlightedFile!
 }
 
@@ -4565,9 +4583,9 @@ type GitBlob implements TreeEntry & File2 {
         """
         highlightLongLines: Boolean = false
         """
-        Return un-highlighted blob contents that are only formatted as a table for use in our blob views.
+        Specifies which format/highlighting technique to use.
         """
-        formatOnly: Boolean = false
+        format: HighlightResponseFormat = HTML_HIGHLIGHT
     ): HighlightedFile!
     """
     Submodule metadata if this tree points to a submodule

cmd/frontend/internal/highlight/highlight.go

@@ -106,10 +106,8 @@ type Params struct {
 	// Metadata provides optional metadata about the code we're highlighting.
 	Metadata Metadata
 
-	// FormatOnly, if true, will skip highlighting and only return the code
-	// in a formatted table view. This is useful if we want to display code
-	// as quickly as possible, without waiting for highlighting.
-	FormatOnly bool
+	// Format defines the response format of the syntax highlighting request.
+	Format gosyntect.HighlightResponseType
 }
 
 // Metadata contains metadata about a request to highlight code. It is used to
@@ -405,7 +403,7 @@ func Code(ctx context.Context, p Params) (response *HighlightedCode, aborted boo
 		return plainResponse, true, nil
 	}
 
-	if p.FormatOnly {
+	if p.Format == gosyntect.FormatHTMLPlaintext {
 		return unhighlightedCode(err, code)
 	}
 
@@ -431,6 +429,7 @@ func Code(ctx context.Context, p Params) (response *HighlightedCode, aborted boo
 		Tracer:           ot.GetTracer(ctx),
 		LineLengthLimit:  maxLineLength,
 		CSS:              true,
+		Engine:           getEngineParameter(filetypeQuery.Engine),
 	}
 
 	// Set the Filetype part of the command if:
@@ -443,7 +442,7 @@ func Code(ctx context.Context, p Params) (response *HighlightedCode, aborted boo
 		query.Filetype = filetypeQuery.Language
 	}
 
-	resp, err := client.Highlight(ctx, query, filetypeQuery.Engine == EngineTreeSitter)
+	resp, err := client.Highlight(ctx, query, p.Format)
 
 	if ctx.Err() == context.DeadlineExceeded {
 		log15.Warn(
@@ -487,7 +486,9 @@ func Code(ctx context.Context, p Params) (response *HighlightedCode, aborted boo
 		return unhighlightedCode(err, code)
 	}
 
-	if filetypeQuery.Engine == EngineTreeSitter {
+	// We need to return SCIP data if explicitly requested or if the selected
+	// engine is tree sitter.
+	if p.Format == gosyntect.FormatJSONSCIP || filetypeQuery.Engine == EngineTreeSitter {
 		document := new(scip.Document)
 		data, err := base64.StdEncoding.DecodeString(resp.Data)
 

cmd/frontend/internal/highlight/language.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/sourcegraph/sourcegraph/internal/conf"
 	"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
+	"github.com/sourcegraph/sourcegraph/internal/gosyntect"
 )
 
 type EngineType int
@@ -20,6 +21,15 @@ const (
 	EngineSyntect
 )
 
+// Converts an engine type to the corresponding parameter value for the syntax
+// highlighting request. Defaults to "syntec".
+func getEngineParameter(engine EngineType) string {
+	if engine == EngineTreeSitter {
+		return gosyntect.SyntaxEngineTreesitter
+	}
+	return gosyntect.SyntaxEngineSyntect
+}
+
 type SyntaxEngineQuery struct {
 	Engine           EngineType
 	Language         string