From 18d1f19dbc2100cd09d449b085159e17c2f2a225 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Sun, 2 Oct 2022 18:05:29 -0500 Subject: [PATCH 1/6] migrate https://github.com/atom/language-javascript/pull/686 The non-tree sitter version is almost completely copied so nothing to say there. There are some inconsistencies/missing operators in the tree-sitter version, for example: - `this` - `new.target` - `import.meta` - The keyword.operator order looks more wrong the more I look at it but that's for another day because of the possible scope regularity change in the future: https://github.com/atom/language-javascript/pull/690 - btw https://github.com/atom/language-javascript/pull/691 looks good to me I should probably do that Even though tree-sitter will have some above changes I added the new operators (`** **= ?? &&= ||= ??= void`) anyway. Well `void` is not new it's just missing. See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators for a list of operators. Of course while the comma is technically an operator, hopefully it's used more to separate elements in arrays or properties in objects. Also `delete` is an operator not control. --- .../grammars/javascript.cson | 508 +++++++++++++----- .../grammars/tree-sitter-javascript.cson | 17 +- .../spec/javascript-spec.coffee | 94 +++- 3 files changed, 469 insertions(+), 150 deletions(-) diff --git a/packages/language-javascript/grammars/javascript.cson b/packages/language-javascript/grammars/javascript.cson index 62011cefab..dccbbc2508 100644 --- a/packages/language-javascript/grammars/javascript.cson +++ b/packages/language-javascript/grammars/javascript.cson @@ -1,3 +1,6 @@ +# Note: Because of the optional chaining operator (?.), "object.property" in the comments usually means ("object.property" or "object?.property") +# Even though sometimes it doesn't seem like using the chaining operator would make any sense, it's still supported. example: Math?.PI +# Negative look-behinds for "." should cover "?." 'scopeName': 'source.js' 'fileTypes': [ 'js' @@ -284,8 +287,8 @@ ] } { - # [.]foo = function... - 'begin': '(?=(\\.)?[a-zA-Z_$][\\w$]*\\s*=\\s*(\\basync\\b\\s*)?\\bfunction\\b)' + # [?.]foo = function... + 'begin': '(?=(\\?\\.|\\.)?[a-zA-Z_$][\\w$]*\\s*=\\s*(\\basync\\b\\s*)?\\bfunction\\b)' 'end': '(?<=})' 'patterns': [ { @@ -300,10 +303,14 @@ 'name': 'meta.function.js' 'patterns': [ { - 'match': '(\\.)?([a-zA-Z_$][\\w$]*)\\s*(=)\\s*' + 'match': '(\\?\\.|\\.)?([a-zA-Z_$][\\w$]*)\\s*(=)\\s*' 'captures': '1': - 'name': 'meta.delimiter.method.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_method' + } + ] '2': 'name': 'entity.name.function.js' '3': @@ -533,10 +540,10 @@ ] } { - # [.]foo = ... => ... + # [.|?.]foo = ... => ... 'begin': '''(?x) (?= - (\\.)?[a-zA-Z_$][\\w$]* + (\\?\\.|\\.)?[a-zA-Z_$][\\w$]* \\s*(=)\\s* ((\\(([^\\(\\)]*)?\\))|[\\w$]+) \\s*=> @@ -546,8 +553,8 @@ (?<=})| ((?! \\s*{| - \\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*\\(| - \\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*[\\w$]+| + \\G(\\?\\.|\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*\\(| + \\G(\\?\\.|\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*[\\w$]+| \\s*/\\*|\\s*// )(?=\\s*\\S)) ''' @@ -564,10 +571,14 @@ 'name': 'meta.function.arrow.js' 'patterns': [ { - 'match': '\\G(\\.)?([a-zA-Z_$][\\w$]*)\\s*(=)' + 'match': '\\G(\\?\\.|\\.)?([a-zA-Z_$][\\w$]*)\\s*(=)' 'captures': '1': - 'name': 'meta.delimiter.method.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_method' + } + ] '2': 'name': 'entity.name.function.js' '3': @@ -729,8 +740,7 @@ 'name': 'entity.name.type.instance.js' 'patterns': [ { - 'match': '\\.' - 'name': 'meta.delimiter.property.period.js' + 'include': '#period_or_optional_property' } ] } @@ -743,7 +753,7 @@ 'name': 'entity.name.type.object.console.js' 'end': '''(?x) (?<=\\)) | (?= - (?! (\\s*//)|(\\s*/\\*)|(\\s*(\\.)\\s* + (?! (\\s*//)|(\\s*/\\*)|(\\s*(\\?\\.|\\.)\\s* (assert|clear|debug|error|info|log|profile|profileEnd|time|timeEnd|warn) \\s*\\( )) \\s*\\S @@ -754,10 +764,14 @@ 'include': '#comments' } { - 'begin': '\\s*(\\.)\\s*(\\w+)\\s*(?=\\()' + 'begin': '\\s*(\\?\\.|\\.)\\s*(\\w+)\\s*(?=\\()' 'beginCaptures': '1': - 'name': 'meta.delimiter.method.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_method' + } + ] '2': 'name': 'support.function.console.js' 'end': '(?<=\\))' @@ -779,7 +793,7 @@ 'end': '''(?x) (?<=E|LN10|LN2|LOG10E|LOG2E|PI|SQRT1_2|SQRT2|\\) ) | (?= - (?! (\\s*//)|(\\s*/\\*)|(\\s*\\.\\s* ( + (?! (\\s*//)|(\\s*/\\*)|(\\s*(\\?\\.|\\.)\\s* ( ((abs|acos|acosh|asin|asinh|atan|atan2|atanh|cbrt|ceil|clz32|cos|cosh|exp| expm1|floor|fround|hypot|imul|log|log10|log1p|log2|max|min|pow|random| round|sign|sin|sinh|sqrt|tan|tanh|trunc)\\s*\\( @@ -793,10 +807,14 @@ } { # Math.random() - 'begin': '\\s*(\\.)\\s*(\\w+)\\s*(?=\\()' + 'begin': '\\s*(\\?\\.|\\.)\\s*(\\w+)\\s*(?=\\()' 'beginCaptures': '1': - 'name': 'meta.delimiter.method.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_method' + } + ] '2': 'name': 'support.function.math.js' 'end': '(?<=\\))' @@ -809,10 +827,14 @@ } { # Math.PI - 'match': '\\s*(\\.)\\s*(\\w+)\\b' + 'match': '\\s*(\\?\\.|\\.)\\s*(\\w+)\\b' 'captures': '1': - 'name': 'meta.delimiter.property.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_property' + } + ] '2': 'name': 'support.constant.property.math.js' } @@ -826,7 +848,7 @@ 'name': 'support.class.promise.js' 'end': '''(?x) (?<=\\)) | (?= - (?! (\\s*//)|(\\s*/\\*)|(\\s*\\.\\s*(all|race|reject|resolve)\\s*\\() )\\s*\\S + (?! (\\s*//)|(\\s*/\\*)|(\\s*(\\?\\.|\\.)\\s*(all|race|reject|resolve)\\s*\\() )\\s*\\S ) ''' 'patterns': [ @@ -835,10 +857,14 @@ } { # Promise.all() - 'begin': '\\s*(\\.)\\s*(\\w+)\\s*(?=\\()' + 'begin': '\\s*(\\?\\.|\\.)\\s*(\\w+)\\s*(?=\\()' 'beginCaptures': '1': - 'name': 'meta.delimiter.method.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_method' + } + ] '2': 'name': 'support.function.promise.js' 'end': '(?<=\\))' @@ -941,6 +967,7 @@ 'name': 'meta.control.yield.js' } { + # This is "..." so don't need to check "?." 'match': '(?:(?<=\\.{3})|(?>=|>>>=|\\|=' + # &&= ||= ??= + 'match': '&&=|\\|\\|=|\\?\\?=' + 'name': 'keyword.operator.assignment.compound.logical.js' + } + { + 'match': '&=|\\^=|\\|=' 'name': 'keyword.operator.assignment.compound.bitwise.js' } + { + 'match': '<<=|>>=|>>>=' + 'name': 'keyword.operator.assignment.compound.bitwise.shift.js' + } + # Bitwise shift to match >> instead of > + # Then comparison to match != instead of ! { 'match': '<<|>>>|>>' 'name': 'keyword.operator.bitwise.shift.js' @@ -1330,7 +1378,8 @@ 'name': 'keyword.operator.comparison.js' } { - 'match': '&&|!!|!|\\|\\|' + # One of: && !! ! || ?? + 'match': '&&|!!|!|\\|\\||\\?\\?' 'name': 'keyword.operator.logical.js' } { @@ -1350,7 +1399,7 @@ 'name': 'keyword.operator.increment.js' } { - 'match': '%|\\*|/|-|\\+' + 'match': '%|\\*\\*|\\*|/|-|\\+' 'name': 'keyword.operator.js' } ] @@ -1447,7 +1496,7 @@ ] } { - 'begin': '(Relay\\.QL|gql)\\s*(`)' + 'begin': '(Relay(\\?\\.|\\.)QL|gql)\\s*(`)' 'beginCaptures': '1': 'name': 'entity.name.function.js' @@ -1559,7 +1608,38 @@ 'match': '(\\.\\.\\.)([a-zA-Z_$][\\w$]*)' 'captures': '1': - 'name': 'keyword.operator.spread.js' + 'name': 'keyword.operator.rest.js' + '2': + 'name': 'variable.parameter.rest.function.js' + } + { + 'include': '$self' + } + { + 'match': '[a-zA-Z_$][\\w$]*' + 'name': 'variable.parameter.function.js' + } + ] + } + { + # Or starting with optional chaining + 'begin': '(\\?\\.)(\\()' + 'beginCaptures': + '1': + 'name': 'meta.delimiter.method.optional.js' + '2': + 'name': 'punctuation.definition.parameters.begin.bracket.round.js' + 'end': '\\)' + 'endCaptures': + '0': + 'name': 'punctuation.definition.parameters.end.bracket.round.js' + 'name': 'meta.parameters.js' + 'patterns': [ + { + 'match': '(\\.\\.\\.)([a-zA-Z_$][\\w$]*)' + 'captures': + '1': + 'name': 'keyword.operator.rest.js' '2': 'name': 'variable.parameter.rest.function.js' } @@ -1662,96 +1742,181 @@ 'method_calls': 'patterns': [ { - # .methodCall(arg1, "arg2", [...]) - 'begin': '(\\.)\\s*([\\w$]+)\\s*(?=\\()' + # [?].methodCall(arg1, "arg2", [...]) + 'begin': '(\\?\\.|\\.)\\s*([\\w$]+)\\s*(?=\\()' 'beginCaptures': '1': - 'name': 'meta.delimiter.method.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_method' + } + ] '2': 'patterns': [ { - 'match': '''(?x) - \\bon(Rowsinserted|Rowsdelete|Rowenter|Rowexit|Resize|Resizestart|Resizeend|Reset| - Readystatechange|Mouseout|Mouseover|Mousedown|Mouseup|Mousemove| - Before(cut|deactivate|unload|update|paste|print|editfocus|activate)| - Blur|Scrolltop|Submit|Select|Selectstart|Selectionchange|Hover|Help| - Change|Contextmenu|Controlselect|Cut|Cellchange|Clock|Close|Deactivate| - Datasetchanged|Datasetcomplete|Dataavailable|Drop|Drag|Dragstart|Dragover| - Dragdrop|Dragenter|Dragend|Dragleave|Dblclick|Unload|Paste|Propertychange|Error| - Errorupdate|Keydown|Keyup|Keypress|Focus|Load|Activate|Afterupdate|Afterprint|Abort)\\b - ''' - 'name': 'support.function.event-handler.js' + 'include': '#method_names' } + ] + 'end': '(?<=\\))' + 'name': 'meta.method-call.js' + 'patterns': [ + { + 'include': '#arguments' + } + ] + } + { + # [?].methodCall?.(arg1, "arg2", [...]) + 'begin': '(\\?\\.|\\.)\\s*([\\w$]+)\\s*(\\?\\.)\\s*(?=\\()' + 'beginCaptures': + '1': + 'patterns': [ { - 'match': '''(?x) - \\b(catch|finally|then|shift|showModelessDialog|showModalDialog|showHelp|scroll|scrollX|scrollByPages| - scrollByLines|scrollY|scrollTo|stop|strike|sizeToContent|sidebar|signText|sort| - sup|sub|substr|substring|splice|split|send|set(Milliseconds|Seconds|Minutes|Hours| - Month|Year|FullYear|Date|UTC(Milliseconds|Seconds|Minutes|Hours|Month|FullYear|Date)| - Time|Hotkeys|Cursor|ZOptions|Active|Resizable|RequestHeader)|search|slice| - savePreferences|small|home|handleEvent|navigate|char|charCodeAt|charAt|concat| - contextual|confirm|compile|clear|captureEvents|call|createStyleSheet|createPopup| - createEventObject|to(GMTString|UTCString|String|Source|UpperCase|LowerCase|LocaleString)| - test|taint|taintEnabled|indexOf|italics|disableExternalCapture|dump|detachEvent|unshift| - untaint|unwatch|updateCommands|join|javaEnabled|pop|push|plugins.refresh|paddings|parse|stringify| - print|prompt|preference|enableExternalCapture|exec|execScript|valueOf|UTC|find|file| - fileModifiedDate|fileSize|fileCreatedDate|fileUpdatedDate|fixed|fontsize|fontcolor| - forward|fromCharCode|watch|link|load|lastIndexOf|anchor|attachEvent|atob|apply|alert| - abort|routeEvents|resize|resizeBy|resizeTo|recalc|returnValue|replace|reverse|reload| - releaseCapture|releaseEvents|go|get(Milliseconds|Seconds|Minutes|Hours|Month|Day|Year|FullYear| - Time|Date|TimezoneOffset|UTC(Milliseconds|Seconds|Minutes|Hours|Day|Month|FullYear|Date)| - Attention|Selection|ResponseHeader|AllResponseHeaders)|moveBy|moveBelow|moveTo| - moveToAbsolute|moveAbove|mergeAttributes|match|margins|btoa|big|bold|borderWidths|blink|back)\\b - ''' - 'name': 'support.function.js' + 'include': '#period_or_optional_method' } + ] + '2': + 'patterns': [ + { + 'include': '#method_names' + } + ] + '3': + 'name': 'meta.delimiter.method.optional.js' + 'end': '(?<=\\))' + 'name': 'meta.method-call.js' + 'patterns': [ + { + 'include': '#arguments' + } + ] + } + { + # [array]?.(arg1, "arg2", [...]) + # The array itself is not in the method-call scope, maybe later + 'begin': '(?<=\\])\\s*(\\?\\.)\\s*(?=\\()' + 'beginCaptures': + '1': + 'name': 'meta.delimiter.method.optional.js' + 'end': '(?<=\\))' + 'name': 'meta.method-call.js' + 'patterns': [ + { + 'include': '#arguments' + } + ] + } + ] + 'method_names': + 'patterns': [ + { + 'match': '''(?x) + \\bon(Rowsinserted|Rowsdelete|Rowenter|Rowexit|Resize|Resizestart|Resizeend|Reset| + Readystatechange|Mouseout|Mouseover|Mousedown|Mouseup|Mousemove| + Before(cut|deactivate|unload|update|paste|print|editfocus|activate)| + Blur|Scrolltop|Submit|Select|Selectstart|Selectionchange|Hover|Help| + Change|Contextmenu|Controlselect|Cut|Cellchange|Clock|Close|Deactivate| + Datasetchanged|Datasetcomplete|Dataavailable|Drop|Drag|Dragstart|Dragover| + Dragdrop|Dragenter|Dragend|Dragleave|Dblclick|Unload|Paste|Propertychange|Error| + Errorupdate|Keydown|Keyup|Keypress|Focus|Load|Activate|Afterupdate|Afterprint|Abort)\\b + ''' + 'name': 'support.function.event-handler.js' + } + { + 'match': '''(?x) + \\b(catch|finally|then|shift|showModelessDialog|showModalDialog|showHelp|scroll|scrollX|scrollByPages| + scrollByLines|scrollY|scrollTo|stop|strike|sizeToContent|sidebar|signText|sort| + sup|sub|substr|substring|splice|split|send|set(Milliseconds|Seconds|Minutes|Hours| + Month|Year|FullYear|Date|UTC(Milliseconds|Seconds|Minutes|Hours|Month|FullYear|Date)| + Time|Hotkeys|Cursor|ZOptions|Active|Resizable|RequestHeader)|search|slice| + savePreferences|small|home|handleEvent|navigate|char|charCodeAt|charAt|concat| + contextual|confirm|compile|clear|captureEvents|call|createStyleSheet|createPopup| + createEventObject|to(GMTString|UTCString|String|Source|UpperCase|LowerCase|LocaleString)| + test|taint|taintEnabled|indexOf|italics|disableExternalCapture|dump|detachEvent|unshift| + untaint|unwatch|updateCommands|join|javaEnabled|pop|push|plugins.refresh|paddings|parse|stringify| + print|prompt|preference|enableExternalCapture|exec|execScript|valueOf|UTC|find|file| + fileModifiedDate|fileSize|fileCreatedDate|fileUpdatedDate|fixed|fontsize|fontcolor| + forward|fromCharCode|watch|link|load|lastIndexOf|anchor|attachEvent|atob|apply|alert| + abort|routeEvents|resize|resizeBy|resizeTo|recalc|returnValue|replace|reverse|reload| + releaseCapture|releaseEvents|go|get(Milliseconds|Seconds|Minutes|Hours|Month|Day|Year|FullYear| + Time|Date|TimezoneOffset|UTC(Milliseconds|Seconds|Minutes|Hours|Day|Month|FullYear|Date)| + Attention|Selection|ResponseHeader|AllResponseHeaders)|moveBy|moveBelow|moveTo| + moveToAbsolute|moveAbove|mergeAttributes|match|margins|btoa|big|bold|borderWidths|blink|back)\\b + ''' + 'name': 'support.function.js' + } + { + 'match': '''(?x) + \\b(acceptNode|add|addEventListener|addTextTrack|adoptNode|after|animate|append| + appendChild|appendData|before|blur|canPlayType|captureStream| + caretPositionFromPoint|caretRangeFromPoint|checkValidity|clear|click| + cloneContents|cloneNode|cloneRange|close|closest|collapse| + compareBoundaryPoints|compareDocumentPosition|comparePoint|contains| + convertPointFromNode|convertQuadFromNode|convertRectFromNode|createAttribute| + createAttributeNS|createCaption|createCDATASection|createComment| + createContextualFragment|createDocument|createDocumentFragment| + createDocumentType|createElement|createElementNS|createEntityReference| + createEvent|createExpression|createHTMLDocument|createNodeIterator| + createNSResolver|createProcessingInstruction|createRange|createShadowRoot| + createTBody|createTextNode|createTFoot|createTHead|createTreeWalker|delete| + deleteCaption|deleteCell|deleteContents|deleteData|deleteRow|deleteTFoot| + deleteTHead|detach|disconnect|dispatchEvent|elementFromPoint|elementsFromPoint| + enableStyleSheetsForSet|entries|evaluate|execCommand|exitFullscreen| + exitPointerLock|expand|extractContents|fastSeek|firstChild|focus|forEach|get| + getAll|getAnimations|getAttribute|getAttributeNames|getAttributeNode| + getAttributeNodeNS|getAttributeNS|getBoundingClientRect|getBoxQuads| + getClientRects|getContext|getDestinationInsertionPoints|getElementById| + getElementsByClassName|getElementsByName|getElementsByTagName| + getElementsByTagNameNS|getItem|getNamedItem|getSelection|getStartDate| + getVideoPlaybackQuality|has|hasAttribute|hasAttributeNS|hasAttributes| + hasChildNodes|hasFeature|hasFocus|importNode|initEvent|insertAdjacentElement| + insertAdjacentHTML|insertAdjacentText|insertBefore|insertCell|insertData| + insertNode|insertRow|intersectsNode|isDefaultNamespace|isEqualNode| + isPointInRange|isSameNode|item|key|keys|lastChild|load|lookupNamespaceURI| + lookupPrefix|matches|move|moveAttribute|moveAttributeNode|moveChild| + moveNamedItem|namedItem|nextNode|nextSibling|normalize|observe|open| + parentNode|pause|play|postMessage|prepend|preventDefault|previousNode| + previousSibling|probablySupportsContext|queryCommandEnabled| + queryCommandIndeterm|queryCommandState|queryCommandSupported|queryCommandValue| + querySelector|querySelectorAll|registerContentHandler|registerElement| + registerProtocolHandler|releaseCapture|releaseEvents|remove|removeAttribute| + removeAttributeNode|removeAttributeNS|removeChild|removeEventListener| + removeItem|replace|replaceChild|replaceData|replaceWith|reportValidity| + requestFullscreen|requestPointerLock|reset|scroll|scrollBy|scrollIntoView| + scrollTo|seekToNextFrame|select|selectNode|selectNodeContents|set|setAttribute| + setAttributeNode|setAttributeNodeNS|setAttributeNS|setCapture| + setCustomValidity|setEnd|setEndAfter|setEndBefore|setItem|setNamedItem| + setRangeText|setSelectionRange|setSinkId|setStart|setStartAfter|setStartBefore| + slice|splitText|stepDown|stepUp|stopImmediatePropagation|stopPropagation| + submit|substringData|supports|surroundContents|takeRecords|terminate|toBlob| + toDataURL|toggle|toString|values|write|writeln)\\b + ''' + 'name': 'support.function.dom.js' + } + { + 'match': "[a-zA-Z_$][\\w$]*" + 'name': 'entity.name.function.js' + } + { + 'match': '\\d[\\w$]*' + 'name': 'invalid.illegal.identifier.js' + } + ] + 'function_calls': + 'patterns': [ + { + # functionCall(arg1, "arg2", [...]) + 'begin': '([\\w$]+)\\s*(?=\\()' + 'beginCaptures': + '1': + 'patterns': [ { 'match': '''(?x) - \\b(acceptNode|add|addEventListener|addTextTrack|adoptNode|after|animate|append| - appendChild|appendData|before|blur|canPlayType|captureStream| - caretPositionFromPoint|caretRangeFromPoint|checkValidity|clear|click| - cloneContents|cloneNode|cloneRange|close|closest|collapse| - compareBoundaryPoints|compareDocumentPosition|comparePoint|contains| - convertPointFromNode|convertQuadFromNode|convertRectFromNode|createAttribute| - createAttributeNS|createCaption|createCDATASection|createComment| - createContextualFragment|createDocument|createDocumentFragment| - createDocumentType|createElement|createElementNS|createEntityReference| - createEvent|createExpression|createHTMLDocument|createNodeIterator| - createNSResolver|createProcessingInstruction|createRange|createShadowRoot| - createTBody|createTextNode|createTFoot|createTHead|createTreeWalker|delete| - deleteCaption|deleteCell|deleteContents|deleteData|deleteRow|deleteTFoot| - deleteTHead|detach|disconnect|dispatchEvent|elementFromPoint|elementsFromPoint| - enableStyleSheetsForSet|entries|evaluate|execCommand|exitFullscreen| - exitPointerLock|expand|extractContents|fastSeek|firstChild|focus|forEach|get| - getAll|getAnimations|getAttribute|getAttributeNames|getAttributeNode| - getAttributeNodeNS|getAttributeNS|getBoundingClientRect|getBoxQuads| - getClientRects|getContext|getDestinationInsertionPoints|getElementById| - getElementsByClassName|getElementsByName|getElementsByTagName| - getElementsByTagNameNS|getItem|getNamedItem|getSelection|getStartDate| - getVideoPlaybackQuality|has|hasAttribute|hasAttributeNS|hasAttributes| - hasChildNodes|hasFeature|hasFocus|importNode|initEvent|insertAdjacentElement| - insertAdjacentHTML|insertAdjacentText|insertBefore|insertCell|insertData| - insertNode|insertRow|intersectsNode|isDefaultNamespace|isEqualNode| - isPointInRange|isSameNode|item|key|keys|lastChild|load|lookupNamespaceURI| - lookupPrefix|matches|move|moveAttribute|moveAttributeNode|moveChild| - moveNamedItem|namedItem|nextNode|nextSibling|normalize|observe|open| - parentNode|pause|play|postMessage|prepend|preventDefault|previousNode| - previousSibling|probablySupportsContext|queryCommandEnabled| - queryCommandIndeterm|queryCommandState|queryCommandSupported|queryCommandValue| - querySelector|querySelectorAll|registerContentHandler|registerElement| - registerProtocolHandler|releaseCapture|releaseEvents|remove|removeAttribute| - removeAttributeNode|removeAttributeNS|removeChild|removeEventListener| - removeItem|replace|replaceChild|replaceData|replaceWith|reportValidity| - requestFullscreen|requestPointerLock|reset|scroll|scrollBy|scrollIntoView| - scrollTo|seekToNextFrame|select|selectNode|selectNodeContents|set|setAttribute| - setAttributeNode|setAttributeNodeNS|setAttributeNS|setCapture| - setCustomValidity|setEnd|setEndAfter|setEndBefore|setItem|setNamedItem| - setRangeText|setSelectionRange|setSinkId|setStart|setStartAfter|setStartBefore| - slice|splitText|stepDown|stepUp|stopImmediatePropagation|stopPropagation| - submit|substringData|supports|surroundContents|takeRecords|terminate|toBlob| - toDataURL|toggle|toString|values|write|writeln)\\b + \\b(isNaN|isFinite|eval|uneval|parseInt|parseFloat|decodeURI| + decodeURIComponent|encodeURI|encodeURIComponent|escape|unescape| + require|set(Interval|Timeout)|clear(Interval|Timeout))\\b ''' - 'name': 'support.function.dom.js' + 'name': 'support.function.js' } { 'match': "[a-zA-Z_$][\\w$]*" @@ -1763,19 +1928,16 @@ } ] 'end': '(?<=\\))' - 'name': 'meta.method-call.js' + 'name': 'meta.function-call.js' 'patterns': [ { 'include': '#arguments' } ] } - ] - 'function_calls': - 'patterns': [ { - # functionCall(arg1, "arg2", [...]) - 'begin': '([\\w$]+)\\s*(?=\\()' + # functionCall?.(arg1, "arg2", [...]) + 'begin': '([\\w$]+)\\s*(\\?\\.)(?=\\()' 'beginCaptures': '1': 'patterns': [ @@ -1796,6 +1958,8 @@ 'name': 'invalid.illegal.identifier.js' } ] + '2': + 'name': 'meta.delimiter.method.optional.js' 'end': '(?<=\\))' 'name': 'meta.function-call.js' 'patterns': [ @@ -1807,61 +1971,113 @@ ] 'objects': 'patterns': [ + # "Obj." is already enough to prove objectness, + # however "Obj.[8]" is impossible, and + # "Obj?.()" is a function { # OBJ in OBJ.prop, OBJ.methodCall() - 'match': '[A-Z][A-Z0-9_$]*(?=\\s*\\.\\s*[a-zA-Z_$]\\w*)' + 'match': '[A-Z][A-Z0-9_$]*(?=\\s*\\??\\.\\s*[a-zA-Z_$]\\w*)' 'name': 'constant.other.object.js' } { # obj in obj.prop, obj.methodCall() - 'match': '[a-zA-Z_$][\\w$]*(?=\\s*\\.\\s*[a-zA-Z_$]\\w*)' + 'match': '[a-zA-Z_$][\\w$]*(?=\\s*\\??\\.\\s*[a-zA-Z_$]\\w*)' + 'name': 'variable.other.object.js' + } + { + # OBJ in OBJ[8], OBJ?.[8] + 'match': '[A-Z][A-Z0-9_$]*(?=\\s*(\\?\\.)?\\s*\\[)' + 'name': 'constant.other.object.js' + } + { + # obj in obj[8], obj?.[8] + 'match': '[a-zA-Z_$][\\w$]*(?=\\s*(\\?\\.)?\\s*\\[)' 'name': 'variable.other.object.js' } ] 'properties': 'patterns': [ + # object.prop includes object?.prop + { + # Specifically ?.[], for example divArray?.[8] + 'begin': '(\\?\\.)\\s*(\\[)' + 'beginCaptures': + '1': + 'name': 'meta.delimiter.property.optional.js' + '2': + 'name': 'meta.brace.square.js' + 'end': '\\]' + 'endCaptures': + '0': + 'name': 'meta.brace.square.js' + 'patterns': [ + { + 'include': '$self' + } + ] + } { # PROP1 in obj.PROP1.prop2, func().PROP1.prop2 - 'match': '(\\.)\\s*([A-Z][A-Z0-9_$]*\\b\\$*)(?=\\s*\\.\\s*[a-zA-Z_$]\\w*)' + 'match': '(\\?\\.|\\.)\\s*([A-Z][A-Z0-9_$]*\\b\\$*)(?=\\s*(\\?\\.|\\.)\\s*[a-zA-Z_$]\\w*)' 'captures': '1': - 'name': 'meta.delimiter.property.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_property' + } + ] '2': 'name': 'constant.other.object.property.js' } { # prop1 in obj.prop1.prop2, func().prop1.prop2 - 'match': '(\\.)\\s*(\\$*[a-zA-Z_$][\\w$]*)(?=\\s*\\.\\s*[a-zA-Z_$]\\w*)' + 'match': '(\\?\\.|\\.)\\s*(\\$*[a-zA-Z_$][\\w$]*)(?=\\s*(\\?\\.|\\.)\\s*[a-zA-Z_$]\\w*)' 'captures': '1': - 'name': 'meta.delimiter.property.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_property' + } + ] '2': 'name': 'variable.other.object.property.js' } { # PROP in obj.PROP, func().PROP - 'match': '(\\.)\\s*([A-Z][A-Z0-9_$]*\\b\\$*)' + 'match': '(\\?\\.|\\.)\\s*([A-Z][A-Z0-9_$]*\\b\\$*)' 'captures': '1': - 'name': 'meta.delimiter.property.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_property' + } + ] '2': 'name': 'constant.other.property.js' } { # prop in obj.prop, func().prop - 'match': '(\\.)\\s*(\\$*[a-zA-Z_$][\\w$]*)' + 'match': '(\\?\\.|\\.)\\s*(\\$*[a-zA-Z_$][\\w$]*)' 'captures': '1': - 'name': 'meta.delimiter.property.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_property' + } + ] '2': 'name': 'variable.other.property.js' } { # 123illegal in obj.123illegal, func().123illegal - 'match': '(\\.)\\s*([0-9][\\w$]*)' + 'match': '(\\?\\.|\\.)\\s*([0-9][\\w$]*)' 'captures': '1': - 'name': 'meta.delimiter.property.period.js' + 'patterns': [ + { + 'include': '#period_or_optional_property' + } + ] '2': 'name': 'invalid.illegal.identifier.js' } @@ -2026,3 +2242,25 @@ ] } ] + 'period_or_optional_property': + 'patterns': [ + { + 'match': '\\.' + 'name': 'meta.delimiter.property.period.js' + } + { + 'match': '\\?\\.' + 'name': 'meta.delimiter.property.optional.js' + } + ] + 'period_or_optional_method': + 'patterns': [ + { + 'match': '\\.' + 'name': 'meta.delimiter.method.period.js' + } + { + 'match': '\\?\\.' + 'name': 'meta.delimiter.method.optional.js' + } + ] diff --git a/packages/language-javascript/grammars/tree-sitter-javascript.cson b/packages/language-javascript/grammars/tree-sitter-javascript.cson index 86919162fc..465b0437b3 100644 --- a/packages/language-javascript/grammars/tree-sitter-javascript.cson +++ b/packages/language-javascript/grammars/tree-sitter-javascript.cson @@ -184,9 +184,13 @@ scopes: '"*="': 'keyword.operator.js' '"/="': 'keyword.operator.js' '"%="': 'keyword.operator.js' + '"**="': 'keyword.operator.js' '"<<="': 'keyword.operator.js' '">>="': 'keyword.operator.js' '">>>="': 'keyword.operator.js' + '"&&="': 'keyword.operator.js' + '"||="': 'keyword.operator.js' + '"??="': 'keyword.operator.js' '"&="': 'keyword.operator.js' '"^="': 'keyword.operator.js' '"|="': 'keyword.operator.js' @@ -196,6 +200,7 @@ scopes: '"*"': 'keyword.operator.js' '"/"': 'keyword.operator.js' '"%"': 'keyword.operator.js' + '"**"': 'keyword.operator.js' '"=="': 'keyword.operator.js' '"==="': 'keyword.operator.js' '"!="': 'keyword.operator.js' @@ -204,25 +209,28 @@ scopes: '"<="': 'keyword.operator.js' '">"': 'keyword.operator.js' '"<"': 'keyword.operator.js' - '":"': 'keyword.operator.js' - '"?"': 'keyword.operator.js' '"&&"': 'keyword.operator.js' '"||"': 'keyword.operator.js' + '"??"': 'keyword.operator.js' '"&"': 'keyword.operator.js' - '"~"': 'keyword.operator.js' '"^"': 'keyword.operator.js' + '"|"': 'keyword.operator.js' + '"~"': 'keyword.operator.js' '">>"': 'keyword.operator.js' '">>>"': 'keyword.operator.js' '"<<"': 'keyword.operator.js' - '"|"': 'keyword.operator.js' '"++"': 'keyword.operator.js' '"--"': 'keyword.operator.js' '"..."': 'keyword.operator.spread.js' + '"?"': 'keyword.operator.js' + '":"': 'keyword.operator.js' '"in"': 'keyword.operator.in' '"instanceof"': 'keyword.operator.instanceof' '"of"': 'keyword.operator.of' '"new"': 'keyword.operator.new' + '"delete"': 'keyword.operator.delete' + '"void"': 'keyword.operator.void' '"typeof"': 'keyword.operator.typeof' '"get"': 'keyword.operator.setter' @@ -254,7 +262,6 @@ scopes: '"async"': 'keyword.control' '"await"': 'keyword.control' '"debugger"': 'keyword.control' - '"delete"': 'keyword.control' 'jsx_attribute > property_identifier': 'entity.other.attribute-name' 'jsx_opening_element > identifier': 'entity.name.tag' diff --git a/packages/language-javascript/spec/javascript-spec.coffee b/packages/language-javascript/spec/javascript-spec.coffee index 192850ec22..15650559c2 100644 --- a/packages/language-javascript/spec/javascript-spec.coffee +++ b/packages/language-javascript/spec/javascript-spec.coffee @@ -554,7 +554,7 @@ describe "JavaScript grammar", -> expect(tokens[1]).toEqual value: '--', scopes: ['source.js', 'keyword.operator.decrement.js'] describe "logical", -> - operators = ["&&", "||", "!"] + operators = ["&&", "||", "!", "??"] it "tokenizes them", -> for operator in operators @@ -598,7 +598,7 @@ describe "JavaScript grammar", -> expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] describe "arithmetic", -> - operators = ["*", "/", "-", "%", "+"] + operators = ["*", "/", "-", "%", "+", "**"] it "tokenizes them", -> for operator in operators @@ -623,22 +623,39 @@ describe "JavaScript grammar", -> describe "compound", -> it "tokenizes them", -> - operators = ["+=", "-=", "*=", "/=", "%="] + operators = ["+=", "-=", "*=", "/=", "%=", "**="] for operator in operators {tokens} = grammar.tokenizeLine('a ' + operator + ' b') expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.assignment.compound.js'] expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + describe "logical", -> + it "tokenizes them", -> + operators = ["||=", "&&=", "??="] + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.assignment.compound.logical.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + describe "bitwise", -> it "tokenizes them", -> - operators = ["<<=", ">>=", ">>>=", "&=", "^=", "|="] + operators = ["&=", "^=", "|="] for operator in operators {tokens} = grammar.tokenizeLine('a ' + operator + ' b') expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.assignment.compound.bitwise.js'] expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + it "tokenizes bitwise shift", -> + operators = ["<<=", ">>=", ">>>="] + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.assignment.compound.bitwise.shift.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + describe "constants", -> it "tokenizes ALL_CAPS variables as constants", -> {tokens} = grammar.tokenizeLine('var MY_COOL_VAR = 42;') @@ -1604,9 +1621,13 @@ describe "JavaScript grammar", -> it "tokenizes the rest parameter", -> {tokens} = grammar.tokenizeLine('(...args) => args[0]') - expect(tokens[1]).toEqual value: '...', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: '...', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'keyword.operator.rest.js'] expect(tokens[2]).toEqual value: 'args', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.rest.function.js'] + {tokens} = grammar.tokenizeLine('(c, ...val) => c + val') + expect(tokens[4]).toEqual value: '...', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'keyword.operator.rest.js'] + expect(tokens[5]).toEqual value: 'val', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.rest.function.js'] + it "tokenizes illegal parameters", -> {tokens} = grammar.tokenizeLine('0abc => {}') expect(tokens[0]).toEqual value: '0abc', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'invalid.illegal.identifier.js'] @@ -1889,6 +1910,64 @@ describe "JavaScript grammar", -> expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] expect(tokens[2]).toEqual value: 'C', scopes: ['source.js', 'constant.other.property.js'] + it "tokenizes the optional chaining operator", -> + {tokens} = grammar.tokenizeLine('obj?.prop') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.delimiter.property.optional.js'] + expect(tokens[2]).toEqual value: 'prop', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('obj?.$_') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.delimiter.property.optional.js'] + expect(tokens[2]).toEqual value: '$_', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('a()?.b()') + expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[3]).toEqual value: '?.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.optional.js'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + + {tokens} = grammar.tokenizeLine('a()?.MY_CONSTANT') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '?.', scopes: ['source.js', 'meta.delimiter.property.optional.js'] + expect(tokens[4]).toEqual value: 'MY_CONSTANT', scopes: ['source.js', 'constant.other.property.js'] + + {tokens} = grammar.tokenizeLine('a.b?.()') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '?.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.optional.js'] + + {tokens} = grammar.tokenizeLine('a?.b?.()') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.optional.js'] + expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '?.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.optional.js'] + + {tokens} = grammar.tokenizeLine('a?.[3]') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.delimiter.property.optional.js'] + expect(tokens[2]).toEqual value: '[', scopes: ['source.js', 'meta.brace.square.js'] + + {tokens} = grammar.tokenizeLine('a()?.[1]') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '?.', scopes: ['source.js', 'meta.delimiter.property.optional.js'] + expect(tokens[4]).toEqual value: '[', scopes: ['source.js', 'meta.brace.square.js'] + + {tokens} = grammar.tokenizeLine('a[4]?.()') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[4]).toEqual value: '?.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.optional.js'] + + {tokens} = grammar.tokenizeLine('a[1]?.b()') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[4]).toEqual value: '?.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.optional.js'] + expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + + {tokens} = grammar.tokenizeLine('a[5]?.b') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[4]).toEqual value: '?.', scopes: ['source.js', 'meta.delimiter.property.optional.js'] + expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'variable.other.property.js'] + + describe "strings and functions", -> it "doesn't confuse them", -> {tokens} = grammar.tokenizeLine("'a'.b(':c(d)')") @@ -1951,11 +2030,6 @@ describe "JavaScript grammar", -> expect(tokens[1]).toEqual value: ' foo ', scopes: ['source.js', 'comment.block.documentation.js'] expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] - it "tokenizes // comments", -> - {tokens} = grammar.tokenizeLine('// comment') - expect(tokens[0]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] - expect(tokens[1]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] - it "tokenizes comments inside constant definitions", -> {tokens} = grammar.tokenizeLine('const a, // comment') expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] From aca05b5f9d63b32bfe70951e7a463944efb6bcf3 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Mon, 19 Dec 2022 14:10:01 -0600 Subject: [PATCH 2/6] update tests ?. is illegal in new update console methods timeStamp has the same support as profile and both are nonstandard so they're both included update global classes Wait a minute, Generator and GeneratorFunction are _not_ global... so actually just any class specified in ECMA262 non global additions: - AsyncFunction - AsyncGeneratorFunction - TypedArray luckily MDN's sidebar of https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/globalThis shows all of these Wait... JSON? That's not a class Math has the scope support.class.math.js for some reason so I guess add "json" likewise for json Never knew about WeakRef and FinalizationRegistry Wonder why it's so unknown; this is a common idea in Rust at least.... So yeah, JSON is separate just like Math Lots of copying and pasting rest vs spread depends on context I don't know why that test failed --- .../grammars/javascript.cson | 86 ++++++++++++++++--- .../spec/javascript-spec.coffee | 65 +++++++++++--- 2 files changed, 124 insertions(+), 27 deletions(-) diff --git a/packages/language-javascript/grammars/javascript.cson b/packages/language-javascript/grammars/javascript.cson index dccbbc2508..e418442015 100644 --- a/packages/language-javascript/grammars/javascript.cson +++ b/packages/language-javascript/grammars/javascript.cson @@ -1,5 +1,6 @@ -# Note: Because of the optional chaining operator (?.), "object.property" in the comments usually means ("object.property" or "object?.property") -# Even though sometimes it doesn't seem like using the chaining operator would make any sense, it's still supported. example: Math?.PI +# "?." means "?." or "." +# Even though sometimes it doesn't seem like using the chaining operator would make any sense, it's still supported. +# Example: Math?.PI # Negative look-behinds for "." should cover "?." 'scopeName': 'source.js' 'fileTypes': [ @@ -540,7 +541,7 @@ ] } { - # [.|?.]foo = ... => ... + # [?.]foo = ... => ... 'begin': '''(?x) (?= (\\?\\.|\\.)?[a-zA-Z_$][\\w$]* @@ -731,7 +732,7 @@ 'name': 'meta.class.js' } { - 'match': '(new)\\s+([\\w$]+[\\w.$]*)' + 'match': '(new)\\s+([\\w$]+(?:\\??\\.[\\w$]*)*)' 'name': 'meta.class.instance.constructor.js' 'captures': '1': @@ -740,7 +741,7 @@ 'name': 'entity.name.type.instance.js' 'patterns': [ { - 'include': '#period_or_optional_property' + 'include': '#period_or_illegal_optional_property' } ] } @@ -754,7 +755,9 @@ 'end': '''(?x) (?<=\\)) | (?= (?! (\\s*//)|(\\s*/\\*)|(\\s*(\\?\\.|\\.)\\s* - (assert|clear|debug|error|info|log|profile|profileEnd|time|timeEnd|warn) + (assert|clear|count|countReset|debug|dir|dirxml|error|group|groupCollapsed| + groupEnd|info|log|profile|profileEnd|table|time|timeEnd|timeLog|trace| + timeStamp|warn) \\s*\\( )) \\s*\\S ) @@ -784,6 +787,45 @@ } ] } + { + # JSON + 'begin': '(? describe "ES6 tagged Relay.QL string templates", -> it "tokenizes them as strings", -> {tokens} = grammar.tokenizeLine('Relay.QL`fragment on Foo { id }`') - expect(tokens[0]).toEqual value: 'Relay.QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] - expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] - expect(tokens[2]).toEqual value: 'fragment on Foo { id }', scopes: ['source.js', 'string.quoted.template.graphql.js'] - expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] + expect(tokens[0]).toEqual value: 'Relay', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'string.quoted.template.graphql.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[4]).toEqual value: 'fragment on Foo { id }', scopes: ['source.js', 'string.quoted.template.graphql.js'] + expect(tokens[5]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] describe "ES6 tagged Relay.QL string templates with interpolation", -> it "tokenizes them as strings", -> {tokens} = grammar.tokenizeLine('Relay.QL`fragment on Foo { ${myFragment} }`') - expect(tokens[0]).toEqual value: 'Relay.QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] - expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] - expect(tokens[2]).toEqual value: 'fragment on Foo { ', scopes: ['source.js', 'string.quoted.template.graphql.js'] - expect(tokens[3]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[4]).toEqual value: 'myFragment', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source'] - expect(tokens[5]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[6]).toEqual value: ' }', scopes: ['source.js', 'string.quoted.template.graphql.js'] - expect(tokens[7]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] + expect(tokens[0]).toEqual value: 'Relay', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'string.quoted.template.graphql.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[4]).toEqual value: 'fragment on Foo { ', scopes: ['source.js', 'string.quoted.template.graphql.js'] + expect(tokens[5]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[6]).toEqual value: 'myFragment', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[8]).toEqual value: ' }', scopes: ['source.js', 'string.quoted.template.graphql.js'] + expect(tokens[9]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] describe "ES6 tagged gql string templates", -> it "tokenizes them as strings", -> @@ -1925,7 +1929,7 @@ describe "JavaScript grammar", -> expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] expect(tokens[3]).toEqual value: '?.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.optional.js'] expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] - + {tokens} = grammar.tokenizeLine('a()?.MY_CONSTANT') expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] expect(tokens[3]).toEqual value: '?.', scopes: ['source.js', 'meta.delimiter.property.optional.js'] @@ -2150,6 +2154,41 @@ describe "JavaScript grammar", -> {tokens} = grammar.tokenizeLine('console .foo();') expect(tokens[3]).toEqual value: 'foo', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + describe "json", -> + it "tokenizes the json object", -> + {tokens} = grammar.tokenizeLine('JSON;') + expect(tokens[0]).toEqual value: 'JSON', scopes: ['source.js', 'support.class.json.js'] + expect(tokens[1]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes json support functions", -> + {tokens} = grammar.tokenizeLine('JSON.parse();') + expect(tokens[0]).toEqual value: 'JSON', scopes: ['source.js', 'support.class.json.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'parse', scopes: ['source.js', 'meta.method-call.js', 'support.function.json.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + lines = grammar.tokenizeLines ''' + JSON + .parse(); + ''' + expect(lines[0][0]).toEqual value: 'JSON', scopes: ['source.js', 'support.class.json.js'] + expect(lines[1][0]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[1][1]).toEqual value: 'parse', scopes: ['source.js', 'meta.method-call.js', 'support.function.json.js'] + expect(lines[1][2]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][3]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][4]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes json custom functions", -> + {tokens} = grammar.tokenizeLine('JSON.pi();') + expect(tokens[0]).toEqual value: 'JSON', scopes: ['source.js', 'support.class.json.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'pi', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + describe "math", -> it "tokenizes the math object", -> {tokens} = grammar.tokenizeLine('Math;') From 60f63ba9cabd08f46f5f67ac6abdc2c96efacb50 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 15 Feb 2023 07:35:30 -0600 Subject: [PATCH 3/6] fix tests given that the entire Relay.QL was entity.name.function, it's not surprising here Therefore this pr just adds "meta.delimiter.<>" to the delimiter in Relay.QL or Relay?.QL --- packages/language-javascript/spec/javascript-spec.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/language-javascript/spec/javascript-spec.coffee b/packages/language-javascript/spec/javascript-spec.coffee index 32ce383ddb..34422d4300 100644 --- a/packages/language-javascript/spec/javascript-spec.coffee +++ b/packages/language-javascript/spec/javascript-spec.coffee @@ -937,7 +937,7 @@ describe "JavaScript grammar", -> it "tokenizes them as strings", -> {tokens} = grammar.tokenizeLine('Relay.QL`fragment on Foo { id }`') expect(tokens[0]).toEqual value: 'Relay', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] - expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'string.quoted.template.graphql.js', 'meta.delimiter.method.period.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js', 'meta.delimiter.method.period.js'] expect(tokens[2]).toEqual value: 'QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] expect(tokens[4]).toEqual value: 'fragment on Foo { id }', scopes: ['source.js', 'string.quoted.template.graphql.js'] @@ -947,7 +947,7 @@ describe "JavaScript grammar", -> it "tokenizes them as strings", -> {tokens} = grammar.tokenizeLine('Relay.QL`fragment on Foo { ${myFragment} }`') expect(tokens[0]).toEqual value: 'Relay', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] - expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'string.quoted.template.graphql.js', 'meta.delimiter.method.period.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js', 'meta.delimiter.method.period.js'] expect(tokens[2]).toEqual value: 'QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] expect(tokens[4]).toEqual value: 'fragment on Foo { ', scopes: ['source.js', 'string.quoted.template.graphql.js'] From c7a90cbbee38f5609517767c03c442d176a88994 Mon Sep 17 00:00:00 2001 From: icecream17 Date: Tue, 16 May 2023 09:04:33 -0500 Subject: [PATCH 4/6] implement illegal ?. in assignment --- .../language-javascript/grammars/javascript.cson | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/language-javascript/grammars/javascript.cson b/packages/language-javascript/grammars/javascript.cson index e418442015..c302fcf4ed 100644 --- a/packages/language-javascript/grammars/javascript.cson +++ b/packages/language-javascript/grammars/javascript.cson @@ -309,7 +309,7 @@ '1': 'patterns': [ { - 'include': '#period_or_optional_method' + 'include': '#period_or_illegal_optional_method' } ] '2': @@ -577,7 +577,7 @@ '1': 'patterns': [ { - 'include': '#period_or_optional_method' + 'include': '#period_or_illegal_optional_method' } ] '2': @@ -2300,6 +2300,17 @@ 'name': 'invalid.illegal.meta.delimiter.property.optional.js' } ] + 'period_or_illegal_optional_method': + 'patterns': [ + { + 'match': '\\.' + 'name': 'meta.delimiter.method.period.js' + } + { + 'match': '\\?\\.' + 'name': 'invalid.illegal.meta.delimiter.method.optional.js' + } + ] 'period_or_optional_property': 'patterns': [ { From 4dde98536e8b6b92b39955f74c9ebee845edd522 Mon Sep 17 00:00:00 2001 From: icecream17 Date: Tue, 16 May 2023 11:58:00 -0500 Subject: [PATCH 5/6] test some illegal ?. --- .../spec/javascript-spec.coffee | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/language-javascript/spec/javascript-spec.coffee b/packages/language-javascript/spec/javascript-spec.coffee index 86ea0b40ae..124f368e87 100644 --- a/packages/language-javascript/spec/javascript-spec.coffee +++ b/packages/language-javascript/spec/javascript-spec.coffee @@ -1966,6 +1966,22 @@ describe "JavaScript grammar", -> expect(tokens[4]).toEqual value: '?.', scopes: ['source.js', 'meta.delimiter.property.optional.js'] expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'variable.other.property.js'] + it "detects some illegal uses of the optional chaining operator", -> + {tokens} = grammar.tokenizeLine('new a?.b') + expect(tokens[1]).toEqual value: 'a', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[2]).toEqual value: '?.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'invalid.illegal.meta.delimiter.property.period.js'] + expect(tokens[3]).toEqual value: 'b', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + + {tokens} = grammar.tokenizeLine('a?.b = function c () {}') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.function.js', 'invalid.illegal.meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine('a?.b = _ => 2') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.function.arrow.js', 'invalid.illegal.meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] describe "strings and functions", -> it "doesn't confuse them", -> From e3f1cb76c0a248766a19afd36e7b2bd95c6491fd Mon Sep 17 00:00:00 2001 From: icecream17 Date: Tue, 16 May 2023 13:09:34 -0500 Subject: [PATCH 6/6] fix tests --- .../language-javascript/spec/javascript-spec.coffee | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/language-javascript/spec/javascript-spec.coffee b/packages/language-javascript/spec/javascript-spec.coffee index 124f368e87..1d0c2857ba 100644 --- a/packages/language-javascript/spec/javascript-spec.coffee +++ b/packages/language-javascript/spec/javascript-spec.coffee @@ -1968,19 +1968,19 @@ describe "JavaScript grammar", -> it "detects some illegal uses of the optional chaining operator", -> {tokens} = grammar.tokenizeLine('new a?.b') - expect(tokens[1]).toEqual value: 'a', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] - expect(tokens[2]).toEqual value: '?.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'invalid.illegal.meta.delimiter.property.period.js'] - expect(tokens[3]).toEqual value: 'b', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[2]).toEqual value: 'a', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[3]).toEqual value: '?.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'invalid.illegal.meta.delimiter.property.optional.js'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] {tokens} = grammar.tokenizeLine('a?.b = function c () {}') expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] - expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.function.js', 'invalid.illegal.meta.delimiter.method.period.js'] + expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.function.js', 'invalid.illegal.meta.delimiter.method.optional.js'] expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'keyword.operator.assignment.js'] {tokens} = grammar.tokenizeLine('a?.b = _ => 2') expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] - expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.function.arrow.js', 'invalid.illegal.meta.delimiter.method.period.js'] + expect(tokens[1]).toEqual value: '?.', scopes: ['source.js', 'meta.function.arrow.js', 'invalid.illegal.meta.delimiter.method.optional.js'] expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] describe "strings and functions", ->