swift-mode-lexer.el

;;; swift-mode-lexer.el --- Major-mode for Apple's Swift programming language, lexer. -*- lexical-binding: t -*-

;; Copyright (C) 2014-2021 taku0, Chris Barrett, Bozhidar Batsov,
;;                         Arthur Evstifeev

;; Author: taku0 (http://github.com/taku0)
;;       Chris Barrett <chris.d.barrett@me.com>
;;       Bozhidar Batsov <bozhidar@batsov.com>
;;       Arthur Evstifeev <lod@pisem.net>

;; This file is not part of GNU Emacs.

;; This program is free software: you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.

;; This program is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
;; along with this program.  If not, see <http://www.gnu.org/licenses/>.

;;; Commentary:

;; Routines for Swift tokens

;; Token is a tuple consists of:
;;
;; - Token type
;; - Token text
;; - Start position (inclusive)
;; - End position (exclusive)

;;; Code:

;; Terminology:
;; (See also docs/string.png)
;;
;;   Interpolated string:
;;     A string containing expressions to be evaluated and inserted into the
;;     string at run time.
;;     Example: "1 + 1 = \(1 + 1)" is evaluated to "1 + 1 = 2" at run time.
;;     https://developer.apple.com/library/content/documentation/Swift/Conceptual/Swift_Programming_Language/StringsAndCharacters.html
;;     https://developer.apple.com/library/content/documentation/Swift/Conceptual/Swift_Programming_Language/LexicalStructure.html
;;
;;   Interpolated expression:
;;     An expression between \( and ) inside a string.
;;     Suppose a string "aaa\( foo() )bbb\( bar() )ccc",
;;     `foo()' and `bar()' are interpolated expression.
;;     https://developer.apple.com/library/content/documentation/Swift/Conceptual/Swift_Programming_Language/LexicalStructure.html
;;     Why not interpolating expression?
;;
;;   String chunk:
;;     A part of single-line/multiline string delimited with quotation marks
;;     or interpolated expressions.
;;     Suppose a string "aaa\( foo() )bbb\( bar() )ccc",
;;     "aaa\(, )bbb\(, and )ccc" are string chunks.
;;
;;     This is not a official term; used only in swift-mode.

(declare-function swift-mode:backward-sexps-until "swift-mode-indent.el"
                  (token-types
                   &optional
                   stop-at-eol-token-types
                   stop-at-bol-token-types))

(declare-function swift-mode:try-backward-generic-parameters
                  "swift-mode-indent.el"
                  ())


(defun swift-mode:token (type text start end)
  "Construct and return a token.

TYPE is the type of the token such as `binary-operator' or {.
TEXT is the text of the token.
START is the start position of the token.
END is the point after the token."
  (list type text start end))

(defun swift-mode:token:type (token)
  "Return the type of TOKEN."
  (nth 0 token))

(defun swift-mode:token:text (token)
  "Return the text of TOKEN."
  (nth 1 token))

(defun swift-mode:token:start (token)
  "Return the start position of TOKEN."
  (nth 2 token))

(defun swift-mode:token:end (token)
  "Return the end position of TOKEN."
  (nth 3 token))

;; Token types is one of the following symbols:
;;
;; - prefix-operator (including try, try?, try!, await, consume, copy, discard,
;;   and each)
;; - postfix-operator
;; - binary-operator (including as, as?, as!, is, =, ., and ->)
;; - attribute (e.g. @objc, @abc(def))
;; - identifier (including keywords, numbers, implicit parameters, and unknown
;;   tokens)
;; - [
;; - ]
;; - {
;; - }
;; - (
;; - )
;; - ,
;; - ;
;; - implicit-;
;; - < (as an angle bracket)
;; - > (as an angle bracket)
;; - supertype-: (colon for supertype declaration or type declaration of let or
;;   var)
;; - case-: (colon for case or default label)
;; - : (part of conditional operator, key-value separator, label-statement
;;   separator)
;; - anonymous-function-parameter-in ("in" after anonymous function parameter)
;; - string-chunk-after-interpolated-expression (part of a string starting with
;;   ")")
;; - string-chunk-before-interpolated-expression (part of a string ending with
;;   "\\(")
;; - outside-of-buffer
;;
;; Additionally, `swift-mode:backward-token-or-list' may return a parenthesized
;; expression as a token with one of the following types:
;; - ()
;; - []
;; - {}
;; - <>

;;; Syntax table

(defconst swift-mode:syntax-table
  (let ((table (make-syntax-table)))
    ;; Whitespace characters
    ;; Word constituents

    ;; Symbol constituents
    (modify-syntax-entry ?_ "_" table)
    (modify-syntax-entry ?$ "_" table)
    (modify-syntax-entry ?@ "_" table)
    (modify-syntax-entry ?# "_" table)

    ;; Punctuation characters
    ;;
    ;; Operators
    ;; https://developer.apple.com/library/ios/documentation/Swift/Conceptual/Swift_Programming_Language/LexicalStructure.html#//apple_ref/doc/uid/TP40014097-CH30-ID410
    ;; TODO Unicode operators
    ;;
    ;; / and * will be overridden below as comment delimiters
    (mapc (lambda (c) (modify-syntax-entry c "." table)) "/=-+!*%<>&|^~?.")
    ;; Separators
    (mapc (lambda (c) (modify-syntax-entry c "." table)) ",;")

    ;; Parenthesis characters
    (modify-syntax-entry ?\( "()" table)
    (modify-syntax-entry ?\) ")(" table)
    (modify-syntax-entry ?\[ "(]" table)
    (modify-syntax-entry ?\] ")[" table)
    (modify-syntax-entry ?{ "(}" table)
    (modify-syntax-entry ?} "){" table)

    ;; String quotes
    (modify-syntax-entry ?\" "\"" table)
    (modify-syntax-entry ?` "\"" table)

    ;; Escape-syntax characters
    (modify-syntax-entry ?\\ "\\" table)

    ;; Character quotes
    ;; Paired delimiters
    ;; Expression prefixes

    ;; Comments
    (modify-syntax-entry ?/  ". 124b" table)
    (modify-syntax-entry ?*  ". 23n"  table)
    (modify-syntax-entry ?\n "> b"    table)
    (modify-syntax-entry ?\r "> b"    table)

    table))

;;; Text properties

;; (See also docs/string_properties.png)
;;
;; Some properties are put by `syntax-propertize-function', that is
;; `swift-mode:syntax-propertize'.
;;
;; The beginning of and end of strings are marked with text property
;; '(syntax-table (15)), which indicates generic string delimiters. Both
;; single-line strings and multiline strings are marked with it. The
;; parentheses surrounding interpolated expressions are also marked with
;; '(syntax-table (15)). The property helps font-lock and the tokenizer to
;; recognize strings.
;;
;; The entire string, including interpolated expressions, are marked with text
;; property '(syntax-multiline t). The property is used by
;; `swift-mode:syntax-propertize-extend-region' to avoid scanning from the
;; middle of strings.
;;
;; The parentheses surrounding interpolated expressions have text property
;; '(swift-mode:matching-parenthesis POS), where POS is the position of the
;; matching parenthesis. Strictly speaking, the POS on the closing parenthesis
;; refers to the backslash before the opening parenthesis. The property speeds
;; up the indentation logic.

(defun swift-mode:syntax-propertize-extend-region (start end)
  "Return region to be propertized.
The returned region contains the region (START . END).
If the region is not modified, return nil.
Intended for `syntax-propertize-extend-region-functions'."
  (syntax-propertize-multiline start end))

(defun swift-mode:syntax-propertize (start end)
  "Update text properties for strings.
Mark the beginning of and the end of single-line/multiline strings and regexes
between the position START and END as general string delimiters.
Intended for `syntax-propertize-function'."
  (remove-text-properties start end
                          '(syntax-table
                            nil
                            syntax-multiline
                            nil
                            swift-mode:matching-parenthesis
                            nil
                            swift-mode:comment
                            nil))
  (let* ((chunk (swift-mode:chunk-after (syntax-ppss start)))
         comment-start)
    (cond
     ((swift-mode:chunk:multiline-string-p chunk)
      (swift-mode:syntax-propertize:end-of-string
       end "\"\"\"" (swift-mode:chunk:pound-count chunk)))

     ((swift-mode:chunk:single-line-string-p chunk)
      (swift-mode:syntax-propertize:end-of-string
       end "\"" (swift-mode:chunk:pound-count chunk)))

     ((swift-mode:chunk:regex-p chunk)
      (swift-mode:syntax-propertize:end-of-regex
       (swift-mode:chunk:start chunk)))

     ((swift-mode:chunk:comment-p chunk)
      (goto-char (swift-mode:chunk:start chunk))
      (setq comment-start (point))
      (swift-mode:forward-comment 1 end)
      (put-text-property comment-start (point) 'swift-mode:comment t))))
  (swift-mode:syntax-propertize:scan end 0))

(defun swift-mode:syntax-propertize:scan (end nesting-level)
  "Update text properties for strings.
Mark the beginning of and the end of single-line/multiline strings and regexes
between the current position and END as general string delimiters.
Assuming the cursor is not on strings, regexes, nor comments.
If NESTING-LEVEL is non-zero, nesting of parentheses are tracked and the scan
stops where the level becomes zero."
  (let ((found-matching-parenthesis nil)
        (pattern (mapconcat #'regexp-quote
                            '("\"\"\"" "\"" "/" "(" ")")
                            "\\|")))
    (while (and (not found-matching-parenthesis)
                (< (point) end)
                (search-forward-regexp pattern end t))
      (cond
       ((member (match-string-no-properties 0) '("\"\"\"" "\""))
        (let ((start (match-beginning 0))
              (quotation (match-string-no-properties 0))
              pound-count)
          (save-excursion
            (goto-char start)
            (skip-chars-backward "#")
            (setq pound-count (- start (point)))
            (setq start (point)))
          (put-text-property start (1+ start)
                             'syntax-table
                             (string-to-syntax "|"))
          (swift-mode:syntax-propertize:end-of-string
           end quotation pound-count)
          (swift-mode:put-syntax-multiline-property start (point))))

       ((equal "/" (match-string-no-properties 0))
        (let ((start (match-beginning 0))
              regex-start)
          (save-excursion
            (goto-char start)
            (skip-chars-backward "#")
            (setq regex-start (point)))
          (cond
           ;; Regexes
           ((swift-mode:syntax-propertize:end-of-regex regex-start)
            (put-text-property regex-start (1+ regex-start)
                               'syntax-table
                               (string-to-syntax "|"))
            (swift-mode:put-syntax-multiline-property regex-start (point)))

           ;; Comments
           ((memq (char-after) '(?/ ?*))
            (goto-char start)
            (swift-mode:forward-comment 1 end)
            (put-text-property start (point) 'swift-mode:comment t))

           ;; Operators
           (t nil))))

       ((and (equal "(" (match-string-no-properties 0))
             (/= nesting-level 0))
        (setq nesting-level (1+ nesting-level)))

       ((and (equal ")" (match-string-no-properties 0))
             (/= nesting-level 0))
        (setq nesting-level (1- nesting-level))
        (when (= nesting-level 0)
          (setq found-matching-parenthesis t)))))
    (unless found-matching-parenthesis
      (goto-char end))
    found-matching-parenthesis))

(defun swift-mode:forward-comment (count &optional end)
  "Move point after spaces and COUNT comments.

If END is given, do not move beyond END.

COUNT must be positive.

Multiline comments can be nested.

If the comment is not closed, move to END if given, or `point-max' if it
is nil.

This function doesn't depend on the syntax tables, so that it works in
`swift-mode:syntax-propertize'."
  (while (< 0 count)
    (skip-chars-forward "\s\t\n")
    (cond
     ;; Single line comment
     ((and (eq (char-after) ?/)
           (eq (char-after (1+ (point))) ?/))
      (forward-line))

     ;; Multiline comment
     ((and (eq (char-after) ?/)
           (eq (char-after (1+ (point))) ?*))
      (swift-mode:forward-multiline-comment (or end (point-max))))

     ;; Not a comment
     (t
      (setq count 1)))
    (setq count (1- count))))

(defun swift-mode:forward-multiline-comment (end)
  "Move point after a multiline comment.

Assuming the point is just before a multiline comment.

Do not move beyond END.

Multiline comments can be nested.

If the comment is not closed, move to END."
  (forward-char 2)
  (let ((pattern (mapconcat #'regexp-quote '("/*" "*/") "\\|"))
        (nesting-level 1))
    (while (and (< 0 nesting-level)
                (< (point) end)
                (search-forward-regexp pattern end t))
      (if (eq (char-before) ?*)
          (setq nesting-level (1+ nesting-level))
        (setq nesting-level (1- nesting-level))))
    (when (or (and (not (zerop nesting-level))
                   (< (point) end))
              (< end (point)))
      (goto-char end))))

(defun swift-mode:put-syntax-multiline-property (start end)
  "Put `syntax-multiline` text propery from START to END.

Also call `font-lock-flush' with START and END."
  (put-text-property start end 'syntax-multiline t)
  (if (fboundp 'font-lock-flush)
      (font-lock-flush start end)
    (if (eq font-lock-fontify-buffer-function #'jit-lock-refontify)
        (jit-lock-refontify start end)
      (font-lock-after-change-function start end (- end start)))))

(defun swift-mode:syntax-propertize:end-of-string (end quotation pound-count)
  "Move point to the end of single-line/multiline string.

Assuming the cursor is on a string.
If the string go beyond END, stop there.
The string should be terminated with QUOTATION, followed by POUND-COUNT of
pound signs."
  (if (and
       (< (point) end)
       (search-forward-regexp (concat (regexp-quote quotation) "\\|(") end t))
      (cond
       ((and (equal quotation (match-string-no-properties 0))
             (not (swift-mode:escaped-p (match-beginning 0) pound-count))
             (progn
               (skip-chars-forward "#" (min end (+ (point) pound-count)))
               (= (- (point) (match-end 0)) pound-count)))
        (put-text-property (1- (point)) (point)
                           'syntax-table
                           (string-to-syntax "|")))

       ((and (equal "(" (match-string-no-properties 0))
             (swift-mode:escaped-p (match-beginning 0) pound-count))
        ;; Found an interpolated expression. Skips the expression.
        ;; We cannot use `scan-sexps' because multiline strings are not yet
        ;; propertized.
        (let ((pos-after-open-paren (point))
              (start
               (save-excursion
                 (backward-char) ;; (
                 (skip-chars-backward "#")
                 (backward-char) ;; \
                 (point))))
          ;; Declares the backslash is not a escape-syntax characters.
          (put-text-property start (1+ start)
                             'syntax-table
                             (string-to-syntax "w"))
          ;; Declares the open parentheses is a generic string delimiter.
          (put-text-property (1- pos-after-open-paren) pos-after-open-paren
                             'syntax-table
                             (string-to-syntax "|"))
          (when (swift-mode:syntax-propertize:scan end 1)
            ;; Found the matching parenthesis. Going further.
            ;; Declares the close parentheses is a generic string delimiter.
            (put-text-property (1- (point)) (point)
                               'syntax-table
                               (string-to-syntax "|"))
            ;; Records the positions.
            (put-text-property (1- (point)) (point)
                               'swift-mode:matching-parenthesis
                               start)
            (put-text-property start pos-after-open-paren
                               'swift-mode:matching-parenthesis
                               (1- (point)))
            (swift-mode:syntax-propertize:end-of-string
             end quotation pound-count))))

       (t
        (swift-mode:syntax-propertize:end-of-string end quotation pound-count)))
    (goto-char end)))


(defun swift-mode:escaped-p (position pound-count)
  "Return t if the POSITION in a string is escaped.

A position is escaped if it is proceeded by POUND-COUNT or more of pound signs
and odd number of backslashes.
Return nil otherwise."
  (let ((p position)
        (backslash-count 0))
    (while (eq (char-before p) ?#)
      (setq p (1- p)))
    (and
     ;; While it is a syntax error to have extra pound signs, we allow them
     ;; here to prevent corruption.
     (<= pound-count (- position p))
     (progn
       (while (eq (char-before p) ?\\)
         (setq backslash-count (1+ backslash-count))
         (setq p (1- p)))
       (= (mod backslash-count 2) 1)))))

(defun swift-mode:syntax-propertize:end-of-regex (start)
  "Move point to the end of regex if any.

START is the position of the open delimiter, including pounds if any.

If START is not a start of a regex, keep the point and return nil.  Otherwise,
return non-nil.

This function doesn't take end parameter since if the closing delimiter is
missing, this function must return nil."
  (let* ((pound-count (save-excursion
                        (goto-char start)
                        (skip-chars-forward "#")
                        (- (point) start)))
         end-of-regex)
    (setq end-of-regex
          (if (zerop pound-count)
              (swift-mode:syntax-propertize:end-of-basic-regex start)
            (swift-mode:syntax-propertize:end-of-extended-regex
             start
             pound-count)))
    (when end-of-regex
      (put-text-property (1- end-of-regex) end-of-regex
                         'syntax-table
                         (string-to-syntax "|")))
    end-of-regex))

(defun swift-mode:syntax-propertize:end-of-basic-regex (start)
  "Move point to the end of regex if any.

START is the position of the open delimiter.

If START is not a start of a regex, keep the point and return nil.  Otherwise,
return non-nil."
  (let* ((pos (point))
         (start-of-contents (1+ start))
         after-last-dot
         (square-brackets-count 0)
         (parentheses-count 0)
         (limit (line-end-position))
         (end-of-regex nil))
    (if (or
         ;; Cannot starts with spaces, tabs, slashes, or asterisks.
         (memq (char-after start-of-contents) '(?\s ?\t ?/ ?*))
         ;; Cannot be a comment closer: /**/+++/.
         (get-text-property start 'swift-mode:comment)
         ;; Cannot be preceded with infix operators while it can be preceded
         ;; with prefix operators.
         (save-excursion
           (goto-char start)
           ;; TODO Unicode operators
           (skip-chars-backward "-/=+!*%<>&|^~?")
           (when (eq (char-before) ?.)
             (setq after-last-dot (point))
             (skip-chars-backward "-/=+!*%<>&|^~?.")
             (unless (eq (char-after) ?.)
               (goto-char (1- after-last-dot))))
           (and
            ;; preceded with an operator
            (/= start (point))
            ;; it is not a prefix operator
            (not (memq (char-before)
                       '(nil ?\s ?\t ?\[ ?\( ?{ ?, ?\; ?:)))
            ;; it does't contain comments: a/**/+/**//b /
            (not (text-property-any (point) start 'swift-mode:comment t)))))
        nil
      (goto-char start-of-contents)
      (while (and (null end-of-regex)
                  (search-forward-regexp "[][()\\/]" limit t))
        (cond
         ((eq (char-before) ?\\)
          (forward-char))
         ((eq (char-before) ?\[)
          (setq square-brackets-count (1+ square-brackets-count)))
         ((eq (char-before) ?\])
          (when (< 0 square-brackets-count)
            (setq square-brackets-count (1- square-brackets-count))))
         ((eq (char-before) ?\()
          (when (zerop square-brackets-count)
            (setq parentheses-count (1+ parentheses-count))))
         ((eq (char-before) ?\))
          (cond
           ((< 0 square-brackets-count)
            nil)
           ((zerop parentheses-count)
            ;; Found an unmatching close parenthesis.  This is not a regex
            ;; literal.
            (goto-char limit))
           (t
            (setq parentheses-count (1- parentheses-count)))))
         ((eq (char-before) ?/)
          (if (memq (char-after) '(?/ ?*))
              (goto-char limit)
            (setq end-of-regex (point)))))))
    (when (and end-of-regex
               (memq (char-before (1- end-of-regex)) '(?\s ?\t))
               (not (swift-mode:escaped-p (- end-of-regex 2) 0)))
      ;; Cannot ends with spaces or tabs unless escaped.
      (setq end-of-regex nil))
    (unless end-of-regex
      (goto-char pos))
    end-of-regex))

(defun swift-mode:syntax-propertize:end-of-extended-regex (start pound-count)
  "Move point to the end of extended regex if any.

START is the position of the open delimiter, including pounds of POUND-COUNT.

If START is not a start of a regex, keep the point and return nil.  Otherwise,
return non-nil."
  (let* ((pos (point))
         (start-of-contents (1+ (+ start pound-count)))
         (starts-with-line-break (save-excursion
                                   (goto-char start-of-contents)
                                   (skip-chars-forward "\s\t")
                                   (eolp)))
         (end-of-regex nil))
    (goto-char start-of-contents)
    (if starts-with-line-break
        (while (and (null end-of-regex)
                    (zerop (forward-line)))
          (skip-chars-forward "\s\t")
          (when (and (eq (char-after) ?/)
                     (progn
                       (forward-char)
                       (eq (skip-chars-forward "#" (+ (point) pound-count))
                           pound-count)))
            (setq end-of-regex (point))))
      (while (and (null end-of-regex)
                  (search-forward-regexp "/#" (line-end-position) t))
        (backward-char)
        (when (and (eq (skip-chars-forward "#" (+ (point) pound-count))
                       pound-count)
                   ;; Inside regex literal, backslashes without pounds are
                   ;; still special.
                   (not (swift-mode:escaped-p (match-beginning 0) 0)))
          (setq end-of-regex (point)))))
    (unless end-of-regex
      (swift-mode:put-syntax-multiline-property start (point))
      (goto-char pos))
    end-of-regex))

;;; Lexers

(defun swift-mode:implicit-semi-p ()
  "Return t if the cursor is after the end of a statement."
  (let
      ((previous-token (save-excursion
                         (swift-mode:backquote-identifier-if-after-dot
                          (swift-mode:backward-token-simple))))
       (next-token (save-excursion
                     (swift-mode:backquote-identifier-if-after-dot
                      (swift-mode:forward-token-simple)))))
    ;; If the cursor is on the empty line, pretend an identifier is on the line.
    (when (and
           (< (swift-mode:token:end previous-token) (line-beginning-position))
           (< (line-end-position) (swift-mode:token:start next-token)))
      (setq next-token (swift-mode:token 'identifier "" (point) (point))))
    (cond
     ((or
       ;; Suppresses implicit semicolon around binary operators and separators.
       (memq (swift-mode:token:type previous-token)
             '(binary-operator \; \, :))
       (memq (swift-mode:token:type next-token)
             '(binary-operator \; \, :))

       ;; Suppresses implicit semicolon after try, try?, try!, and await.
       (member (swift-mode:token:text previous-token)
               '("try" "try?" "try!" "await"))

       ;; Suppress implicit semicolon after open brackets or before close
       ;; brackets.
       (memq (swift-mode:token:type previous-token) '({ \( \[))
       (memq (swift-mode:token:type next-token) '(} \) \]))

       ;; Supress implicit semicolon before/after open angle bracket.
       (and (equal (swift-mode:token:text previous-token) "<")
            (save-excursion
              (goto-char (swift-mode:token:start previous-token))
              (swift-mode:generic-parameter-clause-start-p)))
       (and (equal (swift-mode:token:text next-token) "<")
            (save-excursion
              (goto-char (swift-mode:token:start next-token))
              (swift-mode:generic-parameter-clause-start-p)))

       ;; Suppress implicit semicolon after/before string chunks inside
       ;; interpolated expressions.
       (eq (swift-mode:token:type previous-token)
           'string-chunk-before-interpolated-expression)
       (eq (swift-mode:token:type next-token)
           'string-chunk-after-interpolated-expression)

       ;; Suppress implicit semicolon around keywords that cannot start or end
       ;; statements.
       (member (swift-mode:token:text previous-token)
               '("any" "some" "inout" "borrowing" "consuming" "sending" "in"
                 "where" "isolated" "each"))
       (member (swift-mode:token:text next-token)
               '("any" "some" "inout" "borrowing" "consuming" "sending" "throws"
                 "rethrows" "in" "where" "isolated" "each"))

       ;; Suppress implicit semicolon between throws and open parenthesis.
       (and (equal (swift-mode:token:text previous-token) "throws")
            (eq (swift-mode:token:type next-token) '\()))
      nil)

     ;; Before async
     ;;
     ;; Examples:
     ;;
     ;; func foo() async throws -> Void
     ;; foo { () async throws -> void in }
     ;; let f: () async throws -> Void = g
     ;; get async throws {}
     ;; async let x = foo()
     ;;
     ;; Suppresses implicit semicolon if and only if before let.
     ;;
     ;; Example:
     ;;
     ;; let a = f as (Int, Int)
     ;;   async -> Int
     ;; let b = t as (Int, Int)
     ;; async
     ;;   let c = 1
     ((equal (swift-mode:token:text next-token) "async")
      (equal (swift-mode:token:text (save-excursion
                                      (swift-mode:forward-token-simple)
                                      (swift-mode:forward-token-simple)))
             "let"))

     ;; After async
     ;;
     ;; Suppresses implicit semicolon if before let.
     ((and (equal (swift-mode:token:text previous-token) "async")
           (equal (swift-mode:token:text next-token) "let"))
      nil)

     ;; Suppress implicit semicolon around else
     ((or
       (equal (swift-mode:token:text previous-token) "else")
       (equal (swift-mode:token:text next-token) "else"))
      nil)

     ;; Inserts semicolon before open curly bracket.
     ;;
     ;; Open curly bracket may continue the previous line, but we do not indent
     ;; there. For example, the code below is parsed as `(foo() { x in ... })'
     ;; by the Swift compiler, but we indent it like `foo(); { x in ... }'.
     ;;
     ;; foo()
     ;; { // does not indent here
     ;;   x in
     ;;   ...
     ;; }
     ((eq (swift-mode:token:type next-token) '\{) t)

     ;; Suppress implicit semicolon after attributes.
     ((eq (swift-mode:token:type previous-token) 'attribute)
      nil)

     ;; Suppress implicit semicolon after modifiers.
     ((member (swift-mode:token:text previous-token)
              '("indirect" "convenience" "dynamic" "final" "infix" "lazy"
                "mutating" "nonmutating" "optional" "override" "postfix"
                "prefix" "required" "static" "unowned" "weak" "internal"
                "package" "private" "public" "open" "fileprivate" "nonisolated"
                "distributed"))
      nil)

     ;; internal(set) private(set) public(set) open(set) fileprivate(set)
     ;; unowned(safe) unowned(unsafe) nonisolated(unsafe)
     ((and
       (eq (swift-mode:token:type previous-token) '\))
       (save-excursion
         (and
          (eq (swift-mode:token:type (swift-mode:backward-token-simple)) '\))
          (member (swift-mode:token:text (swift-mode:backward-token-simple))
                  '("set" "safe" "unsafe"))
          (eq (swift-mode:token:type (swift-mode:backward-token-simple)) '\()
          (member (swift-mode:token:text
                   (swift-mode:backquote-identifier-if-after-dot
                    (swift-mode:backward-token-simple)))
                  '("unowned" "internal" "private" "public" "open"
                    "fileprivate" "nonisolated")))))
      nil)

     ;; Suppress implicit semicolon after declaration starters.
     ((member (swift-mode:token:text previous-token)
              '("class" "struct" "actor" "protocol" "enum" "extension" "func"
                "typealias" "associatedtype" "precedencegroup" "operator"
                "macro"))
      nil)

     ;; Insert implicit semicolon before modifiers.
     ;;
     ;; Preceding modifiers takes precedence over this.
     ((member (swift-mode:token:text next-token)
              '("indirect" "convenience" "dynamic" "final" "infix" "lazy"
                "mutating" "nonmutating" "optional" "override" "postfix"
                "prefix" "required" "static" "unowned" "weak" "internal"
                "package" "private" "public" "open" "fileprivate" "nonisolated"
                "distributed"))
      t)

     ;; Inserts implicit semicolon around keywords that forms single keyword
     ;; statements.
     ((or
       (member (swift-mode:token:text previous-token)
               '("break" "continue" "fallthrough"))
       (member (swift-mode:token:text next-token)
               '("break" "continue" "fallthrough")))
      t)

     ;; Suppress implicit semicolon after keywords that cannot end statements.
     ((member (swift-mode:token:text previous-token)
              '("while" "for" "switch" "case" "default" "catch" "if" "guard"
                "let" "var" "throw" "import"))
      nil)

     ;; Suppress import semicolon after `repeat' unless followed by a open
     ;; curly bracket.
     ((and (equal (swift-mode:token:text previous-token) "repeat")
           (not (eq (swift-mode:token:type next-token) '{)))
      nil)

     ;; Inserts implicit semicolon before keywords that starts a new
     ;; statement.
     ((member (swift-mode:token:text next-token)
              '("for" "repeat" "case" "default" "defer" "do"
                "guard" "let" "var" "throw" "import" "return"))
      t)

     ;; Suppress implicit semicolon after return.
     ((equal (swift-mode:token:text previous-token) "return")
      nil)

     ;; Inserts implicit semicolon before `while' unless it is part of
     ;; `repeat...while'.
     ((equal (swift-mode:token:text next-token) "while")
      (save-excursion
        (not
         (and
          (eq (swift-mode:token:type previous-token) '\})
          (progn
            (backward-list)
            (equal (swift-mode:token:text
                    (swift-mode:backquote-identifier-if-after-dot
                     (swift-mode:backward-token-simple)))
                   "repeat"))))))

     ;; Inserts implicit semicolon before keywords that behave like method
     ;; names.
     ((member (swift-mode:token:text next-token)
              '("get" "set" "willSet" "didSet" "subscript" "init" "deinit"))
      t)

     ;; Inserts implicit semicolon before declaration starters.
     ;; Modifiers take precedence over this.
     ;;
     ;; Notes that class-requirement is handled by the `:' rule above:
     ;;
     ;; protocol Foo: // not insert semicolon here
     ;;   class
     ;;
     ;; `protocol' is handled by the next rule
     ((member (swift-mode:token:text next-token)
              '("class" "struct" "actor" "enum" "extension" "func" "typealias"
                "associatedtype" "precedencegroup" "macro"))
      t)

     ;; Inserts implicit semicolon before protocol unless it is followed by <.
     ((equal "protocol" (swift-mode:token:text next-token))
      (not (equal (swift-mode:token:text
                   (save-excursion
                     (swift-mode:forward-token-simple)
                     (swift-mode:forward-token-simple)))
                  "<")))

     ;; Suppress implicit semicolon after keywords that behave like method
     ;; names.
     ;;
     ;; Note that binary operators take precedence over this:
     ;;
     ;; self . // not insert semicolon here
     ;;   init
     ;;
     ;; var x {
     ;;   set // not insert semicolon here
     ;;     (x) {
     ;;   }
     ;; }
     ;;
     ;; var x {
     ;;   set // inserts semicolon here
     ;;   {
     ;;   }
     ;; }
     ((member (swift-mode:token:text previous-token)
              '("set" "willSet" "didSet" "subscript" "init" "deinit"))
      nil)

     ;; Inserts implicit semicolon before open square bracket.
     ;;
     ;; Open square bracket for array indexing cannot appear at the start of a
     ;; line.
     ;; https://github.com/apple/swift/blob/8d4b1cc3c47c7624d57f188d5b227152ccb03163/lib/Parse/ParseExpr.cpp#L1525
     ;;
     ;; Note that the following pattern (i.e. after binary-operator) is handled
     ;; by above case.
     ;;
     ;; let x =
     ;;   [
     ;;     1
     ;;   ]
     ((eq (swift-mode:token:type next-token) '\[) t)

     ;; Inserts implicit semicolon before open parenthesis, unless it is a
     ;; function parameter clause. Suppress implicit semicolon before function
     ;; parameter clause.
     ;;
     ;; Open parenthesis for function arguments cannot appear at the start of a
     ;; line.
     ;; https://github.com/apple/swift/blob/8d4b1cc3c47c7624d57f188d5b227152ccb03163/lib/Parse/ParseExpr.cpp#L1251
     ;;
     ;; Note that the following pattern (i.e. after binary-operator) is handled
     ;; by above case.
     ;;
     ;; let x =
     ;;   (
     ;;     1
     ;;   )
     ((eq (swift-mode:token:type next-token) '\()
      (not (swift-mode:function-parameter-clause-p)))

     ;; Suppress implicit semicolon after the beginning of an interpolated
     ;; expression.
     ((eq (swift-mode:token:type previous-token)
          'string-chunk-before-interpolated-expression)
      nil)

     ;; Otherwise, inserts implicit semicolon.
     (t t))))

(defun swift-mode:function-parameter-clause-p ()
  "Return t if the cursor is before a function/macro parameter clause.

Return nil otherwise."
  (save-excursion
    (let* ((previous-token (swift-mode:backward-token-simple))
           (previous-type (swift-mode:token:type previous-token)))
      (cond
       ((eq previous-type '>)
        (and
         (/= (point)
             ;; FIXME: mutual dependency
             (progn (swift-mode:try-backward-generic-parameters) (point)))
         (swift-mode:function-parameter-clause-p)))
       ((eq previous-type 'identifier)
        (member (swift-mode:token:text (swift-mode:backward-token-simple))
                '("func" "macro")))
       (t nil)))))

(defun swift-mode:supertype-colon-p ()
  "Return t if a colon at the cursor is the colon for supertype.

That is supertype declaration or type declaration of let or var."
  (save-excursion
    (let ((previous-token (swift-mode:backward-token-simple)))
      ;; class Foo<T>: Bar ← supertype colon
      ;; class Foo<T> : Bar ← supertype colon
      ;; class Foo<T where T: Bar<[(Int, String)]>> : Bar ← supertype colon
      ;; case Foo: ← not a supertype colon
      ;; case Foo where foo: ← not a supertype colon
      ;; case let Foo(x) where x is Foo<Int>: ← not a supertype colon
      ;; default: ← not a supertype colon
      ;; foo ? bar : baz ← not a supertype colon
      ;; [
      ;;   foo: ← not a supertype colon
      ;;     bar
      ;; ]
      ;; foo(bar, baz: baz) ← not a supertype colon
      ;; protocol Foo {
      ;;   associatedtype Bar<A>: Baz ← supertype colon
      ;; }
      (or
       ;; FIXME case let Foo(x) where x is Foo<Int>
       (eq (swift-mode:token:type previous-token) '>)
       ;; class Foo: ← supertype colon
       ;; extension Foo: ← supertype colon
       ;; let foo: ← not a supertype colon
       ;; var foo: ← not a supertype colon
       ;; protocol Foo {
       ;;   associatedtype Bar: Baz ← supertype colon
       ;; }
       (member (swift-mode:token:text
                (swift-mode:backquote-identifier-if-after-dot
                 (swift-mode:backward-token-simple)))
               '("class" "extension" "enum" "struct" "actor" "protocol"
                 "typealias" "associatedtype"))))))

(defvar swift-mode:in-recursive-call-of-case-colon-p nil
  "Non-nil if `case-colon-p' is being evaluated.")

(defun swift-mode:case-colon-p ()
  "Return non-nil if the colon at the cursor follows case or default label.

Return nil otherwise."
  (if swift-mode:in-recursive-call-of-case-colon-p
      nil
    (save-excursion
      (setq swift-mode:in-recursive-call-of-case-colon-p t)
      (unwind-protect
          (member
           ;; FIXME:
           ;; This function can be confused by conditional operator.
           ;;
           ;; switch foo {
           ;; case let x where x is Foo ?
           ;;                    a : // This function should return nil but it
           ;;                        // actually returns t.
           ;;                    b: // This function should return t but it
           ;;                       // actually return nil.
           ;;   let y = a ? b : c // This function returns nil correctly for
           ;;                     // this.
           ;; }

           ;; FIXME: mutual dependency
           (swift-mode:token:text
            (swift-mode:backward-sexps-until
             '(implicit-\; \; { \( \[ "case" "default" ":")))
           '("case" "default"))
        (setq swift-mode:in-recursive-call-of-case-colon-p nil)))))

(defun swift-mode:anonymous-parameter-in-p ()
  "Return t if a `in' token at the cursor is for anonymous function parameters."
  (save-excursion
    (eq
     ;; FIXME: mutual dependency
     (swift-mode:token:type (swift-mode:backward-sexps-until
                             '(\; { \( \[ "for")))
     '{)))

(defun swift-mode:generic-parameter-clause-start-p ()
  "Return t if the `<' at the cursor is a start of generic parameters.

Return nil otherwise."
  (save-excursion
    (or (member (swift-mode:token:text (swift-mode:backward-token-simple))
                '("init" "subscript"))
        (member (swift-mode:token:text (swift-mode:backward-token-simple))
                '("typealias" "func" "enum" "struct" "actor" "class" "init"
                  "macro")))))

(defun swift-mode:fix-operator-type (token)
  "Return new operator token with proper token type.

Other properties are the same as the TOKEN."
  ;; Operator type (i.e. prefix, postfix, infix) is decided from spaces or
  ;; comments around the operator.
  ;; https://developer.apple.com/library/ios/documentation/Swift/Conceptual/Swift_Programming_Language/LexicalStructure.html#//apple_ref/doc/uid/TP40014097-CH30-ID410
  ;; https://github.com/apple/swift-evolution/blob/master/proposals/0037-clarify-comments-and-operators.md
  (let*
      ((text (swift-mode:token:text token))
       (start (swift-mode:token:start token))
       (end (swift-mode:token:end token))
       (has-preceding-space (or
                             (= start (point-min))
                             (memq (char-syntax (char-before start)) '(?  ?>))
                             (memq (char-before start) '(?\( ?\[ ?{ ?, ?\; ?:))
                             (nth 4 (save-excursion
                                      (syntax-ppss (1- start))))))
       (has-following-space (or
                             (= end (point-max))
                             (memq (char-syntax (char-after end)) '(?  ?<))
                             (memq (char-after end) '(?\) ?\] ?} ?, ?\; ?:))
                             (save-excursion (goto-char end)
                                             (looking-at "/\\*\\|//"))
                             (= (char-after end) ?\C-j)))
       (has-following-dot (eq (char-after end) ?.))
       (is-declaration (save-excursion
                         ;; i.e.
                         ;; func +++(x1: X, x2: X)
                         ;; or operator declarations.
                         (goto-char start)
                         (member
                          (swift-mode:token:text
                           (swift-mode:backquote-identifier-if-after-dot
                            (swift-mode:backward-token-simple)))
                          '("func" "operator"))))
       (type
        (cond
         (is-declaration 'identifier)
         ((member text '("try" "try?" "try!" "await" "consume" "copy"
                         "discard" "each"))
          'prefix-operator)
         ((equal text ".") 'binary-operator)
         ((and has-preceding-space has-following-space) 'binary-operator)
         (has-preceding-space 'prefix-operator)
         ((or has-following-space has-following-dot) 'postfix-operator)
         (t 'binary-operator))))
    (swift-mode:token type text start end)))

(defun swift-mode:backquote-identifier-if-after-dot (token)
  "Backquote identifier TOKEN, including keywords, if it is after a dot.

See SE-0071:
https://github.com/apple/swift-evolution/blob/master/proposals/0071-member-keywords.md"
  (if (and (string-match "^[a-z]" (swift-mode:token:text token))
           (save-excursion
             (goto-char (swift-mode:token:start token))
             (equal (swift-mode:token:text (swift-mode:backward-token-simple))
                    ".")))
      (swift-mode:token
       'identifier
       (concat "`" (swift-mode:token:text token) "`")
       (swift-mode:token:start token)
       (swift-mode:token:end token))
    token))

(defun swift-mode:forward-token ()
  "Move point forward to the next position of the end of a token.

Return the token object.  If no more tokens available, return a token with
type `outside-of-buffer'."
  (let ((pos (point)))
    (let ((chunk (swift-mode:chunk-after)))
      (when (swift-mode:chunk:comment-p chunk)
        (goto-char (swift-mode:chunk:start chunk))))
    (forward-comment (point-max))
    (cond
     ;; Outside of buffer
     ((eobp)
      (swift-mode:token 'outside-of-buffer "" (point) (point)))

     ;; Implicit semicolon
     ((and (< pos
              (save-excursion
                (swift-mode:goto-non-comment-bol)
                (point)))
           (save-excursion (goto-char pos) (swift-mode:implicit-semi-p)))

      (swift-mode:token 'implicit-\;
                        (buffer-substring-no-properties pos (point))
                        pos
                        (point)))

     ;; Colon
     ((eq (char-after) ?:)
      (swift-mode:token (cond
                         ((swift-mode:supertype-colon-p) 'supertype-:)
                         ((swift-mode:case-colon-p) 'case-:)
                         (t ':))
                        ":"
                        (progn (forward-char) (1- (point)))
                        (point)))

     ;; Start of generic-parameter-clause
     ((and
       (eq (char-after) ?<)
       (swift-mode:generic-parameter-clause-start-p))
      (swift-mode:token '<
                        "<"
                        (progn (forward-char) (1- (point)))
                        (point)))

     (t
      (let ((token (swift-mode:forward-token-simple)))
        (setq token (swift-mode:backquote-identifier-if-after-dot token))

        (when (and (equal (swift-mode:token:text token) "in")
                   (save-excursion
                     (goto-char (swift-mode:token:start token))
                     (swift-mode:anonymous-parameter-in-p)))
          (setq token
                (swift-mode:token
                 'anonymous-function-parameter-in
                 "in"
                 (swift-mode:token:start token)
                 (swift-mode:token:end token))))
        token)))))

(defun swift-mode:forward-token-simple ()
  "Like `swift-mode:forward-token' without recursion.

This function does not return `implicit-;' or `type-:'."
  (forward-comment (point-max))
  (cond
   ;; Outside of buffer
   ((eobp)
    (swift-mode:token 'outside-of-buffer "" (point) (point)))

   ;; End of interpolated expression
   ((and (eq (char-after) ?\))
         (equal (get-text-property (point) 'syntax-table)
                (string-to-syntax "|")))
    (let ((pos-after-comment (point)))
      (swift-mode:forward-string-chunk)
      (swift-mode:token
       'string-chunk-after-interpolated-expression
       (buffer-substring-no-properties pos-after-comment (point))
       pos-after-comment
       (point))))

   ;; Separators and parentheses
   ((memq (char-after) '(?, ?\; ?\{ ?\} ?\[ ?\] ?\( ?\) ?:))
    (forward-char)
    (swift-mode:token (intern (string (char-before)))
                      (string (char-before))
                      (1- (point))
                      (point)))

   ;; Open angle bracket for type parameters
   ;;
   ;; We use a heuristic: spaces are inserted around inequality sign, but not
   ;; for angle bracket, and a type parameter starts with an upper case
   ;; character, a square bracket, a parenthesis, or keyword 'protocol'.
   ((and (eq (char-after) ?<)
         (looking-at "<\\([[:upper:][(]\\|protocol\\)"))
    (forward-char)
    (swift-mode:token '< "<" (1- (point)) (point)))

   ;; Close angle bracket for type parameters
   ;;
   ;; Close angle bracket follows identifier, a square bracket, a parenthesis,
   ;; or another angle bracket (e.g. Foo<Bar<[(Int, String)]>>)
   ((and (eq (char-after) ?>)
         (save-excursion
           ;; You know that regular languages can be reversed. Thus you may
           ;; think that `looking-back' reverses the given regexp and scans
           ;; chars backwards.  Nevertheless, `looking-back' function does not
           ;; do that.  It just repeats `looking-at' with decrementing start
           ;; position until it succeeds. The document says that it is not
           ;; recommended to use.  So using `skip-chars-backward',
           ;; `skip-syntax-backward', and `looking-at' here.
           (skip-chars-backward "])>")
           (skip-syntax-backward "w_")
           (looking-at "[[:upper:]_]")))
    (forward-char)
    (swift-mode:token '> ">" (1- (point)) (point)))

   ;; Regex
   ((and (looking-at "#*/")
         (equal (get-text-property (match-beginning 0) 'syntax-table)
                (string-to-syntax "|")))
    (let ((pos-after-comment (point)))
      (swift-mode:forward-string-chunk)
      (swift-mode:token
       'identifier
       (buffer-substring-no-properties pos-after-comment (point))
       pos-after-comment
       (point))))

   ;; Operator (other than as, try, is, await, consume, copy, discard, or each)
   ;;
   ;; Operators starts with a dot can contains dots. Other operators cannot
   ;; contain dots.
   ;;
   ;; https://developer.apple.com/library/ios/documentation/Swift/Conceptual/Swift_Programming_Language/LexicalStructure.html#//apple_ref/swift/grammar/dot-operator-head
   ;; TODO Unicode operators
   ((looking-at "[-/=+!*%<>&|^~?]+\\|[.][-./=+!*%<>&|^~?]*")
    (let* ((text (match-string-no-properties 0))
           (start (match-beginning 0))
           (end (match-end 0)))
      (when (string-match "^.*?/\\*\\|^.*?//" text)
        ;; e.g. +++/* */ or +++//
        (setq end (- end (- (length text) (- (match-end 0) 2))))
        (setq text (substring text 0 (- (match-end 0) 2))))
      (when (and (string-match "^.*?/" text)
                 (equal (get-text-property (+ start (1- (match-end 0)))
                                           'syntax-table)
                        (string-to-syntax "|")))
        ;; Regex after prefix operator, e.g. +++/<>/
        (setq end (- end (- (length text) (- (match-end 0) 1))))
        (setq text (substring text 0 (- (match-end 0) 1))))
      (goto-char end)
      (swift-mode:fix-operator-type
       (swift-mode:token nil text start end))))

   ;; Backquoted identifier
   ((eq (char-after) ?`)
    (let ((pos-after-comment (point)))
      (swift-mode:forward-string-chunk)
      (swift-mode:token
       'identifier
       (buffer-substring-no-properties pos-after-comment (point))
       pos-after-comment
       (point))))

   ;; String
   ((looking-at "#*\"")
    (let ((pos-after-comment (point)))
      (skip-chars-forward "#")
      (forward-char)
      (swift-mode:end-of-string)
      (swift-mode:token
       'identifier
       (buffer-substring-no-properties pos-after-comment (point))
       pos-after-comment
       (point))))

   ;; Attribute
   ((eq (char-after) ?@)
    (let ((pos-after-comment (point)))
      (forward-symbol 1)
      (let ((pos (point)))
        (forward-comment (point-max))
        (if (eq (char-after) ?\()
            (condition-case nil
                (progn
                  (forward-list 1))
              (scan-error (goto-char pos)))
          (goto-char pos)))
      (swift-mode:token
       'attribute
       (buffer-substring-no-properties pos-after-comment (point))
       pos-after-comment
       (point))))

   ;; Other tokens including identifiers, implicit parameters, keywords, and
   ;; numbers
   (t
    (let*
        ((pos-after-comment (point))
         (text
          (cond
           ;; Identifiers, implicit parameters, keywords, numbers
           ;;
           ;; Note: syntax class _ includes #, @, and $.
           ((memq (char-syntax (char-after)) '(?w ?_))
            (progn (forward-symbol 1)
                   (buffer-substring-no-properties pos-after-comment
                                                   (point))))
           ;; Unknown character type. Treats as a single-letter token.
           (t (forward-char) (string (char-before))))))
      (cond
       ((member text '("as" "try"))
        ;; as?, as!, try?, or try!
        (when (member (char-after) '(?? ?!))
          (forward-char)
          (setq text (concat text (list (char-before)))))
        (swift-mode:token (if (member text '("as" "as?" "as!"))
                              'binary-operator
                            'prefix-operator)
                          text
                          (- (point) (length text))
                          (point)))
       ((equal text "is")
        (swift-mode:token 'binary-operator
                          text
                          (- (point) (length text))
                          (point)))
       ((member text '("await" "consume" "copy" "discard" "each"))
        (swift-mode:token 'prefix-operator
                          text
                          (- (point) (length text))
                          (point)))
       (t
        (swift-mode:token 'identifier
                          text
                          (- (point) (length text))
                          (point))))))))

(defun swift-mode:backward-token ()
  "Move point backward to the previous position of the end of a token.

Return the token object.  If no more tokens available, return a token with
type `outside-of-buffer'."

  (let ((pos (point)))
    (let ((chunk (swift-mode:chunk-after)))
      (when (swift-mode:chunk:comment-p chunk)
        (goto-char (swift-mode:chunk:start chunk))))
    (forward-comment (- (point)))
    (cond
     ;; Outside of buffer
     ((bobp)
      (swift-mode:token 'outside-of-buffer "" (point) (point)))

     ;; Implicit semicolon
     ((and (< (save-excursion
                (swift-mode:goto-non-comment-eol)
                (point))
              pos)
           (save-excursion (goto-char pos) (swift-mode:implicit-semi-p)))
      (swift-mode:token 'implicit-\;
                        (buffer-substring-no-properties (point) pos)
                        (point)
                        pos))

     ;; Colon
     ((eq (char-before) ?:)
      (backward-char)
      (swift-mode:token (cond
                         ((swift-mode:supertype-colon-p) 'supertype-:)
                         ((swift-mode:case-colon-p) 'case-:)
                         (t ':))
                        ":"
                        (point)
                        (1+ (point))))

     ;; Start of generic-parameter-clause
     ((and
       (eq (char-before) ?<)
       (save-excursion
         (backward-char)
         (swift-mode:generic-parameter-clause-start-p)))
      (backward-char)
      (swift-mode:token '<
                        "<"
                        (point)
                        (1+ (point))))

     (t
      (let ((token (swift-mode:backward-token-simple)))
        (setq token (swift-mode:backquote-identifier-if-after-dot token))

        (when (and (equal (swift-mode:token:text token) "in")
                   (save-excursion
                     (goto-char (swift-mode:token:start token))
                     (swift-mode:anonymous-parameter-in-p)))
          (setq token
                (swift-mode:token
                 'anonymous-function-parameter-in
                 "in"
                 (swift-mode:token:start token)
                 (swift-mode:token:end token))))
        token)))))

(defun swift-mode:backward-token-simple ()
  "Like `swift-mode:backward-token' without recursion.

This function does not return `implicit-;' or `type-:'."
  (forward-comment (- (point)))
  (cond
   ;; Outside of buffer
   ((bobp)
    (swift-mode:token 'outside-of-buffer "" (point) (point)))

   ;; Beginning of interpolated expression
   ((and (eq (char-before) ?\()
         (equal (get-text-property (1- (point)) 'syntax-table)
                (string-to-syntax "|")))
    (let ((pos-before-comment (point)))
      (swift-mode:backward-string-chunk)
      (swift-mode:token
       'string-chunk-before-interpolated-expression
       (buffer-substring-no-properties (point) pos-before-comment)
       (point)
       pos-before-comment)))

   ;; Attribute or close-parenthesis
   ((eq (char-before) ?\))
    (let ((pos-before-comment (point)))
      (condition-case nil
          (progn
            (backward-list)
            (forward-comment (- (point)))
            (forward-symbol -1)
            (unless (eq (char-after) ?@)
              (goto-char (1- pos-before-comment))))
        (scan-error (goto-char (1- pos-before-comment))))
      (swift-mode:token
       (if (eq (char-after) ?@) 'attribute '\))
       (buffer-substring-no-properties (point) pos-before-comment)
       (point)
       pos-before-comment)))

   ;; Separators and parentheses
   ((memq (char-before) '(?, ?\; ?\{ ?\} ?\[ ?\] ?\( ?\) ?:))
    (backward-char)
    (swift-mode:token (intern (string (char-after)))
                      (string (char-after))
                      (point)
                      (1+ (point))))

   ;; >! or >?
   ((and (memq (char-before) '(?! ??))
         (eq (char-before (1- (point))) ?>)
         (save-excursion
           (backward-char)
           (eq (swift-mode:token:type (swift-mode:backward-token-simple))
               '>)))
    (backward-char)
    (swift-mode:token (intern (string (char-after)))
                      (string (char-after))
                      (point)
                      (1+ (point))))

   ;; Open angle bracket for type parameters
   ;;
   ;; We use a heuristic: spaces are inserted around inequality sign, but not
   ;; for angle bracket, and a type parameter starts with an upper case
   ;; character, a square bracket, a parenthesis, or keyword `protocol'.
   ((and (eq (char-before) ?<)
         (looking-at "\\([[:upper:][(]\\|protocol\\)"))
    (backward-char)
    (swift-mode:token '< "<" (point) (1+ (point))))

   ;; Close angle bracket for type parameters
   ;;
   ;; Close angle bracket follows identifier, a square bracket, a parenthesis,
   ;; or another angle bracket (e.g. Foo<Bar<[(Int, String)]>>)
   ((and (eq (char-before) ?>)
         (save-excursion
           (skip-chars-backward "])>")
           (skip-syntax-backward "w_")
           (looking-at "[[:upper:]_]")))
    (backward-char)
    (swift-mode:token '> ">" (point) (1+ (point))))

   ;; Regex
   ((and (save-excursion
           (skip-chars-backward "#")
           (eq (char-before) ?/))
         (equal (get-text-property (1- (point)) 'syntax-table)
                (string-to-syntax "|")))
    (let ((pos-before-comment (point)))
      (swift-mode:backward-string-chunk)
      (swift-mode:token
       'identifier
       (buffer-substring-no-properties (point) pos-before-comment)
       (point)
       pos-before-comment)))

   ;; Operator (other than as, try, is, await, consume, copy, discard, or each)
   ;;
   ;; Operators which starts with a dot can contain other dots. Other
   ;; operators cannot contain dots.
   ;;
   ;; https://developer.apple.com/library/ios/documentation/Swift/Conceptual/Swift_Programming_Language/LexicalStructure.html#//apple_ref/swift/grammar/dot-operator-head
   ;; TODO Unicode operators
   ((memq (char-before) '(?. ?- ?/ ?= ?+ ?! ?* ?% ?< ?> ?& ?| ?^ ?~ ??))
    (let ((point-before-comments (point)))
      (skip-chars-backward "-./=+!*%<>&|^~?")
      (cond
       ((save-excursion
          (forward-symbol -1)
          (and (looking-at "\\(as\\|try\\)[?!]")
               (= point-before-comments (match-end 0))))
        ;; as?, as!, try?, try!
        t)
       ((looking-at "[.][-./=+!*%<>&|^~?]*")
        ;; e.g. 1 .++++.++++...+. 1
        t)
       ((and (looking-at "[-/=+!*%<>&|^~?]+")
             (<= point-before-comments (match-end 0)))
        ;; e.g. 1 +++++++++ 1
        t)
       (t
        ;; e.g. 1+++++...++++1, that is (1+++++) ...++++ 1
        (skip-chars-forward "-/=+!*%<>&|^~?")
        (looking-at "[.][-./=+!*%<>&|^~?]*")))
      (let*
          ((start (match-beginning 0))
           (end (min point-before-comments (match-end 0)))
           (text (substring (match-string-no-properties 0) 0 (- end start))))
        (goto-char start)
        (swift-mode:fix-operator-type
         (swift-mode:token nil text start end)))))

   ;; Backquoted identifier
   ((eq (char-before) ?`)
    (let ((pos-before-comment (point)))
      (swift-mode:backward-string-chunk)
      (swift-mode:token
       'identifier
       (buffer-substring-no-properties (point) pos-before-comment)
       (point)
       pos-before-comment)))

   ;; String
   ((save-excursion
      (skip-chars-backward "#")
      (eq (char-before) ?\"))
    (let ((pos-before-comment (point)))
      (skip-chars-backward "#")
      (backward-char)
      (swift-mode:beginning-of-string)
      (swift-mode:token
       'identifier
       (buffer-substring-no-properties (point) pos-before-comment)
       (point)
       pos-before-comment)))

   ;; Other tokens including identifiers, implicit parameters, keywords, and
   ;; numbers
   (t
    (let*
        ((pos-before-comment (point))
         (text
          (cond
           ;; Identifiers, implicit parameters, keywords, numbers
           ;;
           ;; Note: syntax class _ includes #, @, and $.
           ((memq (char-syntax (char-before)) '(?w ?_))
            (progn (forward-symbol -1)
                   (buffer-substring-no-properties (point)
                                                   pos-before-comment)))
           ;; Unknown character type. Treats as a single-letter token.
           (t (backward-char) (string (char-after))))))
      (cond
       ((member text '("is" "as"))
        (swift-mode:token 'binary-operator
                          text
                          (point)
                          (+ (point) (length text))))
       ((member text '("try" "await" "consume" "copy" "discard" "each"))
        (swift-mode:token 'prefix-operator
                          text
                          (point)
                          (+ (point) (length text))))
       ((string-prefix-p "@" text)
        (swift-mode:token 'attribute
                          text
                          (point)
                          (+ (point) (length text))))
       (t
        (swift-mode:token 'identifier
                          text
                          (point)
                          (+ (point) (length text)))))))))

(defun swift-mode:forward-string-chunk ()
  "Skip forward a string chunk.

A string chunk is a part of single-line/multiline string delimited with
quotation marks or interpolated expressions."
  (condition-case nil
      (goto-char (scan-sexps (point) 1))
    (scan-error (goto-char (point-max)))))

(defun swift-mode:backward-string-chunk ()
  "Skip backward a string chunk.

A string chunk is a part of single-line/multiline string delimited with
quotation marks or interpolated expressions."
  (condition-case nil
      (goto-char (scan-sexps (point) -1))
    (scan-error (goto-char (point-min)))))

(defun swift-mode:beginning-of-string ()
  "Move point to the beginning of single-line/multiline string.

Assuming the cursor is on a string."
  (goto-char (or (nth 8 (syntax-ppss)) (point)))
  (let (matching-parenthesis)
    (while (setq matching-parenthesis
                 (get-text-property
                  (point)
                  'swift-mode:matching-parenthesis))
      (goto-char matching-parenthesis)
      (goto-char (nth 8 (syntax-ppss))))
    (point)))

(defun swift-mode:end-of-string ()
  "Move point to the end of single-line/multiline string.

Assuming the cursor is on a string."
  (goto-char (or (nth 8 (syntax-ppss)) (point)))
  (let (matching-parenthesis)
    (swift-mode:forward-string-chunk)
    (while (setq matching-parenthesis
                 (get-text-property
                  (1- (point))
                  'swift-mode:matching-parenthesis))
      (goto-char matching-parenthesis)
      (swift-mode:forward-string-chunk)))
  (point))

(defun swift-mode:goto-non-comment-bol ()
  "Back to the beginning of line that is not inside a comment."
  (forward-line 0)
  (let (chunk)
    (while (progn
             (setq chunk (swift-mode:chunk-after))
             (swift-mode:chunk:comment-p chunk))
      ;; The cursor is in a comment. Backs to the beginning of the comment.
      (goto-char (swift-mode:chunk:start chunk))
      (forward-line 0))))

(defun swift-mode:goto-non-comment-eol ()
  "Proceed to the end of line that is not inside a comment.

If this line ends with a single-line comment, goto just before the comment."
  (end-of-line)
  (let (chunk)
    (while (progn
             (setq chunk (swift-mode:chunk-after))
             (swift-mode:chunk:comment-p chunk))
      ;; The cursor is in a comment.
      (if (swift-mode:chunk:single-line-comment-p chunk)
          ;; This is a single-line comment
          ;; Back to the beginning of the comment.
          (goto-char (swift-mode:chunk:start chunk))
        ;; This is a multiline comment
        ;; Proceed to the end of the comment.
        (goto-char (swift-mode:chunk:start chunk))
        (forward-comment 1)
        (end-of-line)
        (when (and (eobp) (swift-mode:chunk-after))
          (goto-char (swift-mode:chunk:start (swift-mode:chunk-after))))))))

;;; Comment or string chunks

;; A chunk is either a string-chunk, regex, or a comment.
;; It have the type and the start position.

(defun swift-mode:chunk (type start)
  "Return a new chunk with TYPE and START position."
  (list type start))

(defun swift-mode:chunk:type (chunk)
  "Return the type of the CHUNK."
  (nth 0 chunk))

(defun swift-mode:chunk:start (chunk)
  "Return the start position of the CHUNK."
  (nth 1 chunk))

(defun swift-mode:chunk:end (chunk)
  "Return the end position of the CHUNK."
  (save-excursion
    (goto-char (nth 1 chunk))
    (if (swift-mode:chunk:comment-p chunk)
        (forward-comment 1)
      (swift-mode:forward-token))
    (point)))

(defun swift-mode:chunk:comment-p (chunk)
  "Return non-nil if the CHUNK is a comment."
  (memq (swift-mode:chunk:type chunk) '(single-line-comment multiline-comment)))

(defun swift-mode:chunk:string-p (chunk)
  "Return non-nil if the CHUNK is a string."
  (memq (swift-mode:chunk:type chunk) '(single-line-string multiline-string)))

(defun swift-mode:chunk:single-line-comment-p (chunk)
  "Return non-nil if the CHUNK is a single-line comment."
  (eq (swift-mode:chunk:type chunk) 'single-line-comment))

(defun swift-mode:chunk:multiline-comment-p (chunk)
  "Return non-nil if the CHUNK is a multiline comment."
  (eq (swift-mode:chunk:type chunk) 'multiline-comment))

(defun swift-mode:chunk:single-line-string-p (chunk)
  "Return non-nil if the CHUNK is a single-line string."
  (eq (swift-mode:chunk:type chunk) 'single-line-string))

(defun swift-mode:chunk:multiline-string-p (chunk)
  "Return non-nil if the CHUNK is a multiline string."
  (eq (swift-mode:chunk:type chunk) 'multiline-string))

(defun swift-mode:chunk:regex-p (chunk)
  "Return non-nil if the CHUNK is a regex."
  (eq (swift-mode:chunk:type chunk) 'regex))

(defun swift-mode:chunk:pound-count (chunk)
  "Return the number of pound signs before the start position of the CHUNK."
  (save-excursion
    (goto-char (swift-mode:chunk:start chunk))
    (swift-mode:beginning-of-string)
    (skip-chars-forward "#")
    (- (swift-mode:chunk:start chunk) (point))))

(defun swift-mode:chunk-after (&optional parser-state)
  "Return the chunk at the cursor.

If the cursor is outside of strings and comments, return nil.

If PARSER-STATE is given, it is used instead of (syntax-ppss)."
  (save-excursion
    (when (number-or-marker-p parser-state)
      (goto-char parser-state))
    (when (or (null parser-state) (number-or-marker-p parser-state))
      (setq parser-state (save-excursion (syntax-ppss parser-state))))
    (cond
     ((nth 3 parser-state)
      ;; Syntax category "|" is attached to both single-line and multiline
      ;; string delimiters.  So (nth 3 parser-state) may be t even for
      ;; single-line string delimiters.
      (cond
       ((save-excursion (goto-char (nth 8 parser-state))
                        (looking-at "#*\"\"\""))
        (swift-mode:chunk 'multiline-string (nth 8 parser-state)))
       ((save-excursion (goto-char (nth 8 parser-state))
                        (looking-at "#*/"))
        (swift-mode:chunk 'regex (nth 8 parser-state)))
       (t (swift-mode:chunk 'single-line-string (nth 8 parser-state)))))

     ((eq (nth 4 parser-state) t)
      (swift-mode:chunk 'single-line-comment (nth 8 parser-state)))

     ((nth 4 parser-state)
      (swift-mode:chunk 'multiline-comment (nth 8 parser-state)))

     ((and (eq (char-before) ?/) (eq (char-after) ?/))
      (swift-mode:chunk 'single-line-comment (1- (point))))

     ((and (eq (char-before) ?/) (eq (char-after) ?*))
      (swift-mode:chunk 'multiline-comment (1- (point))))

     (t
      nil))))

(defun swift-mode:same-line-p (point1 point2)
  "Return non-nil if POINT1 and POINT2 is on the same line.

Return nil otherwise."
  (= (save-excursion
       (goto-char point1)
       (line-beginning-position))
     (save-excursion
       (goto-char point2)
       (line-beginning-position))))

(provide 'swift-mode-lexer)

;;; swift-mode-lexer.el ends here