Merged in lucatorella/pygments-main (pull request #813)

Add support for @import keyword in Objective-C

AUTHORS

@@ -132,6 +132,7 @@ Other contributors, listed alphabetically, are:
 * Stephen McKamey -- Duel/JBST lexer
 * Brian McKenna -- F# lexer
 * Charles McLaughlin -- Puppet lexer
+* Kurt McKee -- Tera Term macro lexer
 * Lukas Meuser -- BBCode formatter, Lua lexer
 * Cat Miller -- Pig lexer
 * Paul Miller -- LiveScript lexer
@@ -221,5 +222,7 @@ Other contributors, listed alphabetically, are:
 * Rob Zimmerman -- Kal lexer
 * Vincent Zurczak -- Roboconf lexer
 * Rostyslav Golda -- FloScript lexer
+* GitHub, Inc -- DASM16, Augeas, TOML, and Slash lexers
+* Simon Garnotel -- FreeFem++ lexer
 
 Many thanks for all contributions!

CHANGES

@@ -6,37 +6,66 @@ Issue numbers refer to the tracker at
 pull request numbers to the requests at
 <https://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>.
 
+Version 2.4.1
+-------------
+
+- Updated lexers:
+
+  * Coq (#1430)
+  * MSDOS Session (PR#734)
+  * Prolog (#1511)
+  * TypeScript (#1515)
+
+- Support CSS variables in stylesheets (PR#814)
+- Fix F# lexer name (PR#709)
+- Fix ``TerminalFormatter`` using bold for bright text (#1480)
+
 Version 2.4.0
 -------------
-(not released yet)
+(released May 8, 2019)
 
 - Added lexers:
 
+  * Augeas (PR#807)
+  * BBC Basic (PR#806)
+  * Boa (PR#756)
   * Charm++ CI (PR#788)
+  * DASM16 (PR#807)
   * FloScript (PR#750)
+  * FreeFem++ (PR#785)
   * Hspec (PR#790)
+  * Pony (PR#627)
   * SGF (PR#780)
+  * Slash (PR#807)
   * Slurm (PR#760)
+  * Tera Term Language (PR#749)
+  * TOML (PR#807)
   * Unicon (PR#731)
   * VBScript (PR#673)
 
 - Updated lexers:
 
+  * Apache2 (PR#766)
   * Cypher (PR#746)
   * LLVM (PR#792)
+  * Makefiles (PR#766)
   * PHP (#1482)
+  * Rust
   * SQL (PR#672)
   * Stan (PR#774)
   * Stata (PR#800)
   * Terraform (PR#787)
+  * YAML
 
 - Add solarized style (PR#708)
 - Add support for Markdown reference-style links (PR#753)
+- Add license information to generated HTML/CSS files (#1496)
 - Change ANSI color names (PR#777)
 - Fix catastrophic backtracking in the bash lexer (#1494)
 - Fix documentation failing to build using Sphinx 2.0 (#1501)
 - Fix incorrect links in the Lisp and R lexer documentation (PR#775)
 - Fix rare unicode errors on Python 2.7 (PR#798, #1492)
+- Fix lexers popping from an empty stack (#1506)
 - TypoScript uses ``.typoscript`` now (#1498)
 - Updated Trove classifiers and ``pip`` requirements (PR#799)
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2006-2017 by the respective authors (see AUTHORS file).
+Copyright (c) 2006-2019 by the respective authors (see AUTHORS file).
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

doc/languages.rst

@@ -14,8 +14,11 @@ Programming languages
 * AppleScript
 * Assembly (various)
 * Asymptote
+* `Augeas <http://augeas.net>`_
 * Awk
+* BBC Basic
 * Befunge
+* `Boa <http://boa.cs.iastate.edu/docs/index.php>`_
 * Boo
 * BrainFuck
 * C, C++
@@ -31,6 +34,7 @@ Programming languages
 * `Cython <http://cython.org>`_
 * `D <http://dlang.org>`_
 * Dart
+* DCPU-16
 * Delphi
 * Dylan
 * `Elm <http://elm-lang.org/>`_
@@ -41,6 +45,7 @@ Programming languages
 * `Fennel <https://fennel-lang.org/>`_
 * `FloScript <http://ioflo.com/>`_
 * Fortran
+* `FreeFEM++ <https://freefem.org/>`_
 * F#
 * GAP
 * Gherkin (Cucumber)
@@ -70,6 +75,7 @@ Programming languages
 * OCaml
 * PHP
 * `Perl 5 <http://perl.org>`_ and `Perl 6 <https://perl6.org>`_
+* `Pony <https://www.ponylang.io/>`_
 * PovRay
 * PostScript
 * PowerShell
@@ -85,10 +91,13 @@ Programming languages
 * Scheme
 * Scilab
 * `SGF <https://www.red-bean.com/sgf/>`_
+* `Slash <https://github.com/arturadib/Slash-A>`_
 * `Slurm <https://slurm.schedmd.com/overview.html>`_
 * Smalltalk
 * SNOBOL
 * Tcl
+* `Tera Term language <https://ttssh2.osdn.jp/>`_
+* `TOML <https://github.com/toml-lang/toml>`_
 * Vala
 * Verilog
 * VHDL

pygments/__init__.py

@@ -29,7 +29,7 @@
 
 from pygments.util import StringIO, BytesIO
 
-__version__ = '2.3.1'
+__version__ = '2.4.0'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']

pygments/console.py

@@ -30,7 +30,7 @@
 x = 30
 for d, l in zip(dark_colors, light_colors):
     codes[d] = esc + "%im" % x
-    codes[l] = esc + "%i;01m" % x
+    codes[l] = esc + "%im" % (60 + x)
     x += 1
 
 del d, l, x

pygments/formatters/html.py

@@ -5,7 +5,7 @@
 
     Formatter for HTML output.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -41,6 +41,11 @@ def escape_html(text, table=_escape_html_table):
     """Escape &, <, > as well as single and double quotes for HTML."""
     return text.translate(table)
 
+def webify(color):
+    if color.startswith('calc') or color.startswith('var'):
+        return color
+    else:
+        return '#' + color
 
 def _get_ttype_class(ttype):
     fname = STANDARD_TYPES.get(ttype)
@@ -55,6 +60,11 @@ def _get_ttype_class(ttype):
 
 
 CSSFILE_TEMPLATE = '''\
+/*
+generated by Pygments <http://pygments.org>
+Copyright 2006-2019 by the Pygments team.
+Licensed under the BSD license, see LICENSE for details.
+*/
 td.linenos { background-color: #f0f0f0; padding-right: 10px; }
 span.lineno { background-color: #f0f0f0; padding: 0 5px 0 5px; }
 pre { line-height: 125%%; }
@@ -64,7 +74,11 @@ def _get_ttype_class(ttype):
 DOC_HEADER = '''\
 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
    "http://www.w3.org/TR/html4/strict.dtd">
-
+<!--
+generated by Pygments <http://pygments.org>
+Copyright 2006-2019 by the Pygments team.
+Licensed under the BSD license, see LICENSE for details.
+-->
 <html>
 <head>
   <title>%(title)s</title>
@@ -322,11 +336,17 @@ class ``"special"`` (default: ``0``).
         .. versionadded:: 1.6
 
     `filename`
-        A string used to generate a filename when rendering <pre> blocks,
+        A string used to generate a filename when rendering ``<pre>`` blocks,
         for example if displaying source code.
 
         .. versionadded:: 2.1
 
+    `wrapcode`
+        Wrap the code inside ``<pre>`` blocks using ``<code>``, as recommended
+        by the HTML5 specification.
+
+        .. versionadded:: 2.4
+
 
     **Subclassing the HTML formatter**
 
@@ -395,6 +415,7 @@ def __init__(self, **options):
         self.tagsfile = self._decodeifneeded(options.get('tagsfile', ''))
         self.tagurlformat = self._decodeifneeded(options.get('tagurlformat', ''))
         self.filename = self._decodeifneeded(options.get('filename', ''))
+        self.wrapcode = get_bool_opt(options, 'wrapcode', False)
 
         if self.tagsfile:
             if not ctags:
@@ -451,17 +472,17 @@ def _create_stylesheet(self):
             name = self._get_css_class(ttype)
             style = ''
             if ndef['color']:
-                style += 'color: #%s; ' % ndef['color']
+                style += 'color: %s; ' % webify(ndef['color'])
             if ndef['bold']:
                 style += 'font-weight: bold; '
             if ndef['italic']:
                 style += 'font-style: italic; '
             if ndef['underline']:
                 style += 'text-decoration: underline; '
             if ndef['bgcolor']:
-                style += 'background-color: #%s; ' % ndef['bgcolor']
+                style += 'background-color: %s; ' % webify(ndef['bgcolor'])
             if ndef['border']:
-                style += 'border: 1px solid #%s; ' % ndef['border']
+                style += 'border: 1px solid %s; ' % webify(ndef['border'])
             if style:
                 t2c[ttype] = name
                 # save len(ttype) to enable ordering the styles by
@@ -708,6 +729,12 @@ def _wrap_pre(self, inner):
             yield tup
         yield 0, '</pre>'
 
+    def _wrap_code(self, inner):
+        yield 0, '<code>'
+        for tup in inner:
+            yield tup
+        yield 0, '</code>'
+
     def _format_lines(self, tokensource):
         """
         Just format the tokens, without any wrapping tags.
@@ -814,7 +841,10 @@ def wrap(self, source, outfile):
         individual lines, in custom generators. See docstring
         for `format`. Can be overridden.
         """
-        return self._wrap_div(self._wrap_pre(source))
+        if self.wrapcode:
+            return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
+        else:
+            return self._wrap_div(self._wrap_pre(source))
 
     def format_unencoded(self, tokensource, outfile):
         """

pygments/lexer.py

@@ -639,14 +639,20 @@ def get_tokens_unprocessed(self, text, stack=('root',)):
                         if isinstance(new_state, tuple):
                             for state in new_state:
                                 if state == '#pop':
-                                    statestack.pop()
+                                    if len(statestack) > 1:
+                                        statestack.pop()
                                 elif state == '#push':
                                     statestack.append(statestack[-1])
                                 else:
                                     statestack.append(state)
                         elif isinstance(new_state, int):
-                            # pop
-                            del statestack[new_state:]
+                            # pop, but keep at least one state on the stack
+                            # (random code leading to unexpected pops should
+                            # not allow exceptions)
+                            if abs(new_state) >= len(statestack):
+                                del statestack[1:]
+                            else:
+                                del statestack[new_state:]
                         elif new_state == '#push':
                             statestack.append(statestack[-1])
                         else:
@@ -724,14 +730,18 @@ def get_tokens_unprocessed(self, text=None, context=None):
                         if isinstance(new_state, tuple):
                             for state in new_state:
                                 if state == '#pop':
-                                    ctx.stack.pop()
+                                    if len(ctx.stack) > 1:
+                                        ctx.stack.pop()
                                 elif state == '#push':
                                     ctx.stack.append(ctx.stack[-1])
                                 else:
                                     ctx.stack.append(state)
                         elif isinstance(new_state, int):
-                            # pop
-                            del ctx.stack[new_state:]
+                            # see RegexLexer for why this check is made
+                            if abs(new_state) >= len(ctx.stack):
+                                del ctx.state[1:]
+                            else:
+                                del ctx.stack[new_state:]
                         elif new_state == '#push':
                             ctx.stack.append(ctx.stack[-1])
                         else: