test_crystal.py

# -*- coding: utf-8 -*-
"""
    Basic CrystalLexer Test
    ~~~~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from __future__ import unicode_literals
import unittest

from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
    Number, Punctuation, Error
from pygments.lexers import CrystalLexer


class CrystalTest(unittest.TestCase):

    def setUp(self):
        self.lexer = CrystalLexer()
        self.maxDiff = None

    def testRangeSyntax1(self):
        fragment = '1...3\n'
        tokens = [
            (Number.Integer, '1'),
            (Operator, '...'),
            (Number.Integer, '3'),
            (Text, '\n'),
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))

    def testRangeSyntax2(self):
        fragment = '1 .. 3\n'
        tokens = [
            (Number.Integer, '1'),
            (Text, ' '),
            (Operator, '..'),
            (Text, ' '),
            (Number.Integer, '3'),
            (Text, '\n'),
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))

    def testInterpolationNestedCurly(self):
        fragment = (
            '"A#{ (3..5).group_by { |x| x/2}.map '
            'do |k,v| "#{k}" end.join }" + "Z"\n')
        tokens = [
            (String.Double, '"'),
            (String.Double, 'A'),
            (String.Interpol, '#{'),
            (Text, ' '),
            (Punctuation, '('),
            (Number.Integer, '3'),
            (Operator, '..'),
            (Number.Integer, '5'),
            (Punctuation, ')'),
            (Operator, '.'),
            (Name, 'group_by'),
            (Text, ' '),
            (String.Interpol, '{'),
            (Text, ' '),
            (Operator, '|'),
            (Name, 'x'),
            (Operator, '|'),
            (Text, ' '),
            (Name, 'x'),
            (Operator, '/'),
            (Number.Integer, '2'),
            (String.Interpol, '}'),
            (Operator, '.'),
            (Name, 'map'),
            (Text, ' '),
            (Keyword, 'do'),
            (Text, ' '),
            (Operator, '|'),
            (Name, 'k'),
            (Punctuation, ','),
            (Name, 'v'),
            (Operator, '|'),
            (Text, ' '),
            (String.Double, '"'),
            (String.Interpol, '#{'),
            (Name, 'k'),
            (String.Interpol, '}'),
            (String.Double, '"'),
            (Text, ' '),
            (Keyword, 'end'),
            (Operator, '.'),
            (Name, 'join'),
            (Text, ' '),
            (String.Interpol, '}'),
            (String.Double, '"'),
            (Text, ' '),
            (Operator, '+'),
            (Text, ' '),
            (String.Double, '"'),
            (String.Double, 'Z'),
            (String.Double, '"'),
            (Text, '\n'),
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))

    def testOperatorMethods(self):
        fragment = '([] of Int32).[]?(5)\n'
        tokens = [
            (Punctuation, '('),
            (Operator, '['),
            (Operator, ']'),
            (Text, ' '),
            (Keyword, 'of'),
            (Text, ' '),
            (Name.Builtin, 'Int32'),
            (Punctuation, ')'),
            (Operator, '.'),
            (Name.Operator, '[]?'),
            (Punctuation, '('),
            (Number.Integer, '5'),
            (Punctuation, ')'),
            (Text, '\n')
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
    
    def testArrayAccess(self):
        fragment = '[5][5]?\n'
        tokens = [
            (Operator, '['),
            (Number.Integer, '5'),
            (Operator, ']'),
            (Operator, '['),
            (Number.Integer, '5'),
            (Operator, ']?'),
            (Text, '\n')
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))

    def testNumbers(self):
        for kind, testset in [
            (Number.Integer, '0  1  1_000_000  1u8  11231231231121312i64'),
            (Number.Float, '0.0  1.0_f32  1_f32  0f64  1e+4  1e111  1_234.567_890'),
            (Number.Bin, '0b1001_0110  0b0u8'),
            (Number.Oct, '0o17  0o7_i32'),
            (Number.Hex, '0xdeadBEEF'),
        ]:
            for fragment in testset.split():
                self.assertEqual([(kind, fragment), (Text, '\n')],
                                 list(self.lexer.get_tokens(fragment + '\n')))

        for fragment in '01  0b2  0x129g2  0o12358'.split():
            self.assertEqual(next(self.lexer.get_tokens(fragment + '\n'))[0],
                             Error)

    def testChars(self):
        for fragment in ["'a'", "'я'", "'\\u{1234}'", "'\n'"]:
            self.assertEqual([(String.Char, fragment), (Text, '\n')],
                             list(self.lexer.get_tokens(fragment + '\n')))
        self.assertEqual(next(self.lexer.get_tokens("'abc'"))[0], Error)

    def testMacro(self):
        fragment = (
            'def<=>(other : self) : Int\n'
            '{%for field in %w(first_name middle_name last_name)%}\n'
            'cmp={{field.id}}<=>other.{{field.id}}\n'
            'return cmp if cmp!=0\n'
            '{%end%}\n'
            '0\n'
            'end\n')
        tokens = [
            (Keyword, 'def'),
            (Name.Function, '<=>'),
            (Punctuation, '('),
            (Name, 'other'),
            (Text, ' '),
            (Punctuation, ':'),
            (Text, ' '),
            (Keyword.Pseudo, 'self'),
            (Punctuation, ')'),
            (Text, ' '),
            (Punctuation, ':'),
            (Text, ' '),
            (Name.Builtin, 'Int'),
            (Text, '\n'),
            (String.Interpol, '{%'),
            (Keyword, 'for'),
            (Text, ' '),
            (Name, 'field'),
            (Text, ' '),
            (Keyword, 'in'),
            (Text, ' '),
            (String.Other, '%w('),
            (String.Other, 'first_name middle_name last_name'),
            (String.Other, ')'),
            (String.Interpol, '%}'),
            (Text, '\n'),
            (Name, 'cmp'),
            (Operator, '='),
            (String.Interpol, '{{'),
            (Name, 'field'),
            (Operator, '.'),
            (Name, 'id'),
            (String.Interpol, '}}'),
            (Operator, '<=>'),
            (Name, 'other'),
            (Operator, '.'),
            (String.Interpol, '{{'),
            (Name, 'field'),
            (Operator, '.'),
            (Name, 'id'),
            (String.Interpol, '}}'),
            (Text, '\n'),
            (Keyword, 'return'),
            (Text, ' '),
            (Name, 'cmp'),
            (Text, ' '),
            (Keyword, 'if'),
            (Text, ' '),
            (Name, 'cmp'),
            (Operator, '!='),
            (Number.Integer, '0'),
            (Text, '\n'),
            (String.Interpol, '{%'),
            (Keyword, 'end'),
            (String.Interpol, '%}'),
            (Text, '\n'),
            (Number.Integer, '0'),
            (Text, '\n'),
            (Keyword, 'end'),
            (Text, '\n')
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))

    def testLib(self):
        fragment = (
            '@[Link("some")]\nlib LibSome\n'
            '@[CallConvention("X86_StdCall")]\nfun foo="some.foo"(thing : Void*) : LibC::Int\n'
            'end\n')
        tokens = [
            (Operator, '@['),
            (Name.Decorator, 'Link'),
            (Punctuation, '('),
            (String.Double, '"'),
            (String.Double, 'some'),
            (String.Double, '"'),
            (Punctuation, ')'),
            (Operator, ']'),
            (Text, '\n'),
            (Keyword, 'lib'),
            (Text, ' '),
            (Name.Namespace, 'LibSome'),
            (Text, '\n'),
            (Operator, '@['),
            (Name.Decorator, 'CallConvention'),
            (Punctuation, '('),
            (String.Double, '"'),
            (String.Double, 'X86_StdCall'),
            (String.Double, '"'),
            (Punctuation, ')'),
            (Operator, ']'),
            (Text, '\n'),
            (Keyword, 'fun'),
            (Text, ' '),
            (Name.Function, 'foo'),
            (Operator, '='),
            (String.Double, '"'),
            (String.Double, 'some.foo'),
            (String.Double, '"'),
            (Punctuation, '('),
            (Name, 'thing'),
            (Text, ' '),
            (Punctuation, ':'),
            (Text, ' '),
            (Name.Builtin, 'Void'),
            (Operator, '*'),
            (Punctuation, ')'),
            (Text, ' '),
            (Punctuation, ':'),
            (Text, ' '),
            (Name, 'LibC'),
            (Operator, '::'),
            (Name.Builtin, 'Int'),
            (Text, '\n'),
            (Keyword, 'end'),
            (Text, '\n')
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))

    def testEscapedBracestring(self):
        fragment = 'str.gsub(%r{\\\\\\\\}, "/")\n'
        tokens = [
            (Name, 'str'),
            (Operator, '.'),
            (Name, 'gsub'),
            (Punctuation, '('),
            (String.Regex, '%r{'),
            (String.Regex, '\\\\'),
            (String.Regex, '\\\\'),
            (String.Regex, '}'),
            (Punctuation, ','),
            (Text, ' '),
            (String.Double, '"'),
            (String.Double, '/'),
            (String.Double, '"'),
            (Punctuation, ')'),
            (Text, '\n'),
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))