From b2ac06e11fcbd26fb8b2194116317866841d1987 Mon Sep 17 00:00:00 2001 From: Exerro Date: Mon, 16 Nov 2015 20:38:07 +0000 Subject: [PATCH] Parsing things (expression parsing) Paving the way for the dynamic value system. --- src/Todo.txt | 3 +- src/core/DynamicValue.lua | 12 -- src/dynamic/BinaryExpression.lua | 69 +++++++++ src/dynamic/ConstantExpression.lua | 30 ++++ src/dynamic/Expression.lua | 36 +++++ src/dynamic/ExpressionEnvironment.lua | 25 +++ src/dynamic/ExpressionParser.lua | 176 +++++++++++++++++++++ src/dynamic/IdentifierExpression.lua | 36 +++++ src/dynamic/UnaryLeftExpression.lua | 59 +++++++ src/dynamic/UnaryRightExpression.lua | 88 +++++++++++ src/exceptions/Exception.lua | 16 +- src/exceptions/ExpressionException.lua | 14 ++ src/exceptions/ParserException.lua | 26 ++++ src/parsing/Parser.lua | 204 +++++++++++++++++++++++++ src/parsing/Token.lua | 48 ++++++ src/parsing/TokenPosition.lua | 57 +++++++ src/sheets.lua | 57 ++++++- 17 files changed, 937 insertions(+), 19 deletions(-) delete mode 100644 src/core/DynamicValue.lua create mode 100644 src/dynamic/BinaryExpression.lua create mode 100644 src/dynamic/ConstantExpression.lua create mode 100644 src/dynamic/Expression.lua create mode 100644 src/dynamic/ExpressionEnvironment.lua create mode 100644 src/dynamic/ExpressionParser.lua create mode 100644 src/dynamic/IdentifierExpression.lua create mode 100644 src/dynamic/UnaryLeftExpression.lua create mode 100644 src/dynamic/UnaryRightExpression.lua create mode 100644 src/exceptions/ExpressionException.lua create mode 100644 src/exceptions/ParserException.lua create mode 100644 src/parsing/Parser.lua create mode 100644 src/parsing/Token.lua create mode 100644 src/parsing/TokenPosition.lua diff --git a/src/Todo.txt b/src/Todo.txt index 9e2a0bb..1b17cf2 100644 --- a/src/Todo.txt +++ b/src/Todo.txt @@ -25,4 +25,5 @@ Graphics drawSurfaceScaled drawSurfaceRotated TermCanvas - getRedirect \ No newline at end of file + getRedirect + \ No newline at end of file diff --git a/src/core/DynamicValue.lua b/src/core/DynamicValue.lua deleted file mode 100644 index 90977e0..0000000 --- a/src/core/DynamicValue.lua +++ /dev/null @@ -1,12 +0,0 @@ - - -- @once - - -- @ifndef __INCLUDE_sheets - -- @error 'sheets' must be included before including 'sheets.core.DynamicValue' - -- @endif - - -- @print Including sheets.core.DynamicValue - -class "DynamicValue" { - instance = nil; -} diff --git a/src/dynamic/BinaryExpression.lua b/src/dynamic/BinaryExpression.lua new file mode 100644 index 0000000..7bd68ac --- /dev/null +++ b/src/dynamic/BinaryExpression.lua @@ -0,0 +1,69 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.dynamic.BinaryExpression' + -- @endif + + -- @print Including sheets.dynamic.BinaryExpression + +local tostring = { + [OPERATOR_BINARY_ADD] = "+"; + [OPERATOR_BINARY_SUB] = "-"; + [OPERATOR_BINARY_MUL] = "*"; + [OPERATOR_BINARY_DIV] = "/"; + [OPERATOR_BINARY_MOD] = "%"; + [OPERATOR_BINARY_POW] = "^"; +} + +class "BinaryExpression" extends "Expression" { + lvalue = nil; + operator = nil; + rvalue = nil; +} + +function BinaryExpression:BinaryExpression( position, lvalue, operator, rvalue ) + parameters.checkConstructor( self.class, 4, "position", TokenPosition, position, "lvalue", Expression, lvalue, "operator", "number", operator, "rvalue", Expression, rvalue ) + + self.position = position + self.lvalue = lvalue + self.operator = operator + self.rvalue = rvalue +end + +function BinaryExpression:resolve( env ) + local lvalue, rvalue = self.lvalue:resolve( env ), self.rvalue:resolve( env ) + + if type( lvalue ) ~= "number" then + Exception.throw( ParserException( "expected number lvalue, got " .. type( lvalue ), self.position ) ) + elseif type( rvalue ) ~= "number" then + Exception.throw( ParserException( "expected string rvalue, got " .. type( rvalue ), self.position ) ) + end + + if self.operator == OPERATOR_BINARY_ADD then + return lvalue + rvalue + elseif self.operator == OPERATOR_BINARY_SUB then + return lvalue - rvalue + elseif self.operator == OPERATOR_BINARY_MUL then + return lvalue * rvalue + elseif self.operator == OPERATOR_BINARY_DIV then + return lvalue / rvalue + elseif self.operator == OPERATOR_BINARY_MOD then + return lvalue % rvalue + elseif self.operator == OPERATOR_BINARY_POW then + return lvalue ^ rvalue + end +end + +function BinaryExpression:substitute( env ) + self.lvalue:substitute( env ) + self.rvalue:substitute( env ) +end + +function BinaryExpression:isConstant() + return self.lvalue:isConstant() and self.rvalue:isConstant() +end + +function BinaryExpression:tostring() + return "(" .. self.lvalue:tostring() .. ")" .. tostring[self.operator] .. "(" .. self.rvalue:tostring() .. ")" +end diff --git a/src/dynamic/ConstantExpression.lua b/src/dynamic/ConstantExpression.lua new file mode 100644 index 0000000..3f8be38 --- /dev/null +++ b/src/dynamic/ConstantExpression.lua @@ -0,0 +1,30 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.dynamic.ConstantExpression' + -- @endif + + -- @print Including sheets.dynamic.ConstantExpression + +class "ConstantExpression" extends "Expression" { + value = 0; +} + +function ConstantExpression:ConstantExpression( position, value ) + parameters.checkConstructor( self.class, 1, "position", TokenPosition, position ) + self.value = value + self.position = position +end + +function ConstantExpression:resolve( env ) + return self.value +end + +function ConstantExpression:isConstant() + return true +end + +function ConstantExpression:tostring() + return tostring( self.value ) +end diff --git a/src/dynamic/Expression.lua b/src/dynamic/Expression.lua new file mode 100644 index 0000000..2f9ebdb --- /dev/null +++ b/src/dynamic/Expression.lua @@ -0,0 +1,36 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.dynamic.Expression' + -- @endif + + -- @print Including sheets.dynamic.Expression + +class "Expression" { + position = nil; +} + +function Expression:Expression( position ) + self.position = position +end + +function Expression:resolve( env ) + +end + +function Expression:substitute( env ) + +end + +function Expression:isConstant() + +end + +function Expression:tostring() + +end + +function Expression:throw( err ) + return Exception.throw( ExpressionException( self.position:tostring() .. ": " .. tostring( err ), 0 ) ) +end diff --git a/src/dynamic/ExpressionEnvironment.lua b/src/dynamic/ExpressionEnvironment.lua new file mode 100644 index 0000000..37febfe --- /dev/null +++ b/src/dynamic/ExpressionEnvironment.lua @@ -0,0 +1,25 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.dynamic.ExpressionEnvironment' + -- @endif + + -- @print Including sheets.dynamic.ExpressionEnvironment + +class "ExpressionEnvironment" { + environment = {}; +} + +function ExpressionEnvironment:ExpressionEnvironment() + self.environment = {} +end + +function ExpressionEnvironment:set( index, value ) + self.environment[index] = value + return self +end + +function ExpressionEnvironment:get( index ) + return self.environment[index] +end diff --git a/src/dynamic/ExpressionParser.lua b/src/dynamic/ExpressionParser.lua new file mode 100644 index 0000000..70b468d --- /dev/null +++ b/src/dynamic/ExpressionParser.lua @@ -0,0 +1,176 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.dynamic.ExpressionParser' + -- @endif + + -- @print Including sheets.dynamic.ExpressionParser + +local operators = { + ["+"] = true; + ["-"] = OPERATOR_UNARY_MINUS; + ["*"] = true; + ["/"] = true; + ["%"] = true; + ["^"] = true; + ["#"] = OPERATOR_UNARY_LEN; + ["!"] = OPERATOR_UNARY_NOT; +} + +local binary_operators = { + ["+"] = OPERATOR_BINARY_ADD; + ["-"] = OPERATOR_BINARY_SUB; + ["*"] = OPERATOR_BINARY_MUL; + ["/"] = OPERATOR_BINARY_DIV; + ["%"] = OPERATOR_BINARY_MOD; + ["^"] = OPERATOR_BINARY_POW; +} + +local precedence = { + ["+"] = 0; + ["-"] = 0; + ["*"] = 1; + ["/"] = 1; + ["%"] = 1; + ["^"] = 2; +} + +class "ExpressionParser" extends "Parser" { + token = 1; +} + +function ExpressionParser:consume() + if self:finished() then + return Token( TOKEN_EOF, nil, self.position ) + end + + if self:matchString() then + return self:consumeString() + elseif self:matchNumber() then + return self:consumeNumber() + elseif self:matchIdentifier() then + return self:consumeIdentifier() + elseif self:matchWhitespace() then + return self:consumeWhitespace() + else + self.character = self.character + 1 + if operators[self.text:sub( self.character - 1, self.character - 1 )] then + return Token( TOKEN_OPERATOR, self.text:sub( self.character - 1, self.character - 1 ), self:advancePosition( 1 ) ) + end + return Token( TOKEN_SYMBOL, self.text:sub( self.character - 1, self.character - 1 ), self:advancePosition( 1 ) ) + end +end + +function ExpressionParser:peek( n ) + return self.tokens[self.token + ( n or 0 )] or Token( TOKEN_EOF, nil, self.position ) +end + +function ExpressionParser:next() + self.token = self.token + 1 + return self:peek( -1 ) +end + +function ExpressionParser:test( ... ) + return self:peek():matches( ... ) +end + +function ExpressionParser:parseUnaryLeftOperator() + if self:test( TOKEN_OPERATOR, "-" ) or self:test( TOKEN_OPERATOR, "#" ) or self:test( TOKEN_OPERATOR, "!" ) then + return self:next() + end +end + +function ExpressionParser:parseUnaryRightOperator( lvalue ) + if self:test( TOKEN_SYMBOL, "(" ) then + -- stuff + elseif self:test( TOKEN_SYMBOL, "." ) then + -- stuff + end + return lvalue +end + +function ExpressionParser:parseAtom() + if self:test( TOKEN_STRING ) or self:test( TOKEN_INT ) or self:test( TOKEN_FLOAT ) then + return ConstantExpression( self:peek().position, self:next().value ) + elseif self:test( TOKEN_IDENT ) then + return IdentifierExpression( self:peek().position, self:next().value ) + else + Exception.throw( ParserException( "expected constant or identifier, got " .. self:peek().token, self:peek().position ) ) + end +end + +function ExpressionParser:parseNode() + local lops = {} + local node + while true do + local lop = self:parseUnaryLeftOperator() + if lop then + lops[#lops + 1] = lop + else + break + end + end + + node = self:parseAtom() + + while true do + local rop = self:parseUnaryRightOperator( node ) + if rop ~= node then + node = rop + else + break + end + end + + for i = #lops, 1, -1 do + node = UnaryLeftExpression( lops[i].position, operators[lops[i].value], node ) + end + + return node +end + +function ExpressionParser:parseBinaryOperator( lvalue ) + local operator = self:peek().value + local position = self:next().position + local p = precedence[operator] + local rvalue = self:parseNode() + + while true do + if self:test( TOKEN_OPERATOR ) then + local op = self:peek().value + position = self:peek().position + + if not precedence[op] then + Exception.throw( ParserException( "expected binary operator, got " .. self:peek().token, self:peek().position ) ) + end + + if precedence[op] > p then + rvalue = self:parseBinaryOperator( rvalue ) + elseif precedence[op] == p then + return self:parseBinaryOperator( BinaryExpression( position, lvalue, binary_operators[operator], rvalue ) ) + else + return BinaryExpression( position, lvalue, binary_operators[operator], rvalue ) + end + else + return BinaryExpression( position, lvalue, binary_operators[operator], rvalue ) + end + end +end + +function ExpressionParser:parseExpression() + local lvalue = self:parseNode() + + while self:test( TOKEN_OPERATOR ) do + if precedence[self:peek().value] then + lvalue = self:parseBinaryOperator( lvalue ) + else + Exception.throw( ParserException( "expected binary operator, got " .. self:peek().token, self:peek().position ) ) + end + end + if not self:test( TOKEN_EOF ) then + Exception.throw( ParserException( "unexpected " .. self:peek().token, self:peek().position ) ) + end + + return lvalue +end diff --git a/src/dynamic/IdentifierExpression.lua b/src/dynamic/IdentifierExpression.lua new file mode 100644 index 0000000..de99d1f --- /dev/null +++ b/src/dynamic/IdentifierExpression.lua @@ -0,0 +1,36 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.dynamic.IdentifierExpression' + -- @endif + + -- @print Including sheets.dynamic.IdentifierExpression + +class "IdentifierExpression" extends "Expression" { + name = ""; + const_value = nil; +} + +function IdentifierExpression:IdentifierExpression( position, name ) + parameters.checkConstructor( self.class, 2, "position", TokenPosition, position, "name", "string", name ) + + self.name = name + self.position = position +end + +function IdentifierExpression:resolve( env ) + return self.const_value or env:get( self.name ) +end + +function IdentifierExpression:substitute( env ) + self.const_value = env:get( self.name ) +end + +function IdentifierExpression:isConstant() + return self.const_value ~= nil +end + +function IdentifierExpression:tostring() + return self.name +end diff --git a/src/dynamic/UnaryLeftExpression.lua b/src/dynamic/UnaryLeftExpression.lua new file mode 100644 index 0000000..f3f669e --- /dev/null +++ b/src/dynamic/UnaryLeftExpression.lua @@ -0,0 +1,59 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.dynamic.UnaryLeftExpression' + -- @endif + + -- @print Including sheets.dynamic.UnaryLeftExpression + +class "UnaryLeftExpression" extends "Expression" { + operator = nil; + rvalue = nil; +} + +function UnaryLeftExpression:UnaryLeftExpression( position, operator, rvalue ) + parameters.checkConstructor( self.class, 3, "position", TokenPosition, position, "operator", "number", operator, "rvalue", Expression, rvalue ) + + self.operator = operator + self.rvalue = rvalue + self.position = position +end + +function UnaryLeftExpression:resolve( env ) + local rvalue = self.rvalue:resolve( env ) + + if self.operator == OPERATOR_UNARY_MINUS then + if type( rvalue ) == "number" then + return -rvalue + else + Expression.throw() + end + elseif self.operator == OPERATOR_UNARY_LEN then + if type( rvalue ) == "string" then + return #rvalue + else + Exception.throw( ParserException( "expected string rvalue, got " .. type( rvalue ), self.position ) ) + end + elseif operator == OPERATOR_UNARY_NOT then + return not rvalue + end +end + +function UnaryLeftExpression:substitute( env ) + self.rvalue:substitute( env ) +end + +function UnaryLeftExpression:isConstant() + return self.rvalue:isConstant() +end + +function UnaryLeftExpression:tostring() + if self.operator == OPERATOR_UNARY_MINUS then + return "-(" .. self.rvalue:tostring() .. ")" + elseif self.operator == OPERATOR_UNARY_NOT then + return "!(" .. self.rvalue:tostring() .. ")" + elseif self.operator == OPERATOR_UNARY_LEN then + return "#(" .. self.rvalue:tostring() .. ")" + end +end diff --git a/src/dynamic/UnaryRightExpression.lua b/src/dynamic/UnaryRightExpression.lua new file mode 100644 index 0000000..4d9f3c9 --- /dev/null +++ b/src/dynamic/UnaryRightExpression.lua @@ -0,0 +1,88 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.dynamic.UnaryRightExpression' + -- @endif + + -- @print Including sheets.dynamic.UnaryRightExpression + +class "UnaryRightExpression" extends "Expression" { + operator = nil; + lvalue = nil; + data = nil; +} + +function UnaryRightExpression:UnaryRightExpression( position, operator, lvalue, data ) + parameters.checkConstructor( self.class, 4, "position", TokenPosition, position, "operator", "number", operator, "lvalue", Expression, lvalue, "data", operator == OPERATOR_UNARY_CALL and "table" or "string", data ) + + self.operator = operator + self.lvalue = lvalue + self.data = data + self.position = position +end + +function UnaryRightExpression:resolve( env ) + local lvalue = self.lvalue:resolve( env ) + + if self.operator == OPERATOR_UNARY_CALL then + if type( lvalue ) == "function" then + + local args = {} + for i = 1, #self.data do + args[i] = self.data[i]:resolve( env ) + end + + return lvalue( unpack( args ) ) + + else + return self:throw "can't call that" + end + elseif self.operator == OPERATOR_UNARY_INDEX then + if type( lvalue ) == "table" then + return lvalue[self.data] + else + return self:throw "can't index that" + end + end +end + +function UnaryRightExpression:substitute( env ) + + self.lvalue:substitute( env ) + + if self.operator == OPERATOR_UNARY_CALL then + for i = 1, #self.data do + self.data[i]:substitute( env ) + + if self.data[i]:isConstant() then + self.data[i] = self.data[i]:resolve( env ) + end + end + end +end + +function UnaryRightExpression:isConstant() + + if self.lvalue:isConstant() then + if self.operator == OPERATOR_UNARY_CALL then + for i = 1, #self.data do + if not self.data[i]:isConstant() then + return false + + end + end + end + return true + end + return false + +end + +function UnaryRightExpression:tostring() + if self.operator == OPERATOR_UNARY_CALL then + return stuff + elseif self.operator == OPERATOR_UNARY_INDEX then + return "(" .. self.lvalue:tostring() .. ")" .. "." .. self.data + end +end diff --git a/src/exceptions/Exception.lua b/src/exceptions/Exception.lua index 02c6040..b442183 100644 --- a/src/exceptions/Exception.lua +++ b/src/exceptions/Exception.lua @@ -50,19 +50,25 @@ function Exception:getTraceback( initial, delimiter ) parameters.check( 2, "initial", "string", initial, "delimiter", "string", delimiter ) + if #self.trace == 0 then return "" end + return initial .. table.concat( self.trace, delimiter ) end -function Exception:getDataAndTraceback( indent ) - parameters.check( 1, "indent", "number", indent or 1 ) - +function Exception:getData() if type( self.data ) == "string" or class.isClass( self.data ) or class.isInstance( self.data ) then - return tostring( self.data ) .. "\n" .. self:getTraceback( (" "):rep( indent or 1 ) .. "in ", "\n" .. (" "):rep( indent or 1 ) .. "in " ) + return tostring( self.data ) else - return textutils.serialize( self.data ) .. "\n" .. self:getTraceback( (" "):rep( indent or 1 ) .. "in ", "\n" .. (" "):rep( indent or 1 ) .. "in " ) + return textutils.serialize( seld.data ) end end +function Exception:getDataAndTraceback( indent ) + parameters.check( 1, "indent", "number", indent or 1 ) + + return self:getData() .. self:getTraceback( "\n" .. (" "):rep( indent or 1 ) .. "in ", "\n" .. (" "):rep( indent or 1 ) .. "in " ) +end + function Exception:tostring() return tostring( self.name ) .. " exception:\n " .. self:getDataAndTraceback( 4 ) end diff --git a/src/exceptions/ExpressionException.lua b/src/exceptions/ExpressionException.lua new file mode 100644 index 0000000..d6a7f86 --- /dev/null +++ b/src/exceptions/ExpressionException.lua @@ -0,0 +1,14 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.exceptions.ExpressionException' + -- @endif + + -- @print Including sheets.exceptions.ExpressionException + +class "ExpressionException" extends "Exception" + +function ExpressionException:ExpressionException( data, level ) + return self:Exception( "ExpressionException", data, level ) +end diff --git a/src/exceptions/ParserException.lua b/src/exceptions/ParserException.lua new file mode 100644 index 0000000..1a2f49d --- /dev/null +++ b/src/exceptions/ParserException.lua @@ -0,0 +1,26 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.exceptions.ParserException' + -- @endif + + -- @print Including sheets.exceptions.ParserException + +class "ParserException" extends "Exception" { + position = nil; +} + +function ParserException:ParserException( data, position ) + parameters.checkConstructor( self.class, 1, "position", TokenPosition, position ) + self.position = position + return self:Exception( "ParserException", data, 0 ) +end + +function ParserException:getData() + if type( self.data ) == "string" or class.isClass( self.data ) or class.isInstance( self.data ) then + return self.position:tostring() .. ": " .. tostring( self.data ) + else + return self.position:tostring() .. ": " .. textutils.serialize( seld.data ) + end +end diff --git a/src/parsing/Parser.lua b/src/parsing/Parser.lua new file mode 100644 index 0000000..ee59e0e --- /dev/null +++ b/src/parsing/Parser.lua @@ -0,0 +1,204 @@ + + -- @once + + -- @ifndef __INCLUDE_sheets + -- @error 'sheets' must be included before including 'sheets.parsing.Parser' + -- @endif + + -- @print Including sheets.parsing.Parser + +local escapes = { ["n"] = "\n", ["r"] = "\r", ["0"] = "\0" } + +class "Parser" { + position = TokenPosition(); + character = 1; + text = ""; + tokens = {}; +} + +function Parser:Parser( str, source ) + source = source or "string" + str = str or "" + + parameters.checkConstructor( self.class, 2, "string", "string", str, "source", "string", source ) + + self.text = str + self.position = TokenPosition( source, 1, 1 ) +end + +function Parser:throw( err ) + Exception.throw( ParserException( self.position:tostring() .. ": " .. tostring( err ), 0 ) ) +end + +function Parser:advancePosition( n ) + local p = self.position:clone() + self.position:addOn( n or 1 ) + return p +end + +function Parser:finished() + return self.character > #self.text +end + +function Parser:push( token ) + parameters.check( 1, "token", Token, token ) + self.tokens[#self.tokens + 1] = token +end + +function Parser:consume() + if self:finished() then + return Token( TOKEN_EOF, nil, self.position ) + end + + return Token( TOKEN_SYMBOL, self.text:sub( self.character, self.character ), self:advancePosition( 1 ) ) +end + +function Parser:lex() + while true do + local token = self:consume() + if token:matches( TOKEN_EOF ) then + break + else + self:push( token ) + end + end +end + +function Parser:consumeString() + local close = self.text:sub( self.character, self.character ) + local escaped = false + local text = self.text + local str = "" + + local pos = self.position:add( 1 ) + local start = self.position:clone() + + for i = self.character + 1, #text do + local char = text:sub( i, i ) + + if char == "\n" then + pos.character = 0 + pos.line = pos.line + 1 + end + + if escaped then + str = str .. ( escapes[char] or char ) + escaped = false + elseif char == close then + self.character = i + 1 + self.position = pos + 1 + return Token( TOKEN_STRING, str, start ) + else + str = str .. char + end + + pos:addOn( 1 ) + end + + return self:throw( "expected " .. close .. " to end string" ) +end + +function Parser:consumeNumber() + local text = self.text + local n = text:match( "^%d*%.?%d+", self.character ) + local e = text:match( "^e(%-?%d+)", self.character + #n ) + local added = #n + ( e and #e + 1 or 0 ) + local f = text:sub( self.character + added, self.character + added ) == "f" + local isInt = not n:find "%." and ( not e or e:sub( 1, 1 ) ) ~= "-" and not f + local num = tonumber( n ) * 10 ^ tonumber( e or 0 ) + + if f then + added = added + 1 + end + + self.character = self.character + added + + if isInt then + return Token( TOKEN_INT, num, self:advancePosition( added ) ) + else + return Token( TOKEN_FLOAT, num, self:advancePosition( added ) ) + end +end + +function Parser:consumeIdentifier() + local ident = self.text:match( "^[%w_]+", self.character ) + self.character = self.character + #ident + return Token( TOKEN_IDENT, ident, self:advancePosition( #ident ) ) +end + +function Parser:consumeSymbol() + self.character = self.character + 1 + return Token( TOKEN_SYMBOL, self.text:sub( self.character - 1, self.character - 1 ), self:advancePosition( 1 ) ) +end + +function Parser:consumeNewline() + self.position.character = 1 + self.position.line = self.position.line + 1 + return self:consume() +end + +function Parser:consumeWhitespace() + local l = #( self.text:match( "^[^\n%S]+", self.character + 1 ) or "" ) + + if self.text:sub( self.character, self.character ) == "\n" then + self.position.line = self.position.line + 1 + self.position.character = l + 1 + else + self.position:addOn( l + 1 ) + end + + self.character = self.character + 1 + l + + return self:consume() +end + +function Parser:consumeXMLComment() + local comment = self.text:match( "^", self.character ) + + if comment then + local lines = select( 2, comment:gsub( "\n", "" ) ) + local length = #comment:gsub( "^.+\n", "" ) + + self.character = self.character + #comment + + if lines > 0 then + self.position.character = 1 + self.position:addOn( TokenPosition( "", lines, length ) ) + else + self.position:addOn( length ) + end + else + self:throw "expected end to comment ('-->')" + end + + return self:consume() +end + +function Parser:matchString() + local char = self.text:sub( self.character, self.character ) + return char == "'" or char == '"' +end + +function Parser:matchNumber() + return self.text:find( "^%d*%.?%d", self.character ) ~= nil +end + +function Parser:matchIdentifier() + return self.text:find( "^[%w_]", self.character ) ~= nil +end + +function Parser:matchSymbol() + return true +end + +function Parser:matchNewline() + return self.text:sub( self.character, self.character ) == "\n" +end + +function Parser:matchWhitespace() + return self.text:find( "^%s", self.character ) ~= nil +end + +function Parser:matchXMLComment() + return self.text:sub( self.character, self.character + 3 ) == "