TokenInput.zu

#
# The Zimbu compiler written in Zimbu
#
# TokenInput class.
#
# Copyright 2009 Bram Moolenaar  All Rights Reserved.
# Licensed under the Apache License, Version 2.0.  See the LICENSE file or
# obtain a copy at: http://www.apache.org/licenses/LICENSE-2.0
#

IMPORT "Input.zu"
IMPORT "Token.zu"
IMPORT "Tokenizer.zu"

#= Wraps around an Input.  Input does the character level stuff, TokenInput
#= the token level stuff.
CLASS TokenInput @items=public   # TODO: restrict visibility
  Input       $input
  list<Token> $tokenStack     # pushed back tokens

  # Keep track of which Token.Type.id keywords have been used in this file.
  # This is for loading a built-in library only when it is used.
  set<string> $usedIdKeywords


  # Create a TokenInput for |input|.
  NEW(Input input)
    $input = input
    $tokenStack = NEW()
    $usedIdKeywords = NEW()
  }

  # Low level input function.
  # Gets one character at a time, removing CR characters.
  # Uses pushed character if there is one.
  # Returns IO.eof when there is nothing more to read.
  FUNC $get() int
    RETURN $input.get()
  }

  # Push |c| back, a following get() will use it.
  PROC $push(int c)
    $input.push(c)
  }

  # Get the next character and push it back.
  FUNC $peek() int
    int c = $get()
    $push(c)
    RETURN c
  }

  # Read a word, consisting only of ASCII alpha characters and '_'.
  # Does not skip over white space or punctuation.
  # Returns an empty string when the first character is not alpha.
  FUNC $getWord() string
    IO.StringWriter sw = NEW()
    WHILE TRUE
      int c = $get()
      IF !c.isAlpha() && c != '_'
        $push(c)
        BREAK
      }
      sw.write(c.asString())
    }
    RETURN sw.ToString()
  }

  # Skip over white space.  Return TRUE if at least one space was encountered.
  FUNC $skipWhite() bool
    int c = $get()
    IF c != ' '
      $push(c)
      RETURN FALSE
    }
    WHILE c == ' '
      c = $get()
    }
    $push(c)
    RETURN TRUE
  }

  # When there is a previously pushed back token, return it.
  # Otherwise read a token from the file and return it.
  FUNC $getToken() Token
    IF $tokenStack.Size() > 0
      RETURN $tokenStack.remove()
    }
    RETURN Tokenizer.get(THIS)
  }

  # Push |token| back, a following getToken() will use it.
  PROC $pushToken(Token token)
    $tokenStack.add(token)
  }

  # Return the next token to be read without actually reading it.
  FUNC $peekToken() Token
    IF $tokenStack.Size() > 0
      RETURN $tokenStack[$tokenStack.Size() - 1]
    }
    Token token = Tokenizer.get(THIS)
    $tokenStack.add(token)
    RETURN token
  }

  # Push tokens on the stack back to the input.
  # Used when switching from reading tokens to reading input directly.
  PROC $emptyStack()
    WHILE $tokenStack.Size() > 0
      Token t = $tokenStack.remove()
      FOR i IN t.value.Size() - 1 TO 0 STEP -1
        $push(t.value[i])
      }
    }
  }

  # Log an error at the current input position.
  PROC $error(string msg)
    LOG.error(msg, $input.pos)
  }

}