lexer.js

// Removes comments from the given
// source string.
let remove_comments = source => {
  // Using [^\n] as per
  // https://stackoverflow.com/a/3850095/1371131.
  // That will match up to but not including
  // the line break.
  return source.replace(/;.*[^\n]*/g, "");
};

let last = a => a[a.length - 1];

let isNumber = a => /\d/.test(a);
let isDelimiter = a => /[{}\[\]()""]/.test(a)

let delimiterPairs = {
  "{": "}",
  "[": "]",
  "(": ")",
  "\"": "\""
}

let checkDelimiters = (tokenContainer) => {
  let delimiters = [];

  tokenContainer.result.forEach(token => {
    if (isDelimiter(token[0])) {
      if (token[0] === delimiterPairs[last(tokenContainer.unclosedDelimiters)]) {
        tokenContainer.unclosedDelimiters.pop();
      } else {
        tokenContainer.unclosedDelimiters.push(token[0]);
      }
    }
  });
}

// Takes a source string and turns
// it into a sequence of tokens.
let lex = source => {
  // Remove comments
  source = remove_comments(source);

  let tokenContainer = source
    .split("")
    .reduce((acc, curr, i) => {
      switch (acc.mode) {
        case "number": {
          // We want to close the number and ignore
          // the current character
          if (curr === " " || curr === "\n" || curr === ",") {
            acc.mode = null;

            return acc;
          // We want to ignore underscores (they serve
          // merely as optional thousand separators)
          } else if (curr === "_") {
            return acc;
          // A number must not contain the following
          // special characters. For example, in "foo(1 2 3)",
          // the last parenthesis is not part of the number 3,
          // even though they are not separated by a
          // whitespace.
          } else if (/[{}\[\]()#~]/.test(curr)) {
            acc.mode = null;
            acc.result.push([curr, curr]);

            return acc;
          // Consume the next character as part
          // of the number
          } else {
            let resultCount = acc.result.length;
            let currResultCount = acc.result[resultCount - 1].length;

            acc.result[resultCount - 1][currResultCount - 1] += curr;

            return acc;
          }
        }
        case "symbol": {
          // We want to close the symbol and ignore
          // the current character
          if (curr === " " || curr === "\n" || curr === ",") {
            acc.mode = null;

            return acc;
          // A symbol must not contain the following
          // special characters. For example, in "foo()",
          // the parentheses are not part of the symbol,
          // even though they are not separated by a
          // whitespace.
          } else if (/[{}\[\]()#~]/.test(curr)) {
            acc.mode = null;
            acc.result.push([curr, curr]);

            return acc;
          // Consume the next character as part
          // of the symbol
          } else {
            let resultCount = acc.result.length;
            let currResultCount = acc.result[resultCount - 1].length;

            acc.result[resultCount - 1][currResultCount - 1] += curr;

            return acc;
          }
        }
        case "string": {
          // We want to close the string
          if (curr === "\"") {
            acc.mode = null;
            acc.result.push(["\"", "\""]);

            return acc;
          // Consume the next character as part
          // of the string
          } else {
            let resultCount = acc.result.length;
            let currResultCount = acc.result[resultCount - 1].length;

            acc.result[resultCount - 1][currResultCount - 1] += curr;

            return acc;
          }
        }
        case "keyword": {
          // We want to close the keyword
          if (curr === " " || curr === "\n" || curr === ",") {
            acc.mode = null;

            return acc;
          // A keyword must not contain the following
          // special characters. For example, in "... :foo)" --
          // perhaps as the last element of an argument list --
          // the closing parenthesis is not part of the keyword,
          // even though they are not separated by a
          // whitespace.
          } else if (/[{}\[\]()#~]/.test(curr)) {
            acc.mode = null;
            acc.result.push([curr, curr]);

            return acc;
          // Consume the next character as part
          // of the keyword
          } else {
            let resultCount = acc.result.length;
            let currResultCount = acc.result[resultCount - 1].length;

            acc.result[resultCount - 1][currResultCount - 1] += curr;

            return acc;
          }
        }
        default: {
          // We want to open a string
          if (curr === "\"") {
            acc.result.push(["\"", "\""]);
            acc.mode = "string";
            acc.result.push(["string", ""]);

            return acc;
          // We want to open a keyword
          } else if (curr === ":") {
            acc.mode = "keyword";
            acc.result.push(["keyword", ""]);

            return acc;
          // We want to open a number
          } else if (
            isNumber(curr) ||
            (isNumber(source[i + 1]) && (curr === "-" || curr === "+"))
          ) {
            acc.mode = "number";
            // The current character is already the first
            // character in the result
            acc.result.push(["number", curr]);

            return acc;
          // All other cases: open a symbol
          // (or it's a character to be ignored)
          } else {
            if (curr === " " || curr === "\n" || curr === ",") {
              return acc;
            // Turn special characters for maps,
            // sets, function invocations etc into
            // tokens of their own.
            } else if (/[{}\[\]()#~]/.test(curr)) {
              acc.result.push([curr, curr]);

              return acc;
            }

            acc.mode = "symbol";
            // The current character is already the first
            // character in the result
            acc.result.push(["symbol", curr]);

            return acc;
          }
        }
      }
    }, {
      mode: null,
      result: [],
      unclosedDelimiters: []
    });

  checkDelimiters(tokenContainer);

  return tokenContainer;
};

// Next up: handle comments in lex fn instead of
// removing them beforehand. That should allow me
// to display error messages with accurate line
// and character numbers later on.

module.exports = { lex };

// The types of things the parser needs to recognize:
// Function/macro calls
// ✓ Primitives
// Special forms (which look like function calls, and
// maybe they can just *be* function calls (or macro calls), at least
// as far as the consumer of the language is concerned):
// ✓ - def
// ✓ - let (this should be achievable solely in terms of a
// function inside a macro:
// let([a 1
//      b (+ a 2)])
// =
// ((fn [a]
//   ((fn [b]) (+ a 2))) 1)
// ...and so on, ever more deeply nested.
// ✓ - fn
// ✓ (- maybe later: fn! for functions that can only access
//  the values of symbols they are explicitly given in the
//  form of arguments)
// ✓ Blocks
// ✓ Maps, Arrays, Sets
// ✓ Booleans (which look like symbols)
// Destructuring
// ✓ JS interop with .fn, .-attr, fn. for constructors, and set! like in ClojureScript
// ✓ How to use JS operators like +, -, etc?
// ✓ def (as a single statement, no block)
// ✓ if (takes at least one block, optionally a second one)
//
// No statements, no expressions. Only functions.
//
// Closing brackets and braces on same line, not next line
// (as a convention).
//
// Do I need to escape single and double quotes and
// backticks that are part of symbols, keywords, and
// strings? Is that a security issue or only a compile-time
// issue?
//
// ✓ Assume that opening parenthesis means we are invoking
// a function.
// ✓ Special functions that are guaranteed to be pure because
// the compiler raises an exception when the function body
// accesses a symbol the function wasn't explicitly passed
// (I got this idea from Brian Will, he briefly mentioned it
// in one of his videos, but I can't remember which one:
// https://www.youtube.com/user/briantwill/)