Prototype Delimited

source/main.cc

@@ -31,10 +31,10 @@ int main(int argc, char **argv) {
 
     auto start = std::chrono::high_resolution_clock::now();
 
-
-
-    std::string file_name = "/Volumes/Container/Projects/Helix/helix-lang/tests/main.hlx"; // relative to current working dir in POSIX shell (cmd/bash)
-
+    // "D:\projects\helix-lang\tests\main.hlx"
+    //std::string file_name = "/Volumes/Container/Projects/Helix/helix-lang/tests/main.hlx"; // relative to current working dir in POSIX shell (cmd/bash)
+    std::string file_name =
+        "D:\\projects\\helix-lang\\tests\\main.hlx"; // relative to current working dir in Windows shell (cmd/powershell)
     // read the file and tokenize its contents : stage 0
     TokenList tokens = Lexer(file_system::read_file(file_name), file_name).tokenize();
 
@@ -50,16 +50,18 @@ int main(int argc, char **argv) {
     // print the preprocessed tokens
     // print_tokens(tokens);
 
+    auto toks = static_cast<std::vector<Token>>(tokens);
+    auto node = std::make_unique<ast::Parentheses<ast::StringLiteral>>(toks);
 
-    auto node = std::make_unique<ast::Parentheses<ast::StringLiteral>>(tokens);
-
     node->parse();
 
     print(node->to_string());
-
+    /// PRINT THE NODES AND THE NODES WITH
+    ///
+
     // Print the time taken in nanoseconds and milliseconds
     print("time taken: ", diff.count() * 1e+9, " ns");
     print("            ", diff.count() * 1000, " ms");
 
     return 0;
-}
+}

source/parser/ast/include/ast.hh

@@ -226,15 +226,15 @@ struct ParseError {};
 //     { t.to_string() } -> std::string;
 // };
 
-using ParseResult = std::optional<ParseError>;
+using TokenListRef = std::span<token::Token>;
+using ParseResult = std::expected<TokenListRef,ParseError>;
 
-using TokenListRef = std::reference_wrapper<token::TokenList>;
 
 template <typename T>
 struct ASTBase;
 
 template <>
-struct ASTBase<void>:  {
+struct ASTBase<void>  {
     //     virtual std::expected<std::span<Token>,AstError> parse(std::span<Token> tokens) = 0;
     virtual ~ASTBase() = default;
     ASTBase() = default;
@@ -248,14 +248,17 @@ struct ASTBase<void>:  {
 };
 
 template <typename T>
-struct ASTBase : ASTBase<void> {
+struct ASTBase {
     virtual ~ASTBase() = default;
     ASTBase() = default;
     explicit ASTBase(TokenListRef parse_tokens);
     ASTBase(ASTBase &&) = default;
     ASTBase(const ASTBase &) = default;
     ASTBase &operator=(ASTBase &&) = default;
     ASTBase &operator=(const ASTBase &) = delete;
+
+        [[nodiscard]] virtual ParseResult parse() = 0;
+        [[nodiscard]] virtual std::string to_string() const = 0;
 };
 
 template <typename T>

source/parser/ast/include/nodes.hh

@@ -16,12 +16,14 @@
 
 #include <concepts>
 #include <functional>
+#include <include/printV2>
 #include <memory>
 #include <optional>
 #include <type_traits>
 #include <utility>
 #include <variant>
 
+#include "../../../../tests/lib/catch2"
 #include "include/error/error.hh"
 #include "parser/ast/include/ast.hh"
 #include "token/include/token.hh"
@@ -37,15 +39,15 @@
     name &operator=(const name &) = delete;  \
                                              \
   private:                                   \
-    std::optional<TokenListRef> tokens;
+    TokenListRef tokens;
 
 namespace parser::ast {
 using namespace token;
 
 // TODO: Rename to StringLiteral
 
 template <const char quote, const tokens toke_type>
-struct Quoted : ASTBase<Quoted<quote, toke_type>> {
+struct Quoted final : ASTBase<Quoted<quote, toke_type>> {
   public:
     enum class Format : char {
         Invalid,
@@ -58,10 +60,11 @@ struct Quoted : ASTBase<Quoted<quote, toke_type>> {
 
     AST_NODE_METHODS(Quoted);
 
-public:
+  public:
     ParseResult parse() override {
         // Do we have suffixes?
-        Token &toke = tokens->get().front();
+        Token &toke = tokens.front();
+        auto toks = tokens;
 
         if (toke.token_kind() != toke_type) {
             error::Error(error::Line(toke, "Expected a quote literal"));
@@ -72,29 +75,24 @@ public:
 
         if (format == Format::Invalid) {
             error::Error(error::Line(toke.file_name(), toke.line_number(), toke.column_number(), 1,
-                                    "Invalid format specifier"));
+                                     "Invalid format specifier"));
         }
 
         this->value = toke;
-
+        auto siz = this->format == Format::None ? 1 : 2;
         // Remove the format specifier (if it is there) and quotes
-        this->value.set_value(toke.value().substr(
-            this->format == Format::None ?  // Check if the there is a format specifier
-                1  // Remove the quotes only
-                :
-                2, // Remove the quotes and the format specifier
-            toke.value().size() - 3));
+        this->value.set_value(toke.value().substr(siz, toke.value().size() - (siz + 1)));
         // - 1 is the actual length
         // - 2 is removing the \0
         // - 3 removes the quote
 
-        print(this->value.value());
+        this->tokens = this->tokens.subspan(0, 1);
 
         // Check for format
         // TODO: make f"hi {name if !name.empty() else "john doe"}" -> string: "hi {}", fmt_args
         // (astExpr): name if !name.empty() else "john doe"
 
-        return std::nullopt;
+        return toks.subspan(1);
     };
 
     std::string to_string() const override {
@@ -118,15 +116,15 @@ public:
                 break;
         }
 
-        return format + quote + this->value.to_string() + quote;
+        return format + quote + this->value.value() + quote;
     };
 
   private:
     Token value;
     Format format = Format::None;
 };
 
-struct BoolLiteral : ASTBase<BoolLiteral> {
+struct BoolLiteral final : ASTBase<BoolLiteral> {
   public:
     enum class BoolValue : std::uint8_t {
         True = 't',
@@ -145,69 +143,78 @@ struct BoolLiteral : ASTBase<BoolLiteral> {
 using StringLiteral = Quoted<'"', tokens::LITERAL_STRING>;
 using CharLiteral = Quoted<'\'', tokens::LITERAL_CHAR>;
 
-
-template < tokens StartToken, const char StartChar, typename Middle, const char EndChar, const tokens EndTokens>
-struct Delimited:ASTBase<Delimited<StartToken, StartChar, Middle, EndChar, EndTokens>> {
+template <const tokens StartToken, const char StartChar, typename Middle, const char EndChar,
+          const tokens EndTokens>
+struct Delimited final : ASTBase<Delimited<StartToken, StartChar, Middle, EndChar, EndTokens>> {
   public:
     AST_NODE_METHODS(Delimited);
-  public:
-
-    Delimited(TokenList parse_tokens) : tokens(parse_tokens) {}
 
+  public:
     // const char start = StartChar;
     // const char end = EndChar;
-
 
-    ParseResult parse() {
+    virtual ParseResult parse() override final {
+
+        auto toks = tokens;
+
         // Check if the first token is the start token
-        if (tokens->get().front().token_kind() != StartToken) {
-            error::Error(error::Line(tokens->get().front(), "Expected a start token"));
-        }
+        if (tokens.front().token_kind() != StartToken) {
+            error::Error(error::Line(tokens.front(), "Expected a start token"));
+        };
 
-        // construct the middle
-        // make unique 
+        toks = toks.subspan(1);
 
-        this->value = Middle{tokens};
-
-        // Parse the middle
-        this->value.get().parse();
+        this->value.emplace(std::make_unique<Middle>(toks));
+
+        ParseResult foo = this->value.value()->parse();
+
+        if (foo.has_value()) {
+        } else {
+            // TODO: ERROR
+        }
+        toks = foo.value();
 
         // Check if the last token is the end token
-        if (tokens->get().back().token_kind() != EndTokens) {
-            error::Error(error::Line(tokens->get().back(), "Expected an end token"));
+        if (toks.front().token_kind() != EndTokens) {
+            error::Error(error::Line(toks.front(), "Expected an end token"));
         }
-
 
+        toks = toks.subspan(1);
+        this->tokens =
+            this->tokens.subspan(0, static_cast<size_t>(std::addressof(toks.back()) -
+                                                        std::addressof(this->tokens.front()) - 1));
 
-        return std::nullopt;
+        return toks;
     }
 
-    std::string to_string() const {
-        return StartChar + this->value.get().to_string() + EndChar;
+    std::string to_string() const override {
+        // return StartChar + this->value.to_string() + EndChar;
+        return StartChar + this->value.value()->to_string() + EndChar;
     }
-    
+
   private:
-    std::optional<Middle> value = std::nullopt;
+    std::optional<std::unique_ptr<Middle>> value;
 };
 
 template <ASTNode T>
-using Parentheses = Delimited<tokens::PUNCTUATION_OPEN_PAREN,'(', T, ')', tokens::PUNCTUATION_CLOSE_PAREN>;
+using Parentheses =
+    Delimited<tokens::PUNCTUATION_OPEN_PAREN, '(', T, ')', tokens::PUNCTUATION_CLOSE_PAREN>;
 
 template <ASTNode T>
-using CurlyBraces = Delimited<tokens::PUNCTUATION_OPEN_BRACE, '{', T, '}', tokens::PUNCTUATION_CLOSE_BRACE>;
+using CurlyBraces =
+    Delimited<tokens::PUNCTUATION_OPEN_BRACE, '{', T, '}', tokens::PUNCTUATION_CLOSE_BRACE>;
 
 template <ASTNode T>
-using SquareBrack = Delimited<tokens::PUNCTUATION_OPEN_BRACKET, '[', T, ']', tokens::PUNCTUATION_CLOSE_BRACKET>;
+using SquareBrack =
+    Delimited<tokens::PUNCTUATION_OPEN_BRACKET, '[', T, ']', tokens::PUNCTUATION_CLOSE_BRACKET>;
 
 template <ASTNode T>
-using AngleBrace  = Delimited<tokens::PUNCTUATION_OPEN_ANGLE, '<', T, '>', tokens::PUNCTUATION_CLOSE_ANGLE>;
+using AngleBrace =
+    Delimited<tokens::PUNCTUATION_OPEN_ANGLE, '<', T, '>', tokens::PUNCTUATION_CLOSE_ANGLE>;
 
 template <ASTNode T>
-using PipeDelimited = Delimited<tokens::OPERATOR_BITWISE_OR, '|', T, '|', tokens::OPERATOR_BITWISE_OR>;
-
-
-
-
+using PipeDelimited =
+    Delimited<tokens::OPERATOR_BITWISE_OR, '|', T, '|', tokens::OPERATOR_BITWISE_OR>;
 
 /*
     the following parsed and translated:

source/token/include/token.hh

@@ -18,6 +18,7 @@
 #include <memory>
 #include <optional>
 #include <shared_mutex>
+#include <span>
 #include <string>
 #include <string_view>
 #include <vector>
@@ -80,7 +81,7 @@ struct Token {
 class TokenList : public std::vector<Token> {
   private:
     std::string filename;
-    TokenList(std::string filename, std::vector<Token>::const_iterator start,
+    TokenList(const std::string &filename, std::vector<Token>::const_iterator start,
               std::vector<Token>::const_iterator end);
 
     class TokenListIter {
@@ -97,7 +98,7 @@ class TokenList : public std::vector<Token> {
 
         bool operator!=(const TokenListIter &other) const;
         bool operator==(const TokenListIter &other) const;
-        Token* operator->();  // TODO: change if a shared ptr is needed
+        Token *operator->();  // TODO: change if a shared ptr is needed
         TokenListIter &operator*();
         std::reference_wrapper<TokenListIter> operator--();
         std::reference_wrapper<TokenListIter> operator++();
@@ -136,10 +137,9 @@ class TokenList : public std::vector<Token> {
 
     void remove_left();
     void reset();
-    void append(const Token &token);
     TokenList slice(u64 start, u64 end);
 
-    [[nodiscard]] std::string file_name() const;
+    [[nodiscard]] const std::string &file_name() const;
     void insert_remove(TokenList &tokens, u64 start, u64 end);
 
     bool operator==(const TokenList &rhs) const;

source/token/source/lexer.cc

@@ -43,12 +43,12 @@ TokenList Lexer::tokenize() {
         }
 
         token.set_file_name(file_name);
-        tokens.append(token);
+        tokens.push_back(token);
     }
 
     token = get_eof();
     token.set_file_name(file_name);
-    tokens.append(token);
+    tokens.push_back(token);
 
     tokens.reset();
     return tokens;

source/token/source/token_list.cc

@@ -17,6 +17,7 @@
 #include <stdexcept>
 
 #include "include/colors_ansi.hh"
+#include "parser/ast/include/context.hh"
 #include "token/include/generate.hh"
 #include "token/include/token.hh"
 
@@ -25,10 +26,10 @@ TokenList::TokenList(std::string filename)
     : filename(std::move(filename))
     , it(this->cbegin()) {}
 
-TokenList::TokenList(std::string filename, std::vector<Token>::const_iterator start,
+TokenList::TokenList(const std::string& filename, std::vector<Token>::const_iterator start,
                      std::vector<Token>::const_iterator end)
     : std::vector<Token>(start, end)
-    , filename(std::move(filename)) {}
+    , filename(filename) {}
 
 void TokenList::remove_left() {
     this->erase(this->cbegin(), it);
@@ -37,9 +38,8 @@ void TokenList::remove_left() {
 
 void TokenList::reset() { it = this->cbegin(); }
 
-void TokenList::append(const Token &token) { this->push_back(token); }
 
-std::string TokenList::file_name() const { return filename; }
+const std::string& TokenList::file_name() const { return filename; }
 
 TokenList TokenList::slice(u64 start, u64 end) {
     if (end > this->size()) {
@@ -52,6 +52,7 @@ TokenList TokenList::slice(u64 start, u64 end) {
     return {this->filename, this->cbegin() + start_index, this->cbegin() + end_index};
 }
 
+
 /**
  * @brief Replaces tokens in the list from start to end with the provided tokens.
  *

tests/main.hlx

@@ -1,4 +1 @@
-
-("hello world");
-
-#
+(r"hello world");