From 13b63148cd92807ab5a77ec5ea860ab2b571d327 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexandre=20D=2E=20D=C3=ADaz?= <dev@redneboa.es>
Date: Sat, 23 Nov 2024 01:31:56 +0100
Subject: [PATCH] imp: lexer: unknown type

---
 CHANGELOG.md                      |  3 ++-
 src/js/page/trash/constants.mjs   |  1 +
 src/js/page/trash/interpreter.mjs | 27 ++++++++++++++++-----------
 3 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3c7bc1e..7dbfcc8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,10 @@
 # Changelog
 
-**11.6.1**
+**11.7.0**
 
 ```
 IMP: Command 'exportfile': Add CSV format [RFC-4180] (issue #138)
+IMP: Interpreter: Add unknown lexer type (issue $139)
 ```
 
 **11.6.0**
diff --git a/src/js/page/trash/constants.mjs b/src/js/page/trash/constants.mjs
index 0d36b99..2253c87 100644
--- a/src/js/page/trash/constants.mjs
+++ b/src/js/page/trash/constants.mjs
@@ -4,6 +4,7 @@
 
 
 export const LEXER = {
+  Unknown: 0,
   Delimiter: 1,
   Variable: 2,
   ArgumentShort: 3,
diff --git a/src/js/page/trash/interpreter.mjs b/src/js/page/trash/interpreter.mjs
index f8537be..e80c6ac 100644
--- a/src/js/page/trash/interpreter.mjs
+++ b/src/js/page/trash/interpreter.mjs
@@ -251,18 +251,16 @@ export default class Interpreter {
     let offset: number = options?.offset || 0;
     for (let i = 0; i < tokens_len; ++i) {
       const [token_type, token, raw] = tokens[i];
-      if (token_type === LEXER.Space) {
-        offset += raw.length;
-        continue;
+      if (token_type !== LEXER.Space) {
+        tokens_info.push({
+          value: token,
+          raw: raw,
+          type: token_type,
+          start: offset,
+          end: offset + raw.length,
+          index: i,
+        });
       }
-      tokens_info.push({
-        value: token,
-        raw: raw,
-        type: token_type,
-        start: offset,
-        end: offset + raw.length,
-        index: i,
-      });
       offset += raw.length;
     }
     return tokens_info;
@@ -420,6 +418,9 @@ export default class Interpreter {
     }
 
     if (ttype === LEXER.String || ttype === LEXER.StringSimple) {
+      if (typeof prev_token_info !== 'undefined' && LEXERDATA_EXTENDED.includes(prev_token_info[0])) {
+        ttype = LEXER.Unknown;
+      }
       token_san = this.#trimQuotes(token_san);
     }
     return [ttype, token_san, token];
@@ -760,6 +761,10 @@ export default class Interpreter {
       const token = res.inputTokens[0][index];
       let ignore_instr_eoi = false;
       switch (token.type) {
+        case LEXER.Unknown:
+          if (typeof options.ignoreErrors === 'undefined' || !options.ignoreErrors) {
+            throw new InvalidTokenError(token.value, token.start, token.end);
+          }
         case LEXER.Variable:
           {
             ignore_instr_eoi = true;