[#159] Moved Java Tokenizer to its own module

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/ValidatingResponse.java

@@ -2,9 +2,8 @@
 
 import java.util.OptionalInt;
 
-import net.zscript.javareceiver.tokenizer.BlockIterator;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
 import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public abstract class ValidatingResponse implements ZscriptResponse {
     protected final ZscriptExpression expression;

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/ZscriptResponse.java

@@ -2,7 +2,6 @@
 
 import java.util.OptionalInt;
 
-import net.zscript.javareceiver.tokenizer.BlockIterator;
 import net.zscript.model.components.Zchars;
 import net.zscript.model.components.ZscriptStatus;
 

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/commandnodes/ZscriptCommandNode.java

@@ -9,7 +9,7 @@
 import net.zscript.javaclient.commandPaths.ZscriptFieldSet;
 import net.zscript.javaclient.commandbuilder.Respondable;
 import net.zscript.javaclient.commandbuilder.ZscriptResponse;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
+import net.zscript.tokenizer.ZscriptExpression;
 import net.zscript.util.ByteString;
 
 public abstract class ZscriptCommandNode<T extends ZscriptResponse> extends CommandSequenceNode implements Respondable<T> {

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/defaultCommands/AbortCommandNode.java

@@ -1,14 +1,13 @@
 package net.zscript.javaclient.commandbuilder.defaultCommands;
 
-import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.Map;
 
 import net.zscript.javaclient.commandbuilder.commandnodes.CommandSequenceNode;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandNode;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
 import net.zscript.model.components.Zchars;
 import net.zscript.model.components.ZscriptStatus;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public class AbortCommandNode extends ZscriptCommandNode<DefaultResponse> {
 

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/defaultCommands/BlankCommandNode.java

@@ -1,10 +1,8 @@
 package net.zscript.javaclient.commandbuilder.defaultCommands;
 
 import net.zscript.javaclient.commandbuilder.commandnodes.CommandSequenceNode;
-import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandBuilder;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandNode;
-import net.zscript.javaclient.commandbuilder.ZscriptResponse;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public class BlankCommandNode extends ZscriptCommandNode<DefaultResponse> {
 

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/defaultCommands/DefaultResponse.java

@@ -3,7 +3,7 @@
 import java.util.OptionalInt;
 
 import net.zscript.javaclient.commandbuilder.ZscriptResponse;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public class DefaultResponse implements ZscriptResponse {
     private final ZscriptExpression expression;

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/defaultCommands/FailureCommandNode.java

@@ -1,15 +1,13 @@
 package net.zscript.javaclient.commandbuilder.defaultCommands;
 
-import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.Map;
 
-import net.zscript.javaclient.commandbuilder.ZscriptResponse;
 import net.zscript.javaclient.commandbuilder.commandnodes.CommandSequenceNode;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandNode;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
 import net.zscript.model.components.Zchars;
 import net.zscript.model.components.ZscriptStatus;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public class FailureCommandNode extends ZscriptCommandNode<DefaultResponse> {
 

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/defaultCommands/GenericCommandBuilder.java

@@ -1,12 +1,8 @@
 package net.zscript.javaclient.commandbuilder.defaultCommands;
 
-import java.nio.charset.StandardCharsets;
-
-import net.zscript.javaclient.commandbuilder.ZscriptResponse;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandBuilder;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandNode;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
-import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public class GenericCommandBuilder extends ZscriptCommandBuilder<DefaultResponse> {
     public ZscriptCommandBuilder<DefaultResponse> setField(byte key, int value) {

clients/java-client-lib/client-command-api/src/main/java/net/zscript/javaclient/commandbuilder/notifications/NotificationSection.java

@@ -3,7 +3,7 @@
 import net.zscript.javaclient.commandbuilder.Respondable;
 import net.zscript.javaclient.commandbuilder.ZscriptResponse;
 import net.zscript.javaclient.commandbuilder.commandnodes.ResponseCaptor;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public abstract class NotificationSection<T extends ZscriptResponse> implements Respondable<T> {
     public abstract Class<T> getResponseType();

clients/java-client-lib/client-command-api/src/test/java/net/zscript/javaclient/commandbuilder/DemoActivateCommand.java

@@ -2,7 +2,7 @@
 
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandBuilder;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandNode;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public class DemoActivateCommand extends ZscriptCommandNode<DemoActivateCommand.DemoActivateCommandResponse> {
     //    private final boolean alreadyActivated;

clients/java-client-lib/client-command-api/src/test/java/net/zscript/javaclient/commandbuilder/commandNodes/DemoCapabilitiesCommandBuilder.java

@@ -2,11 +2,9 @@
 
 import static java.nio.charset.StandardCharsets.ISO_8859_1;
 
-import net.zscript.javaclient.commandbuilder.ValidatingResponse;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandBuilder;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandNode;
-import net.zscript.javaclient.commandbuilder.defaultCommands.DefaultResponse;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public class DemoCapabilitiesCommandBuilder extends ZscriptCommandBuilder<DemoCapabilitiesCommandResponse> {
     public static final int USER_FIRMWARE     = 0;
@@ -74,4 +72,3 @@ public boolean canFail() {
         return false;
     }
 }
-

clients/java-client-lib/client-command-api/src/test/java/net/zscript/javaclient/commandbuilder/commandNodes/DemoCapabilitiesCommandResponse.java

@@ -1,7 +1,7 @@
 package net.zscript.javaclient.commandbuilder.commandNodes;
 
 import net.zscript.javaclient.commandbuilder.ValidatingResponse;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
+import net.zscript.tokenizer.ZscriptExpression;
 
 public class DemoCapabilitiesCommandResponse extends ValidatingResponse {
     private       int    infoType = DemoCapabilitiesCommandBuilder.USER_FIRMWARE;

clients/java-client-lib/client-command-builders/src/main/resources/templates/commands/JavaCommandBuilder.mustache

@@ -13,7 +13,7 @@ import net.zscript.javaclient.commandbuilder.defaultCommands.*;
 import net.zscript.javaclient.commandbuilder.*;
 import net.zscript.javaclient.commandbuilder.commandnodes.*;
 import net.zscript.javaclient.commandbuilder.notifications.*;
-import net.zscript.javareceiver.tokenizer.*;
+import net.zscript.tokenizer.*;
 import net.zscript.model.components.*;
 
 /**

clients/java-client-lib/client-command-builders/src/test/java/net/zscript/model/modules/testing/test/JavaCommandBuilderResponseTest.java

@@ -1,5 +1,7 @@
 package net.zscript.model.modules.testing.test;
 
+import java.util.function.Consumer;
+
 import static net.zscript.client.modules.test.testing.TestingModule.TestCommand0Command.TestCommand0Response.BitsetRespTestU.Lion;
 import static net.zscript.client.modules.test.testing.TestingModule.TestCommand0Command.TestCommand0Response.BitsetRespTestU.Tabby;
 import static net.zscript.client.modules.test.testing.TestingModule.TestCommand0Command.TestCommand0Response.BitsetRespTestU.Tiger;
@@ -8,20 +10,18 @@
 
 import org.junit.jupiter.api.Test;
 
-import java.util.function.Consumer;
-
 import net.zscript.client.modules.test.testing.TestingModule;
 import net.zscript.client.modules.test.testing.TestingModule.TestCommand0Command.TestCommand0Response.BitsetRespTestV;
 import net.zscript.client.modules.test.testing.TestingModule.TestCommand0Command.TestCommand0Response.EnumRespTestP;
 import net.zscript.client.modules.test.testing.TestingModule.TestCommand0Command.TestCommand0Response.EnumRespTestQ;
+import net.zscript.javaclient.commandbuilder.ZscriptResponse;
 import net.zscript.javaclient.commandbuilder.commandnodes.ResponseCaptor;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandBuilder;
 import net.zscript.javaclient.commandbuilder.commandnodes.ZscriptCommandNode;
-import net.zscript.javaclient.commandbuilder.ZscriptResponse;
-import net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader;
-import net.zscript.javareceiver.tokenizer.TokenExtendingBuffer;
-import net.zscript.javareceiver.tokenizer.Tokenizer;
-import net.zscript.javareceiver.tokenizer.ZscriptTokenExpression;
+import net.zscript.tokenizer.TokenBuffer.TokenReader;
+import net.zscript.tokenizer.TokenExtendingBuffer;
+import net.zscript.tokenizer.Tokenizer;
+import net.zscript.tokenizer.ZscriptTokenExpression;
 
 public class JavaCommandBuilderResponseTest {
     final TokenExtendingBuffer buffer      = new TokenExtendingBuffer();

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/CompleteAddressedResponse.java

@@ -4,12 +4,12 @@
 import java.util.List;
 import java.util.Optional;
 
-import static net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader;
-import static net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader.ReadToken;
+import static net.zscript.tokenizer.TokenBuffer.TokenReader;
+import static net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 
 import net.zscript.javaclient.sequence.ResponseSequence;
-import net.zscript.javareceiver.tokenizer.Tokenizer;
 import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.Tokenizer;
 import net.zscript.util.OptIterator;
 
 public class CompleteAddressedResponse {

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/ZscriptAddress.java

@@ -3,7 +3,7 @@
 import java.util.Arrays;
 import java.util.List;
 
-import static net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader.ReadToken;
+import static net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 
 import net.zscript.model.components.Zchars;
 import net.zscript.util.ByteString.ByteAppendable;

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/CommandExecutionPath.java

@@ -13,12 +13,12 @@
 import java.util.Optional;
 import java.util.Set;
 
-import static net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader.ReadToken;
+import static net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 
-import net.zscript.javareceiver.tokenizer.TokenBufferIterator;
-import net.zscript.javareceiver.tokenizer.Tokenizer;
 import net.zscript.model.ZscriptModel;
 import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.TokenBufferIterator;
+import net.zscript.tokenizer.Tokenizer;
 import net.zscript.util.ByteString.ByteAppendable;
 import net.zscript.util.ByteString.ByteStringBuilder;
 import net.zscript.util.OptIterator;

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ResponseExecutionPath.java

@@ -6,11 +6,11 @@
 import java.util.ListIterator;
 import java.util.Optional;
 
-import static net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader.ReadToken;
+import static net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 
-import net.zscript.javareceiver.tokenizer.TokenBufferIterator;
-import net.zscript.javareceiver.tokenizer.Tokenizer;
 import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.TokenBufferIterator;
+import net.zscript.tokenizer.Tokenizer;
 import net.zscript.util.ByteString.ByteAppendable;
 import net.zscript.util.ByteString.ByteStringBuilder;
 

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ZscriptFieldSet.java

@@ -9,11 +9,11 @@
 
 import static java.util.Collections.emptyList;
 
-import net.zscript.javareceiver.tokenizer.BlockIterator;
-import net.zscript.javareceiver.tokenizer.TokenBuffer;
-import net.zscript.javareceiver.tokenizer.TokenBufferIterator;
-import net.zscript.javareceiver.tokenizer.ZscriptExpression;
 import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.BlockIterator;
+import net.zscript.tokenizer.TokenBuffer;
+import net.zscript.tokenizer.TokenBufferIterator;
+import net.zscript.tokenizer.ZscriptExpression;
 import net.zscript.util.ByteString;
 import net.zscript.util.ByteString.ByteAppendable;
 import net.zscript.util.ByteString.ByteStringBuilder;

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/CommandSequence.java

@@ -2,12 +2,12 @@
 
 import java.util.Collection;
 
-import static net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader.ReadToken;
+import static net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 
 import net.zscript.javaclient.commandPaths.CommandExecutionPath;
-import net.zscript.javareceiver.tokenizer.TokenBufferIterator;
 import net.zscript.model.ZscriptModel;
 import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.TokenBufferIterator;
 import net.zscript.util.ByteString.ByteAppendable;
 import net.zscript.util.ByteString.ByteStringBuilder;
 

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/ResponseSequence.java

@@ -1,9 +1,9 @@
 package net.zscript.javaclient.sequence;
 
 import net.zscript.javaclient.commandPaths.ResponseExecutionPath;
-import net.zscript.javareceiver.tokenizer.TokenBuffer;
-import net.zscript.javareceiver.tokenizer.TokenBufferIterator;
 import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.TokenBuffer;
+import net.zscript.tokenizer.TokenBufferIterator;
 import net.zscript.util.ByteString.ByteAppendable;
 import net.zscript.util.ByteString.ByteStringBuilder;
 

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/ZscriptLockSet.java

@@ -2,7 +2,7 @@
 
 import java.util.BitSet;
 
-import static net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader.ReadToken;
+import static net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 
 import net.zscript.model.components.Zchars;
 import net.zscript.util.ByteString.ByteAppendable;

clients/java-client-lib/client-core/src/test/java/net/zscript/javaclient/addressing/ZscriptAddressTest.java

@@ -10,12 +10,12 @@
 import nl.jqno.equalsverifier.EqualsVerifier;
 import org.junit.jupiter.api.Test;
 
-import net.zscript.javareceiver.tokenizer.TokenBuffer;
-import net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader;
-import net.zscript.javareceiver.tokenizer.TokenBuffer.TokenReader.ReadToken;
-import net.zscript.javareceiver.tokenizer.TokenRingBuffer;
-import net.zscript.javareceiver.tokenizer.Tokenizer;
 import net.zscript.model.components.Zchars;
+import net.zscript.tokenizer.TokenBuffer;
+import net.zscript.tokenizer.TokenBuffer.TokenReader;
+import net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
+import net.zscript.tokenizer.TokenRingBuffer;
+import net.zscript.tokenizer.Tokenizer;
 import net.zscript.util.ByteString;
 
 class ZscriptAddressTest {

clients/java-client-lib/client-core/src/test/java/net/zscript/javaclient/commandPrinting/CommandExecutionPathRegenerationTest.java

@@ -11,8 +11,8 @@
 import org.junit.jupiter.params.provider.MethodSource;
 
 import net.zscript.javaclient.commandPaths.CommandExecutionPath;
-import net.zscript.javareceiver.tokenizer.TokenExtendingBuffer;
-import net.zscript.javareceiver.tokenizer.Tokenizer;
+import net.zscript.tokenizer.TokenExtendingBuffer;
+import net.zscript.tokenizer.Tokenizer;
 
 public class CommandExecutionPathRegenerationTest {
 

clients/java-client-lib/client-core/src/test/java/net/zscript/javaclient/commandPrinting/CompleteCommandGrapherTest.java

@@ -13,8 +13,8 @@
 
 import net.zscript.ascii.AnsiCharacterStylePrinter;
 import net.zscript.javaclient.commandPaths.CommandExecutionPath;
-import net.zscript.javareceiver.tokenizer.TokenExtendingBuffer;
-import net.zscript.javareceiver.tokenizer.Tokenizer;
+import net.zscript.tokenizer.TokenExtendingBuffer;
+import net.zscript.tokenizer.Tokenizer;
 
 public class CompleteCommandGrapherTest {
     CommandGraph.GraphPrintSettings basicSettings = new CommandGraph.GraphPrintSettings(new StandardCommandGrapher.CommandPrintSettings("  ", VerbositySetting.NAME), true, 2,

clients/java-client-lib/client-main/pom.xml

@@ -29,11 +29,18 @@
             <version>[2.0.0,3.0.0)</version>
         </dependency>
 
+        <!-- Test Dependencies -->
         <dependency>
             <groupId>net.zscript</groupId>
             <artifactId>zscript-java-client-command-builders</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>net.zscript</groupId>
+            <artifactId>java-receiver</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

clients/java-client-lib/client-main/src/main/java/net/zscript/javaclient/connectors/RawConnection.java

@@ -14,8 +14,8 @@
 import net.zscript.javaclient.addressing.CompleteAddressedResponse;
 import net.zscript.javaclient.nodes.Connection;
 import net.zscript.javaclient.threading.ZscriptWorkerThread;
-import net.zscript.javareceiver.tokenizer.TokenExtendingBuffer;
-import net.zscript.javareceiver.tokenizer.Tokenizer;
+import net.zscript.tokenizer.TokenExtendingBuffer;
+import net.zscript.tokenizer.Tokenizer;
 
 public abstract class RawConnection implements Connection, AutoCloseable {
     private final ZscriptWorkerThread thread = new ZscriptWorkerThread();

clients/java-client-lib/client-main/src/main/java/net/zscript/javaclient/devices/Device.java

@@ -25,9 +25,9 @@
 import net.zscript.javaclient.commandbuilder.notifications.NotificationId;
 import net.zscript.javaclient.nodes.ZscriptNode;
 import net.zscript.javaclient.sequence.CommandSequence;
-import net.zscript.javareceiver.tokenizer.TokenExtendingBuffer;
-import net.zscript.javareceiver.tokenizer.Tokenizer;
 import net.zscript.model.ZscriptModel;
+import net.zscript.tokenizer.TokenExtendingBuffer;
+import net.zscript.tokenizer.Tokenizer;
 
 public class Device {
     private final ZscriptModel model;