[#163] Moved BlockIterator into util, and implemented BigField iterator

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/BigField.java

@@ -1,8 +1,8 @@
 package net.zscript.javaclient.commandPaths;
 
 import net.zscript.model.components.Zchars;
-import net.zscript.tokenizer.BlockIterator;
 import net.zscript.tokenizer.ZscriptField;
+import net.zscript.util.BlockIterator;
 import net.zscript.util.ByteString;
 import net.zscript.util.ByteString.ByteAppendable;
 import net.zscript.util.ByteString.ByteStringBuilder;
@@ -111,7 +111,6 @@ public boolean isBigField() {
 
     @Override
     public BlockIterator iterator() {
-        // FIXME - this needs implementing, should ByteStrings be BlockIterable?
-        return BlockIterator.EMPTY;
+        return data.iterator();
     }
 }

clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/NumberField.java

@@ -1,7 +1,7 @@
 package net.zscript.javaclient.commandPaths;
 
-import net.zscript.tokenizer.BlockIterator;
 import net.zscript.tokenizer.ZscriptField;
+import net.zscript.util.BlockIterator;
 import net.zscript.util.ByteString;
 import net.zscript.util.ByteString.ByteStringBuilder;
 

receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/core/AbstractOutStream.java

@@ -3,8 +3,8 @@
 import java.nio.charset.StandardCharsets;
 
 import net.zscript.model.components.Zchars;
-import net.zscript.tokenizer.BlockIterator;
 import net.zscript.tokenizer.ZscriptField;
+import net.zscript.util.BlockIterator;
 
 public abstract class AbstractOutStream implements OutStream {
     /**

receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/AddressingContext.java

@@ -8,9 +8,9 @@
 import net.zscript.javareceiver.semanticParser.ContextView;
 import net.zscript.model.components.Zchars;
 import net.zscript.model.components.ZscriptStatus;
-import net.zscript.tokenizer.BlockIterator;
 import net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 import net.zscript.tokenizer.Tokenizer;
+import net.zscript.util.BlockIterator;
 import net.zscript.util.OptIterator;
 
 public class AddressingContext extends AbstractContext {

receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/CommandContext.java

@@ -10,12 +10,12 @@
 import net.zscript.javareceiver.semanticParser.ContextView;
 import net.zscript.model.components.Zchars;
 import net.zscript.model.components.ZscriptStatus;
-import net.zscript.tokenizer.BlockIterator;
 import net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 import net.zscript.tokenizer.ZscriptExpression;
 import net.zscript.tokenizer.ZscriptField;
 import net.zscript.tokenizer.ZscriptTokenExpression;
 import net.zscript.tokenizer.ZscriptTokenField;
+import net.zscript.util.BlockIterator;
 import net.zscript.util.ByteString;
 import net.zscript.util.OptIterator;
 

receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/modules/core/ZscriptCoreModule.java

@@ -4,7 +4,7 @@
 import net.zscript.javareceiver.execution.CommandContext;
 import net.zscript.javareceiver.modules.ZscriptModule;
 import net.zscript.model.components.ZscriptStatus;
-import net.zscript.tokenizer.BlockIterator;
+import net.zscript.util.BlockIterator;
 
 public class ZscriptCoreModule implements ZscriptModule {
     private final ZscriptCapabilitiesCommand capabilitiesCmd = new ZscriptCapabilitiesCommand();

receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/AbstractArrayTokenBuffer.java

@@ -8,6 +8,8 @@
 import static net.zscript.tokenizer.TokenBuffer.isMarker;
 import static net.zscript.tokenizer.TokenBuffer.isSequenceEndMarker;
 
+import net.zscript.util.BlockIterator;
+
 /**
  * Array based implementation of a TokenBuffer - the tokens making up incoming command or response sequences are encoded and accessed here. Rules are:
  * <ol>

receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/TokenBuffer.java

@@ -2,6 +2,8 @@
 
 import java.util.NoSuchElementException;
 
+import net.zscript.util.BlockIterator;
+
 /**
  * Defines the interface for the standard buffer for tokenizing Zscript.
  * <p/>

receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptField.java

@@ -1,5 +1,7 @@
 package net.zscript.tokenizer;
 
+import net.zscript.util.BlockIterator;
+
 /**
  * Represents a Zscript field, which may be either numeric or a big-field.
  */

receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenExpression.java

@@ -6,6 +6,7 @@
 
 import net.zscript.model.components.Zchars;
 import net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
+import net.zscript.util.BlockIterator;
 import net.zscript.util.ByteString;
 import net.zscript.util.OptIterator;
 

receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenField.java

@@ -2,6 +2,7 @@
 
 import net.zscript.model.components.Zchars;
 import net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
+import net.zscript.util.BlockIterator;
 
 /**
  * Given a ReadToken, this provides a "field view" of the token.

receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/TokenRingBufferReaderTest.java

@@ -17,6 +17,7 @@
 import net.zscript.tokenizer.TokenBuffer.TokenReader;
 import net.zscript.tokenizer.TokenBuffer.TokenReader.ReadToken;
 import net.zscript.tokenizer.TokenBuffer.TokenWriter;
+import net.zscript.util.BlockIterator;
 import net.zscript.util.OptIterator;
 
 class TokenRingBufferReaderTest {

receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/ZscriptTokenExpressionTest.java

@@ -8,6 +8,8 @@
 
 import org.junit.jupiter.api.Test;
 
+import net.zscript.util.BlockIterator;
+
 class ZscriptTokenExpressionTest {
     TokenBuffer             buffer      = TokenRingBuffer.createBufferWithCapacity(256);
     Tokenizer               tokenizer   = new Tokenizer(buffer.getTokenWriter(), 2);

receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/BlockIterator.java → util/misc/src/main/java/net/zscript/util/BlockIterator.java

@@ -1,10 +1,8 @@
-package net.zscript.tokenizer;
+package net.zscript.util;
 
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 
-import net.zscript.util.ByteString;
-
 /**
  * Defines an iterator that works over buffers of bytes, revealing them byte at a time, or in contiguous blocks.
  */

util/misc/src/main/java/net/zscript/util/ByteString.java

@@ -3,7 +3,6 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.Arrays;
-import java.util.Iterator;
 import java.util.NoSuchElementException;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -37,13 +36,13 @@ public int size() {
     }
 
     /**
-     * Returns a simple Iterator.
+     * Returns a block Iterator over the bytes in the string.
      *
-     * @return iterator
+     * @return iterator a block iterator
      */
     @Override
-    public Iterator<Byte> iterator() {
-        return new ByteIterator();
+    public BlockIterator iterator() {
+        return new ByteBlockIterator();
     }
 
     /**
@@ -69,6 +68,23 @@ public ByteString writeTo(final OutputStream out) throws IOException {
         return this;
     }
 
+    /**
+     * Similar to System.arraycopy, this allows the bytes to be written to a destination byte-array. If 'max' implies a larger number of bytes than available, or there's
+     * insufficient space in 'dest', then the actual number of bytes copied is correspondingly reduced, and returned. Areas of 'dest' that are before destPos or after the end of
+     * the writing are unaffected.
+     *
+     * @param srcPos  the start index in this ByteString
+     * @param dest    the destination byte-array
+     * @param destPos the start position in the destination array
+     * @param max     the max number of bytes to copy
+     * @return the actual number of bytes written
+     */
+    public int copyTo(int srcPos, byte[] dest, int destPos, int max) {
+        int len = Math.min(Math.min(bytes.length - srcPos, dest.length - destPos), max);
+        System.arraycopy(bytes, srcPos, dest, destPos, len);
+        return len;
+    }
+
     /**
      * Exposes this ByteString as an Appendable to make it easy to append to other ByteStrings.
      *
@@ -149,7 +165,7 @@ public static ByteString from(byte b) {
      * @param <T>      the type of the object
      * @return the resulting ByteString
      */
-    private static <T> ByteString from(T object, ByteAppender<? super T> appender) {
+    public static <T> ByteString from(T object, ByteAppender<? super T> appender) {
         return builder().append(object, appender).build();
     }
 
@@ -595,7 +611,7 @@ default ByteString toByteString(T object) {
     /**
      * Simple array iterator
      */
-    private class ByteIterator implements Iterator<Byte> {
+    private class ByteBlockIterator implements BlockIterator {
         int index = 0;
 
         @Override
@@ -610,6 +626,20 @@ public Byte next() {
             }
             return bytes[index++];
         }
+
+        @Override
+        public byte[] nextContiguous() {
+            return nextContiguous(bytes.length);
+        }
+
+        @Override
+        public byte[] nextContiguous(int maxLength) {
+            int    n     = Math.min(maxLength, bytes.length - index);
+            byte[] block = new byte[n];
+            int    len   = copyTo(index, block, 0, n);
+            index += len;
+            return block;
+        }
     }
 
     /**

util/misc/src/test/java/net/zscript/util/BlockIteratorTest.java

@@ -0,0 +1,25 @@
+package net.zscript.util;
+
+import java.util.NoSuchElementException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import org.junit.jupiter.api.Test;
+
+class BlockIteratorTest {
+    @Test
+    public void shouldFindNothingInEmptyIterator() {
+        assertThat(BlockIterator.EMPTY.hasNext()).isFalse();
+        assertThatThrownBy(BlockIterator.EMPTY::next).isInstanceOf(NoSuchElementException.class);
+        assertThatThrownBy(BlockIterator.EMPTY::nextContiguous).isInstanceOf(NoSuchElementException.class);
+        assertThatThrownBy(() -> BlockIterator.EMPTY.nextContiguous(12)).isInstanceOf(NoSuchElementException.class);
+    }
+
+    @Test
+    public void shouldAppendIterator() {
+        ByteString b1 = ByteString.from("this is a test", (s, b) -> b.appendUtf8(s));
+        ByteString b2 = ByteString.from(b1.iterator());
+        assertThat(b2.asString()).isEqualTo("this is a test");
+    }
+}

util/misc/src/test/java/net/zscript/util/ByteStringTest.java

@@ -2,10 +2,10 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.NoSuchElementException;
 
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatIndexOutOfBoundsException;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -68,6 +68,21 @@ public void shouldIterateBytes() {
         assertThatThrownBy(iter::next).isInstanceOf(NoSuchElementException.class);
     }
 
+    @Test
+    public void shouldIterateBlockBytes() {
+        var str  = ByteString.builder().appendUtf8("hello, London calling, £5 is a fiver").build();
+        var iter = str.iterator();
+
+        assertThat(iter.hasNext()).isTrue();
+        assertThat(iter.next()).isEqualTo((byte) 'h');
+        assertThat(iter.nextContiguous(8)).isEqualTo("ello, Lo".getBytes(ISO_8859_1));
+        assertThat(iter.nextContiguous(11)).isEqualTo("ndon callin".getBytes(ISO_8859_1));
+        assertThat(iter.nextContiguous(6)).containsExactly('g', ',', ' ', 0xc2, 0xa3, '5');
+        assertThat(iter.hasNext()).isTrue();
+        assertThat(iter.nextContiguous()).isEqualTo(" is a fiver".getBytes(ISO_8859_1));
+        assertThat(iter.hasNext()).isFalse();
+    }
+
     @Test
     public void shouldInitializeNewBuilderWithExistingContent() {
         var b = ByteString.builder().appendByte('Z').appendNumeric16(0x01a2);
@@ -152,7 +167,7 @@ public void shouldThrowOnOutOfRangeNumericValue() {
     @ParameterizedTest
     @CsvSource({ "0xf1e2d3c4,Zf1e2d3c4", "0xe2d3c4,Ze2d3c4", "0x3c4,Z3c4", "0x10000,Z10000", "0xffff,Zffff", "0,Z" })
     public void shouldWriteToByteArrayNumbers32(long value, String expected) {
-        assertThat(ByteString.builder().appendByte('Z').appendNumeric32(value).toByteArray()).containsExactly(expected.getBytes(StandardCharsets.ISO_8859_1));
+        assertThat(ByteString.builder().appendByte('Z').appendNumeric32(value).toByteArray()).containsExactly(expected.getBytes(ISO_8859_1));
     }
 
     @Test