From feb0d46e5dd908ad5a2f5d5685f55ca1cbd43851 Mon Sep 17 00:00:00 2001 From: susanw1 Date: Thu, 12 Sep 2024 00:17:05 +0100 Subject: [PATCH] [#163] renamed various iterator methods for clarity --- .../test/JavaCommandBuilderResponseTest.java | 2 +- .../addressing/CompleteAddressedResponse.java | 4 +-- .../javaclient/addressing/ZscriptAddress.java | 2 +- .../commandPaths/CommandExecutionPath.java | 2 +- .../commandPaths/ResponseExecutionPath.java | 2 +- .../commandPaths/ZscriptFieldSet.java | 4 +-- .../javaclient/sequence/CommandSequence.java | 2 +- .../javaclient/sequence/ResponseSequence.java | 2 +- .../addressing/ZscriptAddressTest.java | 2 +- .../execution/AddressingContext.java | 10 +++---- .../execution/CommandContext.java | 6 ++-- .../scriptSpaces/ScriptSpaceWriteCommand.java | 2 +- .../semanticParser/SemanticParser.java | 6 ++-- .../treeParser/ZscriptTreeParser.java | 10 +++---- .../semanticParser/ParserActionTester.java | 6 ++-- .../tokenizer/AbstractArrayTokenBuffer.java | 10 +++---- .../net/zscript/tokenizer/TokenBuffer.java | 17 ++++++----- .../tokenizer/ZscriptTokenExpression.java | 6 ++-- .../zscript/tokenizer/ZscriptTokenField.java | 2 +- .../tokenizer/TokenRingBufferReaderTest.java | 30 +++++++++---------- .../tokenizer/ZscriptTokenExpressionTest.java | 4 +-- 21 files changed, 66 insertions(+), 65 deletions(-) diff --git a/clients/java-client-lib/client-command-builders/src/test/java/net/zscript/model/modules/testing/test/JavaCommandBuilderResponseTest.java b/clients/java-client-lib/client-command-builders/src/test/java/net/zscript/model/modules/testing/test/JavaCommandBuilderResponseTest.java index d1a25260..dea15b73 100644 --- a/clients/java-client-lib/client-command-builders/src/test/java/net/zscript/model/modules/testing/test/JavaCommandBuilderResponseTest.java +++ b/clients/java-client-lib/client-command-builders/src/test/java/net/zscript/model/modules/testing/test/JavaCommandBuilderResponseTest.java @@ -237,7 +237,7 @@ private void checkResponse(final String responseChar ResponseCaptor captor = ResponseCaptor.create(); ZscriptCommandNode cmd = commandBuilder.capture(captor).build(); - listener.accept(cmd.parseResponse(new ZscriptTokenExpression(tokenReader::iterator))); + listener.accept(cmd.parseResponse(new ZscriptTokenExpression(tokenReader::tokenIterator))); } } diff --git a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/CompleteAddressedResponse.java b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/CompleteAddressedResponse.java index ee8c1a84..c4160570 100644 --- a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/CompleteAddressedResponse.java +++ b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/CompleteAddressedResponse.java @@ -17,7 +17,7 @@ public class CompleteAddressedResponse { private final ResponseSequence content; public static CompleteAddressedResponse parse(TokenReader reader) { - OptIterator iterEnding = reader.iterator(); + OptIterator iterEnding = reader.tokenIterator(); for (Optional opt = iterEnding.next(); opt.isPresent(); opt = iterEnding.next()) { if (opt.get().isSequenceEndMarker()) { if (opt.get().getKey() != Tokenizer.NORMAL_SEQUENCE_END) { @@ -25,7 +25,7 @@ public static CompleteAddressedResponse parse(TokenReader reader) { } } } - OptIterator iter = reader.iterator(); + OptIterator iter = reader.tokenIterator(); List addresses = new ArrayList<>(); ResponseSequence seq = null; for (Optional opt = iter.next(); opt.isPresent(); opt = iter.next()) { diff --git a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/ZscriptAddress.java b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/ZscriptAddress.java index 388321da..093f1c7e 100644 --- a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/ZscriptAddress.java +++ b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/addressing/ZscriptAddress.java @@ -52,7 +52,7 @@ public static ZscriptAddress parse(ReadToken token) { } // Note, tokenizer will only write one Address; subsequent addresses are inside the single token envelope. - int[] parts = token.getNextTokens().stream() + int[] parts = token.tokenIterator().stream() .takeWhile(t -> Zchars.isAddressing(t.getKey())) .mapToInt(ReadToken::getData16) .toArray(); diff --git a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/CommandExecutionPath.java b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/CommandExecutionPath.java index b00a6d55..7cdc5b5d 100644 --- a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/CommandExecutionPath.java +++ b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/CommandExecutionPath.java @@ -65,7 +65,7 @@ private static List createLinkedPaths(ReadToken start) { return builders; } - TokenBufferIterator iterator = start.getNextTokens(); + TokenBufferIterator iterator = start.tokenIterator(); for (Optional opt = iterator.next(); opt.isPresent(); opt = iterator.next()) { ReadToken token = opt.get(); if (last.getStart() == null) { diff --git a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ResponseExecutionPath.java b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ResponseExecutionPath.java index 2a4efbb7..6e0e1df7 100644 --- a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ResponseExecutionPath.java +++ b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ResponseExecutionPath.java @@ -50,7 +50,7 @@ private static List createLinkedPath(ReadToken start) { if (start == null) { return builders; } - TokenBufferIterator iterator = start.getNextTokens(); + TokenBufferIterator iterator = start.tokenIterator(); for (Optional opt = iterator.next(); opt.isPresent(); opt = iterator.next()) { ReadToken token = opt.get(); if (last.getStart() == null) { diff --git a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ZscriptFieldSet.java b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ZscriptFieldSet.java index e585b2ff..a641336c 100644 --- a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ZscriptFieldSet.java +++ b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/commandPaths/ZscriptFieldSet.java @@ -34,11 +34,11 @@ public static ZscriptFieldSet fromTokens(TokenBuffer.TokenReader.ReadToken start Arrays.fill(fields, -1); boolean hasClash = false; - TokenBufferIterator iterator = start.getNextTokens(); + TokenBufferIterator iterator = start.tokenIterator(); for (Optional opt = iterator.next(); opt.filter(t -> !t.isMarker()).isPresent(); opt = iterator.next()) { TokenBuffer.TokenReader.ReadToken token = opt.get(); if (Zchars.isBigField(token.getKey())) { - bigFields.add(new BigField(ByteString.from(token.blockIterator()), token.getKey() == Zchars.Z_BIGFIELD_QUOTED)); + bigFields.add(new BigField(ByteString.from(token.dataIterator()), token.getKey() == Zchars.Z_BIGFIELD_QUOTED)); } else { if (fields[token.getKey() - 'A'] != -1 || token.getDataSize() > 2 || !Zchars.isNumericKey(token.getKey())) { hasClash = true; diff --git a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/CommandSequence.java b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/CommandSequence.java index 0933df00..9671d6c4 100644 --- a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/CommandSequence.java +++ b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/CommandSequence.java @@ -41,7 +41,7 @@ public static CommandSequence from(CommandExecutionPath path, int echoField, Zsc public static CommandSequence parse(ZscriptModel model, ReadToken start, boolean supports32Locks) { ZscriptLockSet locks = null; int echoField = -1; - TokenBufferIterator iter = start.getNextTokens(); + TokenBufferIterator iter = start.tokenIterator(); ReadToken current = iter.next().orElse(null); while (current != null && (current.getKey() == Zchars.Z_LOCKS || current.getKey() == Zchars.Z_ECHO)) { if (current.getKey() == Zchars.Z_LOCKS) { diff --git a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/ResponseSequence.java b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/ResponseSequence.java index b8280687..6a55d2ca 100644 --- a/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/ResponseSequence.java +++ b/clients/java-client-lib/client-core/src/main/java/net/zscript/javaclient/sequence/ResponseSequence.java @@ -18,7 +18,7 @@ public static ResponseSequence parse(TokenBuffer.TokenReader.ReadToken start) { int echoField = -1; int responseField = -1; - TokenBufferIterator iter = start.getNextTokens(); + TokenBufferIterator iter = start.tokenIterator(); TokenBuffer.TokenReader.ReadToken current = iter.next().orElse(null); if (current == null || current.getKey() != Zchars.Z_RESPONSE_MARK) { diff --git a/clients/java-client-lib/client-core/src/test/java/net/zscript/javaclient/addressing/ZscriptAddressTest.java b/clients/java-client-lib/client-core/src/test/java/net/zscript/javaclient/addressing/ZscriptAddressTest.java index 88828850..76873360 100644 --- a/clients/java-client-lib/client-core/src/test/java/net/zscript/javaclient/addressing/ZscriptAddressTest.java +++ b/clients/java-client-lib/client-core/src/test/java/net/zscript/javaclient/addressing/ZscriptAddressTest.java @@ -54,7 +54,7 @@ public void shouldParseSimpleAddress() { public void shouldNotParseMultipleAddresses() { tokenize("@12.34.56 @78.ab Z1A2+34\n"); - List addresses = tokenReader.iterator().stream() + List addresses = tokenReader.tokenIterator().stream() .filter(t -> t.getKey() == Zchars.Z_ADDRESSING) .map(ZscriptAddress::parse) .collect(toList()); diff --git a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/AddressingContext.java b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/AddressingContext.java index 6566cf60..50019687 100644 --- a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/AddressingContext.java +++ b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/AddressingContext.java @@ -22,11 +22,11 @@ public AddressingContext(final Zscript zscript, final ContextView contextView) { } public BlockIterator getAddressedData() { - OptIterator iterator = contextView.getReader().iterator(); + OptIterator iterator = contextView.getReader().tokenIterator(); for (Optional opt = iterator.next(); opt.isPresent(); opt = iterator.next()) { ReadToken token = opt.get(); if (token.getKey() == Tokenizer.ADDRESSING_FIELD_KEY) { - return token.blockIterator(); + return token.dataIterator(); } } throw new IllegalStateException("No data with addressing"); @@ -34,7 +34,7 @@ public BlockIterator getAddressedData() { public OptIterator getAddressSegments() { return new OptIterator<>() { - final OptIterator internal = contextView.getReader().iterator(); + final OptIterator internal = contextView.getReader().tokenIterator(); @Override public Optional next() { @@ -58,7 +58,7 @@ public boolean status(byte status) { } public int getAddressedDataSize() { - final OptIterator it = contextView.getReader().iterator(); + final OptIterator it = contextView.getReader().tokenIterator(); for (Optional opt = it.next(); opt.isPresent(); opt = it.next()) { ReadToken token = opt.get(); if (token.getKey() == Tokenizer.ADDRESSING_FIELD_KEY) { @@ -72,7 +72,7 @@ public boolean verify() { boolean hasReachedData = false; int i = 0; - OptIterator iterator = contextView.getReader().iterator(); + OptIterator iterator = contextView.getReader().tokenIterator(); for (Optional opt = iterator.next(); opt.isPresent(); opt = iterator.next()) { ReadToken token = opt.get(); i++; diff --git a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/CommandContext.java b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/CommandContext.java index a7433e45..5344e737 100644 --- a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/CommandContext.java +++ b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/execution/CommandContext.java @@ -32,7 +32,7 @@ public CommandContext(final Zscript zscript, final ContextView contextView, fina super(contextView); this.zscript = zscript; this.out = out; - this.expression = new ZscriptTokenExpression(() -> contextView.getReader().iterator()); + this.expression = new ZscriptTokenExpression(() -> contextView.getReader().tokenIterator()); } public CommandOutStream getOutStream() { @@ -65,8 +65,8 @@ public boolean hasBigField() { * * @return fully big-field-aware data iterator */ - public BlockIterator getBigField() { - return expression.getBigField(); + public BlockIterator bigFieldDataIterator() { + return expression.bigFieldDataIterator(); } @Override diff --git a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/modules/scriptSpaces/ScriptSpaceWriteCommand.java b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/modules/scriptSpaces/ScriptSpaceWriteCommand.java index b1844fbb..b60ea352 100644 --- a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/modules/scriptSpaces/ScriptSpaceWriteCommand.java +++ b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/modules/scriptSpaces/ScriptSpaceWriteCommand.java @@ -37,7 +37,7 @@ public static void execute(List spaces, CommandContext ctx) { } Tokenizer tok = new Tokenizer(writer.getTokenWriter(), 2); - for (Iterator iterator = ctx.getBigField(); iterator.hasNext(); ) { + for (Iterator iterator = ctx.bigFieldDataIterator(); iterator.hasNext(); ) { tok.accept(iterator.next()); } writer.commitChanges(); diff --git a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/semanticParser/SemanticParser.java b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/semanticParser/SemanticParser.java index 089376b8..99d8898c 100644 --- a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/semanticParser/SemanticParser.java +++ b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/semanticParser/SemanticParser.java @@ -68,7 +68,7 @@ private void dealWithTokenBufferFlags() { * @return the Marker token that was found, or empty if none */ private Optional seekMarker(final boolean seekSeqEnd, final boolean flush) { - final TokenBufferIterator it = reader.iterator(); + final TokenBufferIterator it = reader.tokenIterator(); Optional token; Predicate p = seekSeqEnd ? ReadToken::isSequenceEndMarker : ReadToken::isMarker; @@ -470,7 +470,7 @@ private ActionType flowControl(final byte marker) { */ private boolean seekSecondMarker() { // starts pointing at a marker - we want to know if there's another one after it, or if we're out of tokens! - final TokenBufferIterator it = reader.iterator(); + final TokenBufferIterator it = reader.tokenIterator(); // skip the current (Marker) token, and then it.next(); @@ -507,7 +507,7 @@ private void parseSequenceLevel() { state = State.ERROR_LOCKS; return; } - locks = LockSet.from(first.blockIterator()); + locks = LockSet.from(first.dataIterator()); hasLocks = true; } reader.flushFirstReadToken(); // safe as key is checked, not a Marker diff --git a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/treeParser/ZscriptTreeParser.java b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/treeParser/ZscriptTreeParser.java index b031934f..3f23619e 100644 --- a/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/treeParser/ZscriptTreeParser.java +++ b/receivers/jvm/java-receiver/src/main/java/net/zscript/javareceiver/treeParser/ZscriptTreeParser.java @@ -16,7 +16,7 @@ public static ZscriptSequenceUnitPlace parseAll(ReadToken start) { int lowest = 0; int parenCount = 0; - OptIterator it = start.getNextTokens(); + OptIterator it = start.tokenIterator(); for (Optional opt = it.next(); opt.isPresent() && !opt.get().isSequenceEndMarker(); opt = it.next()) { ReadToken t = opt.get(); if (t.getKey() == Tokenizer.CMD_END_OPEN_PAREN) { @@ -42,7 +42,7 @@ static ZscriptSequenceUnit parse(ReadToken start, byte endMarker, int parenStart boolean hadNonParen = false; - OptIterator it = start.getNextTokens(); + OptIterator it = start.tokenIterator(); for (Optional opt = it.next(); opt.isPresent() && (parenLevel != 0 || opt.get().getKey() != endMarker) && !opt.get().isSequenceEndMarker(); opt = it.next()) { ReadToken t = opt.get(); @@ -71,7 +71,7 @@ static ZscriptSequenceUnit parse(ReadToken start, byte endMarker, int parenStart } ReadToken firstOnLevel = null; - it = start.getNextTokens(); + it = start.tokenIterator(); parenLevel = parenStartCount; for (Optional opt = it.next(); opt.isPresent() && (parenLevel != 0 || opt.get().getKey() != endMarker) && !isSequenceEndMarker(opt.get().getKey()); opt = it.next()) { @@ -100,7 +100,7 @@ static ZscriptSequenceUnit parse(ReadToken start, byte endMarker, int parenStart boolean hasTokens = false; boolean lastWasOpen = false; parenLevel = parenStartCount; - it = start.getNextTokens(); + it = start.tokenIterator(); Optional opt; for (opt = it.next(); opt.isPresent() && (parenLevel != 0 && !isSequenceEndMarker(opt.get().getKey()) || opt.get().getKey() != endMarker); opt = it.next()) { @@ -127,7 +127,7 @@ static ZscriptSequenceUnit parse(ReadToken start, byte endMarker, int parenStart } } if (!hasTokens) { - it = start.getNextTokens(); + it = start.tokenIterator(); for (opt = it.next(); opt.isPresent() && !opt.get().isMarker(); opt = it.next()) { } // System.out.println("Command2: " + (char) start.getKey() + " : " + Integer.toHexString(Byte.toUnsignedInt(endMarker))); diff --git a/receivers/jvm/java-receiver/src/test/java/net/zscript/javareceiver/semanticParser/ParserActionTester.java b/receivers/jvm/java-receiver/src/test/java/net/zscript/javareceiver/semanticParser/ParserActionTester.java index f4bd6f30..7b77f03d 100644 --- a/receivers/jvm/java-receiver/src/test/java/net/zscript/javareceiver/semanticParser/ParserActionTester.java +++ b/receivers/jvm/java-receiver/src/test/java/net/zscript/javareceiver/semanticParser/ParserActionTester.java @@ -33,7 +33,7 @@ void parseSnippet(final String text, final List actionTypes) { // feed all chars into tokens/buffer text.chars().forEachOrdered(c -> tokenizer.accept((byte) c)); - buffer.getTokenReader().iterator().forEach(t -> System.out.print(t + " ")); + buffer.getTokenReader().tokenIterator().forEach(t -> System.out.print(t + " ")); System.out.println(); actionTypes.forEach(t -> { @@ -52,7 +52,7 @@ void parseSnippet(final String text, String output) { // feed all chars into tokens/buffer text.chars().forEachOrdered(c -> tokenizer.accept((byte) c)); - buffer.getTokenReader().iterator().forEach(t -> System.out.print(t + " ")); + buffer.getTokenReader().tokenIterator().forEach(t -> System.out.print(t + " ")); System.out.println(); SemanticAction a1; @@ -68,7 +68,7 @@ void parseSnippet(final String text, ActionType expectedActionType, State endSta // feed all chars into tokens/buffer text.chars().forEachOrdered(c -> tokenizer.accept((byte) c)); - buffer.getTokenReader().iterator().forEach(t -> System.out.print(t + " ")); + buffer.getTokenReader().tokenIterator().forEach(t -> System.out.print(t + " ")); System.out.println(); SemanticAction action = parser.getAction(); diff --git a/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/AbstractArrayTokenBuffer.java b/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/AbstractArrayTokenBuffer.java index f5daa401..6c163f07 100644 --- a/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/AbstractArrayTokenBuffer.java +++ b/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/AbstractArrayTokenBuffer.java @@ -295,7 +295,7 @@ private class ArrayTokenBufferReader implements TokenReader { * Token Iterator which iterates all the available tokens that have been (completely) written to the buffer. It's aware of the extension tokens and excludes them from the * iteration. * - * @see #iterator() + * @see #tokenIterator() */ private class ArrayBufferTokenIterator implements TokenBufferIterator { private int index; @@ -349,7 +349,7 @@ public TokenBufferFlags getFlags() { * {@inheritDoc} */ @Override - public TokenBufferIterator iterator() { + public TokenBufferIterator tokenIterator() { return new ArrayBufferTokenIterator(readStart); } @@ -377,7 +377,7 @@ public ReadToken getFirstReadToken() { */ @Override public void flushFirstReadToken() { - final TokenBufferIterator it = iterator(); + final TokenBufferIterator it = tokenIterator(); it.next(); it.flushBuffer(); } @@ -414,12 +414,12 @@ private ArrayBufferToken(final int index) { * {@inheritDoc} */ @Override - public TokenBufferIterator getNextTokens() { + public TokenBufferIterator tokenIterator() { return new ArrayBufferTokenIterator(index); } @Override - public BlockIterator blockIterator() { + public BlockIterator dataIterator() { if (isMarker()) { throw new IllegalStateException("Cannot get data from marker token"); } diff --git a/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/TokenBuffer.java b/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/TokenBuffer.java index 0f819453..51b18627 100644 --- a/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/TokenBuffer.java +++ b/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/TokenBuffer.java @@ -159,7 +159,7 @@ interface TokenReader { * * @return an iterator over ReadTokens available. */ - TokenBufferIterator iterator(); + TokenBufferIterator tokenIterator(); /** * Checks whether any tokens can be read. @@ -195,12 +195,6 @@ default byte getFirstKey() { * Represents a single Token in the token buffer. */ interface ReadToken { - /** - * Creates an OptIterator over the completed tokens from this token's position forward, such that calling next() will first supply this token. - * - * @return an iterator over available ReadTokens. - */ - TokenBufferIterator getNextTokens(); /** * Determines the key associated with this token. @@ -235,13 +229,20 @@ default boolean isSequenceEndMarker() { return TokenBuffer.isSequenceEndMarker(getKey()); } + /** + * Creates an OptIterator over the completed tokens from this token's position forward, such that calling next() will first supply this token. + * + * @return an iterator over available ReadTokens. + */ + TokenBufferIterator tokenIterator(); + /** * Exposes this token's data as a stream of bytes or byte[] blocks. It manages aggregating across extension segments, and breaks caused by circularity in ring-buffers. * * @return an iterator that exposes the data either as a succession of individual bytes, or as byte[] blocks of contiguous bytes. * @throws IllegalStateException if this token is any kind of Marker - check first! */ - BlockIterator blockIterator(); + BlockIterator dataIterator(); /** * Exposes the data as a single number (uint32). diff --git a/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenExpression.java b/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenExpression.java index fb184d32..24235fe3 100644 --- a/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenExpression.java +++ b/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenExpression.java @@ -72,7 +72,7 @@ public int getBigFieldSize() { @Override public ByteString getBigFieldAsByteString() { - return ByteString.from(getBigField()); + return ByteString.from(bigFieldDataIterator()); } /** @@ -81,7 +81,7 @@ public ByteString getBigFieldAsByteString() { * * @return fully big-field-aware data iterator */ - public BlockIterator getBigField() { + public BlockIterator bigFieldDataIterator() { return new BlockIterator() { final OptIterator it = iteratorToMarker(); @@ -96,7 +96,7 @@ public boolean hasNext() { ReadToken token = opt.get(); if (Zchars.isBigField(token.getKey())) { // note that the token's blockIterator handles extension blocks - internal = token.blockIterator(); + internal = token.dataIterator(); if (internal.hasNext()) { return true; } diff --git a/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenField.java b/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenField.java index 19275bad..cf6be3ce 100644 --- a/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenField.java +++ b/receivers/jvm/java-tokenizer/src/main/java/net/zscript/tokenizer/ZscriptTokenField.java @@ -27,6 +27,6 @@ public boolean isBigField() { @Override public BlockIterator iterator() { - return token.blockIterator(); + return token.dataIterator(); } } diff --git a/receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/TokenRingBufferReaderTest.java b/receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/TokenRingBufferReaderTest.java index 092b55da..4b7713b4 100644 --- a/receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/TokenRingBufferReaderTest.java +++ b/receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/TokenRingBufferReaderTest.java @@ -111,7 +111,7 @@ private void testTokenSimilarity(ReadToken found, TokenExpectation expected) { } } ByteArrayOutputStream st = new ByteArrayOutputStream(); - Streams.stream(found.blockIterator()).forEach(t -> st.write(t)); + Streams.stream(found.dataIterator()).forEach(t -> st.write(t)); assertThat(st.toByteArray()).containsExactly(expected.data); } if (expected.isExtended) { @@ -139,7 +139,7 @@ void shouldIterateNormalTokensCorrectly() { assertThat(reader.hasReadToken()).isTrue(); assertThat(reader.getFirstReadToken().getKey()).isEqualTo(tokens.get(0).key); Iterator expectations = tokens.iterator(); - OptIterator iterator = reader.iterator(); + OptIterator iterator = reader.tokenIterator(); for (Optional opt = iterator.next(); opt.isPresent(); opt = iterator.next()) { ReadToken token = opt.get(); try { @@ -159,7 +159,7 @@ void shouldIterateMarkerTokensCorrectly() { assertThat(reader.hasReadToken()).isTrue(); assertThat(reader.getFirstReadToken().getKey()).isEqualTo(tokens.get(0).key); Iterator expectations = tokens.iterator(); - OptIterator iterator = reader.iterator(); + OptIterator iterator = reader.tokenIterator(); for (Optional opt = iterator.next(); opt.isPresent(); opt = iterator.next()) { ReadToken token = opt.get(); try { @@ -183,7 +183,7 @@ void shouldIterateMixedTokensCorrectly() { assertThat(reader.hasReadToken()).isTrue(); assertThat(reader.getFirstReadToken().getKey()).isEqualTo(tokens.get(0).key); Iterator expectations = tokens.iterator(); - OptIterator iterator = reader.iterator(); + OptIterator iterator = reader.tokenIterator(); for (Optional opt = iterator.next(); opt.isPresent(); opt = iterator.next()) { ReadToken token = opt.get(); try { @@ -204,7 +204,7 @@ void readTokensShouldMatchWrittenTokens() { writeNormalTokens(r, 10, 2); writeExtendedToken(r, 400); writeNormalTokens(r, 10, 2); - testIteratorCorrectness(reader.iterator(), 0); + testIteratorCorrectness(reader.tokenIterator(), 0); } @Test @@ -216,7 +216,7 @@ void flushFirstShouldFlush() { writeExtendedToken(r, 400); writeNormalTokens(r, 10, 2); reader.flushFirstReadToken(); - testIteratorCorrectness(reader.iterator(), 1); + testIteratorCorrectness(reader.tokenIterator(), 1); } @Test @@ -229,10 +229,10 @@ void iteratorsStartedFromTokensShouldMatch() { writeNormalTokens(r, 10, 2); int i = 0; - OptIterator iterator = reader.iterator(); + OptIterator iterator = reader.tokenIterator(); for (Optional opt = iterator.next(); opt.isPresent(); opt = iterator.next()) { ReadToken token = opt.get(); - testIteratorCorrectness(token.getNextTokens(), i++); + testIteratorCorrectness(token.tokenIterator(), i++); } } @@ -240,25 +240,25 @@ void iteratorsStartedFromTokensShouldMatch() { void shouldIterateTokenDataThroughExtensions() { Random r = new Random(SEED); writeExtendedToken(r, 800); - testIteratorCorrectness(reader.iterator(), 0); + testIteratorCorrectness(reader.tokenIterator(), 0); } @Test void shouldIterateTokenDataThroughLoopingBuffer() { Random r = new Random(SEED); writeExtendedToken(r, 800); - TokenBufferIterator it = reader.iterator(); + TokenBufferIterator it = reader.tokenIterator(); it.next(); it.flushBuffer(); writeExtendedToken(r, 800); - testIteratorCorrectness(reader.iterator(), 1); + testIteratorCorrectness(reader.tokenIterator(), 1); } @Test void shouldIterateTokenDataInContiguousChunks() { Random r = new Random(SEED); writeExtendedToken(r, 800); - BlockIterator data = reader.iterator().next().get().blockIterator(); + BlockIterator data = reader.tokenIterator().next().get().dataIterator(); int i = 0; for (BlockIterator iterator = data; iterator.hasNext(); ) { byte[] nextCont = iterator.nextContiguous(); @@ -271,7 +271,7 @@ void shouldIterateTokenDataInContiguousChunks() { void shouldIterateTokenDataInLimtedContiguousChunks() { Random r = new Random(SEED); writeExtendedToken(r, 800); - BlockIterator data = reader.iterator().next().get().blockIterator(); + BlockIterator data = reader.tokenIterator().next().get().dataIterator(); int i = 0; for (BlockIterator iterator = data; iterator.hasNext(); ) { assertThat(iterator.nextContiguous(200)).containsExactly(Arrays.copyOfRange(tokens.get(0).data, i, Math.min(i + 200, 800))); @@ -287,11 +287,11 @@ void shouldIterateTokenDataInLimtedContiguousChunksAroundEnd() { int initialLength = 700; Random r = new Random(SEED); writeExtendedToken(r, initialLength); - TokenBufferIterator it = reader.iterator(); + TokenBufferIterator it = reader.tokenIterator(); it.next(); it.flushBuffer(); writeExtendedToken(r, 800); - BlockIterator data = reader.iterator().next().get().blockIterator(); + BlockIterator data = reader.tokenIterator().next().get().dataIterator(); int i = 0; for (BlockIterator iterator = data; iterator.hasNext(); ) { int offset = 200; diff --git a/receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/ZscriptTokenExpressionTest.java b/receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/ZscriptTokenExpressionTest.java index 8d0eae68..d3761a25 100644 --- a/receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/ZscriptTokenExpressionTest.java +++ b/receivers/jvm/java-tokenizer/src/test/java/net/zscript/tokenizer/ZscriptTokenExpressionTest.java @@ -13,7 +13,7 @@ class ZscriptTokenExpressionTest { Tokenizer tokenizer = new Tokenizer(buffer.getTokenWriter(), 2); TokenBuffer.TokenReader tokenReader = buffer.getTokenReader(); - ZscriptTokenExpression zscriptExpr = new ZscriptTokenExpression(() -> tokenReader.iterator()); + ZscriptTokenExpression zscriptExpr = new ZscriptTokenExpression(() -> tokenReader.tokenIterator()); @Test public void shouldGetFields() { @@ -53,7 +53,7 @@ public void shouldGetMultipleBigFields() { assertThat(zscriptExpr.getBigFieldSize()).isEqualTo(11); assertThat(zscriptExpr.getBigFieldData()).containsExactly("Hello World".getBytes(ISO_8859_1)); - BlockIterator iter = zscriptExpr.getBigField(); + BlockIterator iter = zscriptExpr.bigFieldDataIterator(); assertThat(iter.next()).isEqualTo((byte) 'H'); assertThat(iter.nextContiguous(7)).containsExactly("ello".getBytes(ISO_8859_1)); assertThat(iter.nextContiguous()).containsExactly(" ".getBytes(ISO_8859_1));