From 45a163cb73511612af116f01c5b8bbddcf012811 Mon Sep 17 00:00:00 2001 From: kannar Date: Mon, 8 Apr 2024 15:41:21 +0200 Subject: [PATCH 01/11] match sample.json model --- .../biscuitsec/biscuit/token/SamplesTest.java | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java b/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java index 02549fe1..a670c875 100644 --- a/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java +++ b/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java @@ -165,18 +165,6 @@ public void setSymbols(List symbols) { } } - class Token { - List blocks; - - public List getBlocks() { - return blocks; - } - - public void setBlocks(List blocks) { - this.blocks = blocks; - } - } - class TestCase { String title; @@ -189,7 +177,7 @@ public void setTitle(String title) { } String filename; - List tokens; + List token; JsonElement validations; public String getFilename() { @@ -200,12 +188,12 @@ public void setFilename(String filename) { this.filename = filename; } - public List getTokens() { - return tokens; + public List getToken() { + return token; } - public void setTokens(List tokens) { - this.tokens = tokens; + public void setTokens(List token) { + this.token = token; } public JsonElement getValidations() { From 2b7aa44cfbbf965f5253eae5e78ec57a588e0d4b Mon Sep 17 00:00:00 2001 From: kannar Date: Mon, 8 Apr 2024 17:25:12 +0200 Subject: [PATCH 02/11] add datalog parser for block --- .../biscuit/token/builder/parser/Parser.java | 44 +++++++++++++++++-- .../biscuit/builder/parser/ParserTest.java | 39 +++++++++++++++- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java index dbd1c2c8..25a42464 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java +++ b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java @@ -1,7 +1,9 @@ package org.biscuitsec.biscuit.token.builder.parser; import biscuit.format.schema.Schema; +import io.vavr.collection.Stream; import org.biscuitsec.biscuit.crypto.PublicKey; +import org.biscuitsec.biscuit.datalog.SymbolTable; import org.biscuitsec.biscuit.token.Policy; import io.vavr.Tuple2; import io.vavr.Tuple4; @@ -10,12 +12,48 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.List; -import java.util.HashSet; +import java.util.*; import java.util.function.Function; public class Parser { + /** + * Takes a datalog string with \n as datalog line separator. It tries to parse + * each line using fact, rule, check and scope sequentially. + * + * If one succeeds it returns Right(Block) + * else it returns a Map[lineNumber, List[Error]] + * + * @param index block index + * @param baseSymbols symbols table + * @param s datalog string to parse + * @return Either>, Block> + */ + public static Either>, Block> datalog(long index, SymbolTable baseSymbols, String s) { + Block blockBuilder = new Block(index, baseSymbols); + Map> errors = new HashMap<>(); + + Stream.of(s.split("\n")).zipWithIndex().forEach(indexedLine -> { + Integer lineNumber = indexedLine._2; + String codeLine = indexedLine._1; + List lineErrors = new ArrayList<>(); + + fact(codeLine).bimap(lineErrors::add, r -> r._2).map(blockBuilder::add_fact); + rule(codeLine).bimap(lineErrors::add, r -> r._2).map(blockBuilder::add_rule); + check(codeLine).bimap(lineErrors::add, r -> r._2).map(blockBuilder::add_check); + scope(codeLine).bimap(lineErrors::add, r -> r._2).map(blockBuilder::add_scope); + + if (lineErrors.size() > 3) { + errors.put(lineNumber, lineErrors); + } + }); + + if (!errors.isEmpty()) { + return Either.left(errors); + } + + return Either.right(blockBuilder); + } + public static Either> fact(String s) { Either> res = fact_predicate(s); if (res.isLeft()) { diff --git a/src/test/java/org/biscuitsec/biscuit/builder/parser/ParserTest.java b/src/test/java/org/biscuitsec/biscuit/builder/parser/ParserTest.java index 3c76890b..fff15f76 100644 --- a/src/test/java/org/biscuitsec/biscuit/builder/parser/ParserTest.java +++ b/src/test/java/org/biscuitsec/biscuit/builder/parser/ParserTest.java @@ -5,6 +5,7 @@ import org.biscuitsec.biscuit.datalog.SymbolTable; import org.biscuitsec.biscuit.datalog.TemporarySymbolTable; import org.biscuitsec.biscuit.datalog.expressions.Op; +import org.biscuitsec.biscuit.token.Biscuit; import org.biscuitsec.biscuit.token.builder.parser.Error; import org.biscuitsec.biscuit.token.builder.parser.Parser; import io.vavr.Tuple2; @@ -15,7 +16,7 @@ import org.junit.jupiter.api.Test; import static org.biscuitsec.biscuit.datalog.Check.Kind.One; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; import java.util.*; @@ -360,4 +361,40 @@ void testParens() throws org.biscuitsec.biscuit.error.Error.Execution { assertEquals(new org.biscuitsec.biscuit.datalog.Term.Integer(9), value2); assertEquals("(1 + 2) * 3", ex2.print(s2).get()); } + + @Test + void testDatalogSucceeds() throws org.biscuitsec.biscuit.error.Error.Parser { + SymbolTable symbols = Biscuit.default_symbol_table(); + + String l1 = "fact1(1)"; + String l2 = "fact2(\"2\")"; + String l3 = "rule1(2) <- fact2(\"2\")"; + String l4 = "check if rule1(2)"; + String toParse = String.join("\n", Arrays.asList(l1, l2, l3, l4)); + + Either>, Block> output = Parser.datalog(1, symbols, toParse); + assertTrue(output.isRight()); + + Block validBlock = new Block(1, symbols); + validBlock.add_fact(l1); + validBlock.add_fact(l2); + validBlock.add_rule(l3); + validBlock.add_check(l4); + + output.forEach(block -> + assertArrayEquals(block.build().to_bytes().get(), validBlock.build().to_bytes().get()) + ); + } + + @Test + void testDatalogFailed() { + SymbolTable symbols = Biscuit.default_symbol_table(); + + String l1 = "fact(1)"; + String l2 = "check fact(1)"; // typo missing "if" + String toParse = String.join("\n", Arrays.asList(l1, l2)); + + Either>, Block> output = Parser.datalog(1, symbols, toParse); + assertTrue(output.isLeft()); + } } \ No newline at end of file From 0eae499bf0d05f9e9c5491ee481e74bacef8735b Mon Sep 17 00:00:00 2001 From: kannar Date: Mon, 8 Apr 2024 18:06:15 +0200 Subject: [PATCH 03/11] add symbol table constructor from symbols --- .../java/org/biscuitsec/biscuit/datalog/SymbolTable.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java b/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java index 489dae2f..7cc969c2 100644 --- a/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java +++ b/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java @@ -262,6 +262,12 @@ public SymbolTable(SymbolTable s) { publicKeys.addAll(s.publicKeys); } + public SymbolTable(List symbols) { + this.symbols = new ArrayList<>(); + this.symbols.addAll(symbols); + this.publicKeys = new ArrayList<>(); + } + public SymbolTable(List symbols, List publicKeys) { this.symbols = new ArrayList<>(); this.symbols.addAll(symbols); From 80cecfa52afd87f1d1896892b14a6c11fb7d8169 Mon Sep 17 00:00:00 2001 From: kannar Date: Tue, 9 Apr 2024 00:15:24 +0200 Subject: [PATCH 04/11] add pk --- .../biscuit/token/builder/Block.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java b/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java index 302e6930..ca89e57e 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java +++ b/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java @@ -32,6 +32,8 @@ public class Block { List rules; List checks; List scopes; + List publicKeys; + Option externalKey; public Block(long index, SymbolTable base_symbols) { this.index = index; @@ -43,6 +45,28 @@ public Block(long index, SymbolTable base_symbols) { this.rules = new ArrayList<>(); this.checks = new ArrayList<>(); this.scopes = new ArrayList<>(); + this.publicKeys = new ArrayList<>(); + this.externalKey = Option.none(); + } + + public Block setExternalKey(Option externalKey) { + this.externalKey = externalKey; + return this; + } + + public Block addPublicKey(PublicKey publicKey) { + this.publicKeys.add(publicKey); + return this; + } + + public Block addPublicKeys(List publicKeys) { + this.publicKeys.addAll(publicKeys); + return this; + } + + public Block setPublicKeys(List publicKeys) { + this.publicKeys = publicKeys; + return this; } public Block add_fact(org.biscuitsec.biscuit.token.builder.Fact f) { @@ -121,10 +145,12 @@ public org.biscuitsec.biscuit.token.Block build() { publicKeys.add(this.symbols.publicKeys().get(i)); } + publicKeys.addAll(this.publicKeys); + SchemaVersion schemaVersion = new SchemaVersion(this.facts, this.rules, this.checks, this.scopes); return new org.biscuitsec.biscuit.token.Block(symbols, this.context, this.facts, this.rules, this.checks, - this.scopes, publicKeys, Option.none(), schemaVersion.version()); + this.scopes, publicKeys, this.externalKey, schemaVersion.version()); } public Block check_right(String right) { From 76fa45eaea70971dd561173db7aeea655c09d6dd Mon Sep 17 00:00:00 2001 From: kannar Date: Mon, 8 Apr 2024 20:39:20 +0200 Subject: [PATCH 05/11] block enable to add symbol --- .../java/org/biscuitsec/biscuit/token/builder/Block.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java b/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java index ca89e57e..6a17bbdf 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java +++ b/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java @@ -69,6 +69,11 @@ public Block setPublicKeys(List publicKeys) { return this; } + public Block addSymbol(String symbol) { + this.symbols.add(symbol); + return this; + } + public Block add_fact(org.biscuitsec.biscuit.token.builder.Fact f) { this.facts.add(f.convert(this.symbols)); return this; From 903ebea457e77f4c010b2cb3a33055e7b6b6eaf6 Mon Sep 17 00:00:00 2001 From: kannar Date: Tue, 9 Apr 2024 19:58:51 +0200 Subject: [PATCH 06/11] use long --- .../org/biscuitsec/biscuit/token/builder/parser/Parser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java index 25a42464..f8d21676 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java +++ b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java @@ -562,10 +562,10 @@ public static Either> integer(String s) { return Either.left(new Error(s, "not an integer")); } - Integer i = Integer.parseInt(s.substring(0, index2)); + long i = Long.parseLong(s.substring(0, index2)); String remaining = s.substring(index2); - return Either.right(new Tuple2(remaining, (Term.Integer) Utils.integer(i.intValue()))); + return Either.right(new Tuple2(remaining, (Term.Integer) Utils.integer(i))); } public static Either> date(String s) { From 4ceee3a7faee502deff396d50126d4cfc586587d Mon Sep 17 00:00:00 2001 From: kannar Date: Mon, 8 Apr 2024 18:06:41 +0200 Subject: [PATCH 07/11] wip: add block comparison between sample definition and its serialization --- .../org/biscuitsec/biscuit/token/Block.java | 7 +- .../biscuit/token/builder/Block.java | 2 - .../builder/parser/ExpressionParser.java | 82 ++++++----- .../biscuit/token/builder/parser/Parser.java | 127 ++++++++++++++++-- .../biscuit/builder/parser/ParserTest.java | 98 +++++++++++++- .../biscuit/datalog/ExpressionTest.java | 24 ++++ .../biscuitsec/biscuit/token/SamplesTest.java | 97 +++++++++++-- 7 files changed, 376 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/biscuitsec/biscuit/token/Block.java b/src/main/java/org/biscuitsec/biscuit/token/Block.java index 382224a8..2309fad7 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/Block.java +++ b/src/main/java/org/biscuitsec/biscuit/token/Block.java @@ -90,11 +90,16 @@ public String print(SymbolTable symbol_table) { s.append(this.symbols.symbols); s.append("\n\t\tcontext: "); s.append(this.context); + s.append("\n\t\tscopes: ["); + for (Scope scope : this.scopes) { + s.append("\n\t\t\t"); + s.append(symbol_table.print_scope(scope)); + } if(this.externalKey.isDefined()) { s.append("\n\t\texternal key: "); s.append(this.externalKey.get().toString()); } - s.append("\n\t\tfacts: ["); + s.append("\n\t\t]\n\t\tfacts: ["); for (Fact f : this.facts) { s.append("\n\t\t\t"); s.append(symbol_table.print_fact(f)); diff --git a/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java b/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java index 6a17bbdf..c4591e03 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java +++ b/src/main/java/org/biscuitsec/biscuit/token/builder/Block.java @@ -150,8 +150,6 @@ public org.biscuitsec.biscuit.token.Block build() { publicKeys.add(this.symbols.publicKeys().get(i)); } - publicKeys.addAll(this.publicKeys); - SchemaVersion schemaVersion = new SchemaVersion(this.facts, this.rules, this.checks, this.scopes); return new org.biscuitsec.biscuit.token.Block(symbols, this.context, this.facts, this.rules, this.checks, diff --git a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/ExpressionParser.java b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/ExpressionParser.java index 9c3c00a2..6887dc80 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/ExpressionParser.java +++ b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/ExpressionParser.java @@ -306,43 +306,57 @@ public static Either> expr7(String s) { return Either.left(res1.getLeft()); } Tuple2 t1 = res1.get(); - s = space(t1._1); - Expression e1 = t1._2; - if(!s.startsWith(".")) { - return Either.right(new Tuple2<>(s, e1)); - } - s = s.substring(1); - - Either> res2 = binary_op7(s); - if (res2.isLeft()) { - return Either.left(res2.getLeft()); - } - Tuple2 t2 = res2.get(); - s = space(t2._1); - Expression.Op op = t2._2; - - if(!s.startsWith("(")) { - return Either.left(new Error(s, "missing (")); - } - - s = space(s.substring(1)); + s = t1._1; + Expression e = t1._2; - Either> res3 = expr(s); - if (res3.isLeft()) { - return Either.left(res3.getLeft()); - } + while(true) { + s = space(s); + if(s.isEmpty()) { + break; + } - Tuple2 t3 = res3.get(); + if (!s.startsWith(".")) { + return Either.right(new Tuple2<>(s, e)); + } - s = space(t3._1); - if(!s.startsWith(")")) { - return Either.left(new Error(s, "missing )")); + s = s.substring(1); + Either> res2 = binary_op7(s); + if (!res2.isLeft()) { + Tuple2 t2 = res2.get(); + s = space(t2._1); + Expression.Op op = t2._2; + + if (!s.startsWith("(")) { + return Either.left(new Error(s, "missing (")); + } + + s = space(s.substring(1)); + + Either> res3 = expr7(s); + if (res3.isLeft()) { + return Either.left(res3.getLeft()); + } + + Tuple2 t3 = res3.get(); + + s = space(t3._1); + if (!s.startsWith(")")) { + return Either.left(new Error(s, "missing )")); + } + s = space(s.substring(1)); + Expression e2 = t3._2; + + e = new Expression.Binary(op, e, e2); + } else { + if (s.startsWith("length()")) { + e = new Expression.Unary(Expression.Op.Length, e); + s = s.substring(9); + } + } } - s = space(s.substring(1)); - Expression e2 = t3._2; - return Either.right(new Tuple2<>(s, new Expression.Binary(op, e1, e2))); + return Either.right(new Tuple2<>(s, e)); } public static Either> expr_term(String s) { @@ -517,6 +531,12 @@ public static Either> binary_op6(String s) } public static Either> binary_op7(String s) { + if(s.startsWith("intersection")) { + return Either.right(new Tuple2<>(s.substring(12), Expression.Op.Intersection)); + } + if(s.startsWith("union")) { + return Either.right(new Tuple2<>(s.substring(5), Expression.Op.Union)); + } if(s.startsWith("contains")) { return Either.right(new Tuple2<>(s.substring(8), Expression.Op.Contains)); } diff --git a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java index f8d21676..54eee4d5 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java +++ b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java @@ -16,6 +16,10 @@ import java.util.function.Function; public class Parser { + public static Either>, Block> datalog(long index, SymbolTable baseSymbols, String s) { + return datalog(index, baseSymbols, null, s); + } + /** * Takes a datalog string with \n as datalog line separator. It tries to parse * each line using fact, rule, check and scope sequentially. @@ -25,26 +29,82 @@ public class Parser { * * @param index block index * @param baseSymbols symbols table + * @param blockSymbols block's custom symbols table (added to baseSymbols) * @param s datalog string to parse * @return Either>, Block> */ - public static Either>, Block> datalog(long index, SymbolTable baseSymbols, String s) { + public static Either>, Block> datalog(long index, SymbolTable baseSymbols, SymbolTable blockSymbols, String s) { Block blockBuilder = new Block(index, baseSymbols); - Map> errors = new HashMap<>(); - Stream.of(s.split("\n")).zipWithIndex().forEach(indexedLine -> { - Integer lineNumber = indexedLine._2; - String codeLine = indexedLine._1; - List lineErrors = new ArrayList<>(); + // empty block code + if (s.isEmpty()) { + return Either.right(blockBuilder); + } - fact(codeLine).bimap(lineErrors::add, r -> r._2).map(blockBuilder::add_fact); - rule(codeLine).bimap(lineErrors::add, r -> r._2).map(blockBuilder::add_rule); - check(codeLine).bimap(lineErrors::add, r -> r._2).map(blockBuilder::add_check); - scope(codeLine).bimap(lineErrors::add, r -> r._2).map(blockBuilder::add_scope); + if (blockSymbols != null) { + blockSymbols.symbols.forEach(blockBuilder::addSymbol); + } - if (lineErrors.size() > 3) { - errors.put(lineNumber, lineErrors); - } + Map> errors = new HashMap<>(); + + s = removeCommentsAndWhitespaces(s); + String[] codeLines = s.split(";"); + + Stream.of(codeLines) + .zipWithIndex() + .forEach(indexedLine -> { + String code = indexedLine._1.strip(); + + if (!code.isEmpty()) { + int lineNumber = indexedLine._2; + System.out.println("NEW CODE LINE"); + System.out.println(code); + List lineErrors = new ArrayList<>(); + + boolean parsed = false; + parsed = rule(code).fold(e -> { + lineErrors.add(e); + return false; + }, r -> { + blockBuilder.add_rule(r._2); + return true; + }); + + if (!parsed) { + parsed = scope(code).fold(e -> { + lineErrors.add(e); + return false; + }, r -> { + blockBuilder.add_scope(r._2); + return true; + }); + } + + if (!parsed) { + parsed = fact(code).fold(e -> { + lineErrors.add(e); + return false; + }, r -> { + blockBuilder.add_fact(r._2); + return true; + }); + } + + if (!parsed) { + parsed = check(code).fold(e -> { + lineErrors.add(e); + return false; + }, r -> { + blockBuilder.add_check(r._2); + return true; + }); + } + + if (!parsed) { + lineErrors.forEach(System.out::println); + errors.put(lineNumber, lineErrors); + } + } }); if (!errors.isEmpty()) { @@ -569,7 +629,7 @@ public static Either> integer(String s) { } public static Either> date(String s) { - Tuple2 t = take_while(s, (c) -> c != ' ' && c != ',' && c != ')'); + Tuple2 t = take_while(s, (c) -> c != ' ' && c != ',' && c != ')' && c != ']'); try { OffsetDateTime d = OffsetDateTime.parse(t._1); @@ -709,4 +769,43 @@ public static Tuple2 take_while(String s, Function(s.substring(0, index), s.substring(index)); } + + public static String removeCommentsAndWhitespaces(String s) { + s = removeComments(s); + s = s.replace("\n", "").replace("\\\"", "\"").strip(); + return s; + } + + public static String removeComments(String str) { + StringBuilder result = new StringBuilder(); + String remaining = str; + + while (!remaining.isEmpty()) { + remaining = space(remaining); // Skip leading whitespace + if (remaining.startsWith("/*")) { + // Find the end of the multiline comment + remaining = remaining.substring(2); // Skip "/*" + String finalRemaining = remaining; + Tuple2 split = take_while(remaining, c -> !finalRemaining.startsWith("*/")); + remaining = split._2.length() > 2 ? split._2.substring(2) : ""; // Skip "*/" + } else if (remaining.startsWith("//")) { + // Find the end of the single-line comment + remaining = remaining.substring(2); // Skip "//" + Tuple2 split = take_while(remaining, c -> c != '\n' && c != '\r'); + remaining = split._2; + if (!remaining.isEmpty()) { + result.append(remaining.charAt(0)); // Preserve line break + remaining = remaining.substring(1); + } + } else { + // Take non-comment text until the next comment or end of string + String finalRemaining = remaining; + Tuple2 split = take_while(remaining, c -> !finalRemaining.startsWith("/*") && !finalRemaining.startsWith("//")); + result.append(split._1); + remaining = split._2; + } + } + + return result.toString(); + } } diff --git a/src/test/java/org/biscuitsec/biscuit/builder/parser/ParserTest.java b/src/test/java/org/biscuitsec/biscuit/builder/parser/ParserTest.java index fff15f76..9ec36a1f 100644 --- a/src/test/java/org/biscuitsec/biscuit/builder/parser/ParserTest.java +++ b/src/test/java/org/biscuitsec/biscuit/builder/parser/ParserTest.java @@ -133,6 +133,43 @@ void testRuleWithExpressionOrdering() { res); } + @Test + void expressionIntersectionAndContainsTest() { + Either> res = + Parser.expression("[1, 2, 3].intersection([1, 2]).contains(1)"); + + assertEquals(Either.right(new Tuple2<>("", + new Expression.Binary( + Expression.Op.Contains, + new Expression.Binary( + Expression.Op.Intersection, + new Expression.Value(Utils.set(new HashSet<>(Arrays.asList(Utils.integer(1), Utils.integer(2), Utils.integer(3))))), + new Expression.Value(Utils.set(new HashSet<>(Arrays.asList(Utils.integer(1), Utils.integer(2))))) + ), + new Expression.Value(Utils.integer(1)) + ))), res); + } + + @Test + void expressionIntersectionAndContainsAndLengthEqualsTest() { + Either> res = + Parser.expression("[1, 2, 3].intersection([1, 2]).length() == 2"); + + assertEquals(Either.right(new Tuple2<>("", + new Expression.Binary( + Expression.Op.Equal, + new Expression.Unary( + Expression.Op.Length, + new Expression.Binary( + Expression.Op.Intersection, + new Expression.Value(Utils.set(new HashSet<>(Arrays.asList(Utils.integer(1), Utils.integer(2), Utils.integer(3))))), + new Expression.Value(Utils.set(new HashSet<>(Arrays.asList(Utils.integer(1), Utils.integer(2))))) + ) + ), + new Expression.Value(Utils.integer(2)) + ))), res); + } + @Test void ruleWithFreeExpressionVariables() { Either> res = @@ -366,11 +403,11 @@ void testParens() throws org.biscuitsec.biscuit.error.Error.Execution { void testDatalogSucceeds() throws org.biscuitsec.biscuit.error.Error.Parser { SymbolTable symbols = Biscuit.default_symbol_table(); - String l1 = "fact1(1)"; + String l1 = "fact1(1, 2)"; String l2 = "fact2(\"2\")"; String l3 = "rule1(2) <- fact2(\"2\")"; String l4 = "check if rule1(2)"; - String toParse = String.join("\n", Arrays.asList(l1, l2, l3, l4)); + String toParse = String.join(";", Arrays.asList(l1, l2, l3, l4)); Either>, Block> output = Parser.datalog(1, symbols, toParse); assertTrue(output.isRight()); @@ -386,15 +423,70 @@ void testDatalogSucceeds() throws org.biscuitsec.biscuit.error.Error.Parser { ); } + @Test + void testDatalogSucceedsArrays() throws org.biscuitsec.biscuit.error.Error.Parser { + SymbolTable symbols = Biscuit.default_symbol_table(); + + String l1 = "check if [2, 3].union([2])"; + String toParse = String.join(";", List.of(l1)); + + Either>, Block> output = Parser.datalog(1, symbols, toParse); + assertTrue(output.isRight()); + + Block validBlock = new Block(1, symbols); + validBlock.add_check(l1); + + output.forEach(block -> + assertArrayEquals(block.build().to_bytes().get(), validBlock.build().to_bytes().get()) + ); + } + + @Test + void testDatalogSucceedsArraysContains() throws org.biscuitsec.biscuit.error.Error.Parser { + SymbolTable symbols = Biscuit.default_symbol_table(); + + String l1 = "check if [2019-12-04T09:46:41Z, 2020-12-04T09:46:41Z].contains(2020-12-04T09:46:41Z)"; + String toParse = String.join(";", List.of(l1)); + + Either>, Block> output = Parser.datalog(1, symbols, toParse); + assertTrue(output.isRight()); + + Block validBlock = new Block(1, symbols); + validBlock.add_check(l1); + + output.forEach(block -> + assertArrayEquals(block.build().to_bytes().get(), validBlock.build().to_bytes().get()) + ); + } + @Test void testDatalogFailed() { SymbolTable symbols = Biscuit.default_symbol_table(); String l1 = "fact(1)"; String l2 = "check fact(1)"; // typo missing "if" - String toParse = String.join("\n", Arrays.asList(l1, l2)); + String toParse = String.join(";", Arrays.asList(l1, l2)); Either>, Block> output = Parser.datalog(1, symbols, toParse); assertTrue(output.isLeft()); } + + @Test + void testDatalogRemoveComment() throws org.biscuitsec.biscuit.error.Error.Parser { + SymbolTable symbols = Biscuit.default_symbol_table(); + + String l0 = "// test comment"; + String l1 = "fact1(1, 2);"; + String l2 = "fact2(\"2\");"; + String l3 = "rule1(2) <- fact2(\"2\");"; + String l4 = "// another comment"; + String l5 = "/* test multiline"; + String l6 = "comment */ check if rule1(2);"; + String l7 = " /* another multiline"; + String l8 = "comment */"; + String toParse = String.join("", Arrays.asList(l0, l1, l2, l3, l4, l5, l6, l7, l8)); + + Either>, Block> output = Parser.datalog(1, symbols, toParse); + assertTrue(output.isRight()); + } } \ No newline at end of file diff --git a/src/test/java/org/biscuitsec/biscuit/datalog/ExpressionTest.java b/src/test/java/org/biscuitsec/biscuit/datalog/ExpressionTest.java index c11fbebb..5e38166b 100644 --- a/src/test/java/org/biscuitsec/biscuit/datalog/ExpressionTest.java +++ b/src/test/java/org/biscuitsec/biscuit/datalog/ExpressionTest.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; public class ExpressionTest { @@ -114,4 +115,27 @@ public void testNegativeContainsStr() throws Error.Execution { e.evaluate(new HashMap<>(), new TemporarySymbolTable(symbols)) ); } + + @Test + public void testIntersectionAndContains() throws Error.Execution { + SymbolTable symbols = new SymbolTable(); + + Expression e = new Expression(new ArrayList(Arrays.asList( + new Op.Value(new Term.Set(new HashSet<>(Arrays.asList(new Term.Integer(1), new Term.Integer(2), new Term.Integer(3))))), + new Op.Value(new Term.Set(new HashSet<>(Arrays.asList(new Term.Integer(1), new Term.Integer(2))))), + new Op.Binary(Op.BinaryOp.Intersection), + new Op.Value(new Term.Integer(1)), + new Op.Binary(Op.BinaryOp.Contains) + ))); + + assertEquals( + "[1, 2, 3].intersection([1, 2]).contains(1)", + e.print(symbols).get() + ); + + assertEquals( + new Term.Bool(true), + e.evaluate(new HashMap<>(), new TemporarySymbolTable(symbols)) + ); + } } diff --git a/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java b/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java index a670c875..ff097c0a 100644 --- a/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java +++ b/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java @@ -4,15 +4,20 @@ import com.google.gson.*; import com.google.protobuf.MapEntry; import io.vavr.Tuple2; +import io.vavr.control.Option; import org.biscuitsec.biscuit.crypto.KeyPair; import org.biscuitsec.biscuit.crypto.PublicKey; import org.biscuitsec.biscuit.datalog.Rule; import org.biscuitsec.biscuit.datalog.RunLimits; +import org.biscuitsec.biscuit.datalog.SymbolTable; import org.biscuitsec.biscuit.datalog.TrustedOrigins; import org.biscuitsec.biscuit.error.Error; import io.vavr.control.Either; import io.vavr.control.Try; import org.biscuitsec.biscuit.token.builder.Check; +import org.biscuitsec.biscuit.token.builder.Expression; +import org.biscuitsec.biscuit.token.builder.parser.ExpressionParser; +import org.biscuitsec.biscuit.token.builder.parser.Parser; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; @@ -22,6 +27,7 @@ import java.time.Duration; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; @@ -39,6 +45,47 @@ Stream jsonTest() { return sample.testcases.stream().map(t -> process_testcase(t, publicKey, keyPair)); } + void compareBlocks(List sampleBlocks, List blocks) { + assertEquals(sampleBlocks.size(), blocks.size()); + List> comparisonList = IntStream.range(0, sampleBlocks.size()) + .mapToObj(i -> new Tuple2<>(sampleBlocks.get(i), blocks.get(i))) + .collect(Collectors.toList()); + + // for each token we start from the base symbol table + SymbolTable baseSymbols = new SymbolTable(); + + io.vavr.collection.Stream.ofAll(comparisonList).zipWithIndex().forEach(indexedItem -> { + compareBlock(baseSymbols, indexedItem._2, indexedItem._1._1, indexedItem._1._2); + }); + } + + void compareBlock(SymbolTable baseSymbols, long sampleBlockIndex, Block sampleBlock, org.biscuitsec.biscuit.token.Block block) { + Option sampleExternalKey = sampleBlock.getExternalKey(); + List samplePublicKeys = sampleBlock.getPublicKeys(); + String sampleDatalog = sampleBlock.getCode().replace("\"","\\\""); + SymbolTable sampleSymbols = new SymbolTable(sampleBlock.symbols); + + Either>, org.biscuitsec.biscuit.token.builder.Block> outputSample = Parser.datalog(sampleBlockIndex, baseSymbols, sampleDatalog); + assertTrue(outputSample.isRight()); + + if (!block.publicKeys.isEmpty()) { + outputSample.get().addPublicKeys(samplePublicKeys); + } + + if (!block.externalKey.isDefined()) { + sampleSymbols.symbols.forEach(baseSymbols::add); + } else { + SymbolTable freshSymbols = new SymbolTable(); + sampleSymbols.symbols.forEach(freshSymbols::add); + outputSample.get().setExternalKey(sampleExternalKey); + } + + System.out.println(outputSample.get().build().print(sampleSymbols)); + System.out.println(block.symbols.symbols); + System.out.println(block.print(sampleSymbols)); + assertArrayEquals(outputSample.get().build().to_bytes().get(), block.to_bytes().get()); + } + DynamicTest process_testcase(final TestCase testCase, final PublicKey publicKey, final KeyPair privateKey) { return DynamicTest.dynamicTest(testCase.title + ": "+testCase.filename, () -> { System.out.println("Testcase name: \""+testCase.title+"\""); @@ -57,6 +104,12 @@ DynamicTest process_testcase(final TestCase testCase, final PublicKey publicKey, Biscuit token = Biscuit.from_bytes(data, publicKey); assertArrayEquals(token.serialize(), data); + List allBlocks = new ArrayList<>(); + allBlocks.add(token.authority); + allBlocks.addAll(token.blocks); + + compareBlocks(testCase.token, allBlocks); + List revocationIds = token.revocation_identifiers(); JsonArray validationRevocationIds = validation.getAsJsonArray("revocation_ids"); assertEquals(revocationIds.size(), validationRevocationIds.size()); @@ -142,19 +195,11 @@ DynamicTest process_testcase(final TestCase testCase, final PublicKey publicKey, }); } - class Block { List symbols; - - public String getCode() { - return code; - } - - public void setCode(String code) { - this.code = code; - } - String code; + List public_keys; + String external_key; public List getSymbols() { return symbols; @@ -163,6 +208,38 @@ public List getSymbols() { public void setSymbols(List symbols) { this.symbols = symbols; } + + public String getCode() { return code; } + + public void setCode(String code) { this.code = code; } + + public List getPublicKeys() { + return this.public_keys.stream() + .map(pk -> + Parser.publicKey(pk).fold(e -> { throw new IllegalArgumentException(e.toString());}, r -> r._2) + ) + .collect(Collectors.toList()); + } + + public void setPublicKeys(List publicKeys) { + this.public_keys = publicKeys.stream() + .map(PublicKey::toString) + .collect(Collectors.toList()); + } + + public Option getExternalKey() { + if (this.external_key != null) { + PublicKey externalKey = Parser.publicKey(this.external_key) + .fold(e -> { throw new IllegalArgumentException(e.toString());}, r -> r._2); + return Option.of(externalKey); + } else { + return Option.none(); + } + } + + public void setExternalKey(Option externalKey) { + this.external_key = externalKey.map(PublicKey::toString).getOrElse((String) null); + } } class TestCase { From ee11ee67204591f433af9dff2081436194937c96 Mon Sep 17 00:00:00 2001 From: Geoffroy Couprie Date: Wed, 1 May 2024 13:54:50 +0200 Subject: [PATCH 08/11] Update src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aurélien Mino <155828+murdos@users.noreply.github.com> --- src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java b/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java index 7cc969c2..9034d783 100644 --- a/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java +++ b/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java @@ -263,7 +263,7 @@ public SymbolTable(SymbolTable s) { } public SymbolTable(List symbols) { - this.symbols = new ArrayList<>(); + this.symbols = new ArrayList<>(symbols); this.symbols.addAll(symbols); this.publicKeys = new ArrayList<>(); } From 2cf6c0133bb373ee91da63df8c57f00a50500913 Mon Sep 17 00:00:00 2001 From: Geoffroy Couprie Date: Wed, 1 May 2024 14:24:33 +0200 Subject: [PATCH 09/11] Update src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aurélien Mino <155828+murdos@users.noreply.github.com> --- src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java b/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java index 9034d783..0216b29d 100644 --- a/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java +++ b/src/main/java/org/biscuitsec/biscuit/datalog/SymbolTable.java @@ -264,7 +264,6 @@ public SymbolTable(SymbolTable s) { public SymbolTable(List symbols) { this.symbols = new ArrayList<>(symbols); - this.symbols.addAll(symbols); this.publicKeys = new ArrayList<>(); } From 451ede6c5c1d0e1b6f09480e148e1dcb5f932d09 Mon Sep 17 00:00:00 2001 From: Geoffroy Couprie Date: Sat, 15 Jun 2024 16:25:56 +0200 Subject: [PATCH 10/11] compare block serialization and implement block vrsion adaptation it is not mandatory to use the lowest possible block version number, but it allows us to validate that we generate the same serialized block as the reference implementation --- .../org/biscuitsec/biscuit/token/Block.java | 47 ++++++++++++++++++- .../biscuitsec/biscuit/token/SamplesTest.java | 21 ++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/biscuitsec/biscuit/token/Block.java b/src/main/java/org/biscuitsec/biscuit/token/Block.java index 5931d52f..22d14a64 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/Block.java +++ b/src/main/java/org/biscuitsec/biscuit/token/Block.java @@ -2,6 +2,8 @@ import biscuit.format.schema.Schema; import org.biscuitsec.biscuit.crypto.PublicKey; +import org.biscuitsec.biscuit.datalog.expressions.Expression; +import org.biscuitsec.biscuit.datalog.expressions.Op; import org.biscuitsec.biscuit.error.Error; import org.biscuitsec.biscuit.datalog.*; import org.biscuitsec.biscuit.token.format.SerializedBiscuit; @@ -13,6 +15,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import static io.vavr.API.Left; import static io.vavr.API.Right; @@ -155,10 +158,52 @@ public Schema.Block serialize() { b.addPublicKeys(pk.serialize()); } - b.setVersion(SerializedBiscuit.MAX_SCHEMA_VERSION); + b.setVersion(getSchemaVersion()); return b.build(); } + int getSchemaVersion() { + boolean containsScopes = !this.scopes.isEmpty(); + boolean containsCheckAll = false; + boolean containsV4 = false; + + for (Rule r: this.rules) { + containsScopes |= !r.scopes().isEmpty(); + for(Expression e: r.expressions()) { + containsV4 |= containsV4Op(e); + } + } + for(Check c: this.checks) { + containsCheckAll |= c.kind() == Check.Kind.All; + + for (Rule q: c.queries()) { + containsScopes |= !q.scopes().isEmpty(); + for(Expression e: q.expressions()) { + containsV4 |= containsV4Op(e); + } + } + } + + if(containsScopes || containsCheckAll || containsV4) { + return SerializedBiscuit.MAX_SCHEMA_VERSION; + } else { + return SerializedBiscuit.MIN_SCHEMA_VERSION; + } + } + + boolean containsV4Op(Expression e) { + for (Op op: e.getOps()) { + if (op instanceof Op.Binary) { + Op.BinaryOp o = ((Op.Binary) op).getOp(); + if (o == Op.BinaryOp.BitwiseAnd || o == Op.BinaryOp.BitwiseOr || o == Op.BinaryOp.BitwiseXor || o == Op.BinaryOp.NotEqual) { + return true; + } + } + } + + return false; + } + /** * Deserializes a block from its Protobuf representation * diff --git a/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java b/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java index ff097c0a..d029708a 100644 --- a/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java +++ b/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java @@ -18,6 +18,7 @@ import org.biscuitsec.biscuit.token.builder.Expression; import org.biscuitsec.biscuit.token.builder.parser.ExpressionParser; import org.biscuitsec.biscuit.token.builder.parser.Parser; +import org.biscuitsec.biscuit.token.format.SignedBlock; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; @@ -30,6 +31,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +import static org.biscuitsec.biscuit.token.Block.from_bytes; import static org.junit.jupiter.api.Assertions.*; class SamplesTest { @@ -104,11 +106,26 @@ DynamicTest process_testcase(final TestCase testCase, final PublicKey publicKey, Biscuit token = Biscuit.from_bytes(data, publicKey); assertArrayEquals(token.serialize(), data); - List allBlocks = new ArrayList<>(); + /*List allBlocks = new ArrayList<>(); allBlocks.add(token.authority); allBlocks.addAll(token.blocks); - compareBlocks(testCase.token, allBlocks); + compareBlocks(testCase.token, allBlocks);*/ + byte[] ser_block_authority = token.authority.to_bytes().get(); + System.out.println(Arrays.toString(ser_block_authority)); + System.out.println(Arrays.toString(token.serializedBiscuit.authority.block)); + org.biscuitsec.biscuit.token.Block deser_block_authority = from_bytes(ser_block_authority, token.authority.externalKey).get(); + assertEquals(token.authority.print(token.symbols), deser_block_authority.print(token.symbols)); + assert(Arrays.equals(ser_block_authority, token.serializedBiscuit.authority.block)); + + for(int i = 0; i < token.blocks.size() - 1; i++) { + org.biscuitsec.biscuit.token.Block block = token.blocks.get(i); + SignedBlock signed_block = token.serializedBiscuit.blocks.get(i); + byte[] ser_block = block.to_bytes().get(); + org.biscuitsec.biscuit.token.Block deser_block = from_bytes(ser_block,block.externalKey).get(); + assertEquals(block.print(token.symbols), deser_block.print(token.symbols)); + assert(Arrays.equals(ser_block, signed_block.block)); + } List revocationIds = token.revocation_identifiers(); JsonArray validationRevocationIds = validation.getAsJsonArray("revocation_ids"); From 0f2148b636309d3a8b5f659e3260018d043b06ad Mon Sep 17 00:00:00 2001 From: Geoffroy Couprie Date: Sat, 15 Jun 2024 16:42:27 +0200 Subject: [PATCH 11/11] fix --- .../biscuit/token/builder/parser/Parser.java | 2 -- .../java/org/biscuitsec/biscuit/token/SamplesTest.java | 10 +++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java index 54eee4d5..26e45b15 100644 --- a/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java +++ b/src/main/java/org/biscuitsec/biscuit/token/builder/parser/Parser.java @@ -57,8 +57,6 @@ public static Either>, Block> datalog(long index, Symbo if (!code.isEmpty()) { int lineNumber = indexedLine._2; - System.out.println("NEW CODE LINE"); - System.out.println(code); List lineErrors = new ArrayList<>(); boolean parsed = false; diff --git a/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java b/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java index d029708a..9e13dbed 100644 --- a/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java +++ b/src/test/java/org/biscuitsec/biscuit/token/SamplesTest.java @@ -82,8 +82,11 @@ void compareBlock(SymbolTable baseSymbols, long sampleBlockIndex, Block sampleBl outputSample.get().setExternalKey(sampleExternalKey); } - System.out.println(outputSample.get().build().print(sampleSymbols)); + org.biscuitsec.biscuit.token.Block generatedSampleBlock = outputSample.get().build(); + System.out.println(generatedSampleBlock.symbols.symbols); System.out.println(block.symbols.symbols); + + System.out.println(outputSample.get().build().print(sampleSymbols)); System.out.println(block.print(sampleSymbols)); assertArrayEquals(outputSample.get().build().to_bytes().get(), block.to_bytes().get()); } @@ -106,11 +109,12 @@ DynamicTest process_testcase(final TestCase testCase, final PublicKey publicKey, Biscuit token = Biscuit.from_bytes(data, publicKey); assertArrayEquals(token.serialize(), data); - /*List allBlocks = new ArrayList<>(); + List allBlocks = new ArrayList<>(); allBlocks.add(token.authority); allBlocks.addAll(token.blocks); - compareBlocks(testCase.token, allBlocks);*/ + compareBlocks(testCase.token, allBlocks); + byte[] ser_block_authority = token.authority.to_bytes().get(); System.out.println(Arrays.toString(ser_block_authority)); System.out.println(Arrays.toString(token.serializedBiscuit.authority.block));