From cc80728b3958749c359ed283e4309554d551f65f Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 4 Nov 2024 18:07:13 +0100 Subject: [PATCH 01/44] dynamic struct hashing --- src/credentials/dynamic-record.test.ts | 46 +++++++------- src/credentials/dynamic-record.ts | 87 +++++++++++++++++--------- 2 files changed, 81 insertions(+), 52 deletions(-) diff --git a/src/credentials/dynamic-record.test.ts b/src/credentials/dynamic-record.test.ts index 30d191b..c77d60b 100644 --- a/src/credentials/dynamic-record.test.ts +++ b/src/credentials/dynamic-record.test.ts @@ -17,7 +17,7 @@ import { } from './dynamic-record.ts'; import { DynamicString } from './dynamic-string.ts'; import { NestedProvable } from '../nested.ts'; -import { mapEntries, mapObject, zipObjects } from '../util.ts'; +import { mapEntries, mapObject, zip, zipObjects } from '../util.ts'; import { test } from 'node:test'; import assert from 'assert'; import { hashCredential } from '../credential.ts'; @@ -51,14 +51,24 @@ let originalStruct = OriginalWrappedInStruct.fromValue(input); // subset schema and circuit that doesn't know the full original layout +const Fifth = DynamicRecord( + { field: Field, string: DynamicString({ maxLength: 5 }) }, + { maxEntries: 5 } +); + +// TODO fix this not being all equal +zip( + Fifth.provable.toFields(Fifth.from({ field: 2, string: '...' })), + Fifth.provable.toFields(Fifth.from({ field: 2, string: String.from('...') })) +).map(([a, b], i) => + console.log(a.toBigInt(), b.toBigInt(), a.equals(b).toBoolean()) +); + const Subschema = DynamicRecord( { // not necessarily in order third: DynamicString({ maxLength: 10 }), - fifth: DynamicRecord( - { field: Field, string: DynamicString({ maxLength: 5 }) }, - { maxEntries: 5 } - ), + fifth: Fifth, first: Field, }, { maxEntries: 10 } @@ -75,29 +85,23 @@ async function circuit() { record.get('first').assertEquals(1, 'first'); Provable.assertEqual(String, record.get('third'), String.from('something')); - // TODO fix hashing so that this works - // const Fifth = DynamicRecord( - // { field: Field, string: DynamicString({ maxLength: 5 }) }, - // { maxEntries: 5 } - // ); - // Provable.assertEqual( - // Fifth, - // record.get('fifth'), - // Fifth.from({ field: 2, string: '...' }) - // ); + Provable.assertEqual( + Fifth, + record.get('fifth'), + Fifth.from({ field: 2, string: String.from('...') }) + ); }); await test('DynamicRecord.getAny()', () => { record.getAny(Bool, 'second').assertEquals(true, 'second'); record.getAny(UInt64, 'fourth').assertEquals(UInt64.from(123n)); - // TODO fix hashing so that this no longer works - // records should be hashed in dynamic record form - const Fifth = Struct({ field: Field, string: String }); + // this works because structs are hashed in dynamic record form + const FifthStruct = Struct({ field: Field, string: String }); Provable.assertEqual( - Fifth, - record.getAny(Fifth, 'fifth'), - Fifth.fromValue({ field: 2, string: '...' }) + FifthStruct, + record.getAny(FifthStruct, 'fifth'), + FifthStruct.fromValue({ field: 2, string: '...' }) ); assert.throws(() => record.getAny(Bool, 'missing'), /Key not found/); diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index acb48bd..3186b7c 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -22,9 +22,9 @@ import { import { TypeBuilder } from '../provable-type-builder.ts'; import { assert, - assertDefined, assertExtendsShape, assertHasProperty, + hasProperty, mapEntries, mapObject, pad, @@ -47,12 +47,6 @@ export { extractProperty, }; -type GenericRecord = DynamicRecord<{}>; - -function GenericRecord(options: { maxEntries: number }) { - return DynamicRecord({}, options); -} - type DynamicRecord = DynamicRecordBase; function DynamicRecord< @@ -70,17 +64,11 @@ function DynamicRecord< ProvableType.get(type).empty() ); - TypeBuilder.shape({ - entries: array(Option(Struct({ key: Field, value: Field })), maxEntries), - actual: Unconstrained.withEmpty(emptyTKnown), - }) - .build() - .empty(); - return class DynamicRecord extends DynamicRecordBase { - // TODO: actually, the type should be From<> for the known subfields and unchanged for the unknown ones - // or anything really, when we have general hashing? - static from(value: From): DynamicRecordBase { + // accepted type is From<> for the known subfields and unchanged for the unknown ones + static from & UnknownRecord>( + value: T + ): DynamicRecordBase { return DynamicRecord.provable.fromValue(value); } @@ -108,11 +96,16 @@ function DynamicRecord< // validate that `actual` (at least) contains all known keys assertExtendsShape(actual, knownShape); - let entries = Object.entries(actual).map(([key, value]) => { + let actual_ = actual; //mapObject(actual, (value, key) => { + // return key in shape + // ? ProvableType.get(shape[key]).fromValue(value) + // : value; + // }); + + let entries = Object.entries(actual_).map(([key, value]) => { let type = NestedProvable.get( key in knownShape - ? // ? (knownShape[key] as any) - NestedProvable.fromValue(value) // TODO change after making hashing general + ? (knownShape[key] as any) : NestedProvable.fromValue(value) ); return { @@ -120,7 +113,10 @@ function DynamicRecord< value: packToField(type, type.fromValue(value)).toBigInt(), }; }); - return { entries: pad(entries, maxEntries, undefined), actual }; + return { + entries: pad(entries, maxEntries, undefined), + actual: actual_, + }; }, distinguish(x) { return x instanceof DynamicRecordBase; @@ -138,6 +134,18 @@ function DynamicRecord< } const OptionField = Option(Field); +const OptionKeyValue = Option(Struct({ key: Field, value: Field })); + +type GenericRecord = GenericRecordBase; + +function GenericRecord({ maxEntries }: { maxEntries: number }) { + // TODO provable + return class GenericRecord extends GenericRecordBase { + get maxEntries() { + return maxEntries; + } + }; +} class GenericRecordBase { entries: Option<{ key: Field; value: Field }>[]; @@ -152,9 +160,20 @@ class GenericRecordBase { throw Error('Need subclass'); } - static from(_: UnknownRecord): GenericRecordBase { - // TODO this could be implemented - throw Error('Need subclass'); + static from(actual: UnknownRecord): GenericRecordBase { + let entries = Object.entries(actual).map(([key, value]) => { + let type = NestedProvable.get(NestedProvable.fromValue(value)); + return { + key: packStringToField(key), + value: packToField(type, type.fromValue(value)), + }; + }); + let options = pad( + entries.map((entry) => OptionKeyValue.from(entry)), + this.prototype.maxEntries, + OptionKeyValue.none() + ); + return new this({ entries: options, actual: Unconstrained.from(actual) }); } getAny(valueType: A, key: string) { @@ -232,33 +251,39 @@ function packStringToField(string: string) { return Poseidon.hash(fields); } -function packToField(type: ProvableType, value: T) { +function packToField(type: ProvableType, value: T): Field { + // identify "record" types + if (isStruct(type) || value instanceof GenericRecordBase) { + return hashRecord(value); + } let fields = toFieldsPacked(type, value); if (fields.length === 1) return fields[0]!; return Poseidon.hash(fields); } function hashRecord(data: unknown) { - if (data instanceof DynamicRecord.Base) return data.hash(); + if (data instanceof GenericRecordBase) return data.hash(); assert( typeof data === 'object' && data !== null, 'Expected DynamicRecord or plain object as data' ); - let entryHashes = mapEntries(data, (key, value) => { + let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { let type = NestedProvable.get(NestedProvable.fromValue(value)); return [packStringToField(key), packToField(type, value)]; }); return Poseidon.hash(entryHashes.flat()); } +function isStruct(type: ProvableType): type is Struct { + return hasProperty(type, '_isStruct') && type._isStruct === true; +} + // compatible key extraction function extractProperty(data: unknown, key: string): unknown { if (data instanceof DynamicRecord.Base) return data.get(key); - assertHasProperty(data, key); - let value = data[key]; - assertDefined(value, `Key not found: "${key}"`); - return value; + assertHasProperty(data, key, `Key not found: "${key}"`); + return data[key]; } // serialize/deserialize From df1a01b6add02d03847d7885fb7197aa6b73eac7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 4 Nov 2024 18:15:01 +0100 Subject: [PATCH 02/44] fix weird error --- src/credentials/dynamic-record.test.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/credentials/dynamic-record.test.ts b/src/credentials/dynamic-record.test.ts index c77d60b..c88cbe9 100644 --- a/src/credentials/dynamic-record.test.ts +++ b/src/credentials/dynamic-record.test.ts @@ -52,18 +52,10 @@ let originalStruct = OriginalWrappedInStruct.fromValue(input); // subset schema and circuit that doesn't know the full original layout const Fifth = DynamicRecord( - { field: Field, string: DynamicString({ maxLength: 5 }) }, + { field: Field, string: String }, { maxEntries: 5 } ); -// TODO fix this not being all equal -zip( - Fifth.provable.toFields(Fifth.from({ field: 2, string: '...' })), - Fifth.provable.toFields(Fifth.from({ field: 2, string: String.from('...') })) -).map(([a, b], i) => - console.log(a.toBigInt(), b.toBigInt(), a.equals(b).toBoolean()) -); - const Subschema = DynamicRecord( { // not necessarily in order @@ -88,7 +80,7 @@ async function circuit() { Provable.assertEqual( Fifth, record.get('fifth'), - Fifth.from({ field: 2, string: String.from('...') }) + Fifth.from({ field: 2, string: '...' }) ); }); From be40c0854bc86fa80de684b897c15967367005f4 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 4 Nov 2024 20:53:24 +0100 Subject: [PATCH 03/44] clean up --- src/credentials/dynamic-record.ts | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index 3186b7c..2623d70 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -96,13 +96,7 @@ function DynamicRecord< // validate that `actual` (at least) contains all known keys assertExtendsShape(actual, knownShape); - let actual_ = actual; //mapObject(actual, (value, key) => { - // return key in shape - // ? ProvableType.get(shape[key]).fromValue(value) - // : value; - // }); - - let entries = Object.entries(actual_).map(([key, value]) => { + let entries = Object.entries(actual).map(([key, value]) => { let type = NestedProvable.get( key in knownShape ? (knownShape[key] as any) @@ -113,10 +107,7 @@ function DynamicRecord< value: packToField(type, type.fromValue(value)).toBigInt(), }; }); - return { - entries: pad(entries, maxEntries, undefined), - actual: actual_, - }; + return { entries: pad(entries, maxEntries, undefined), actual }; }, distinguish(x) { return x instanceof DynamicRecordBase; From 8439a5d5837b6169a37c5978fd44df0f4ea3985c Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 4 Nov 2024 21:04:53 +0100 Subject: [PATCH 04/44] split stuff into another file --- src/credential.ts | 4 +-- src/credentials/dynamic-hash.ts | 47 +++++++++++++++++++++++++++ src/credentials/dynamic-record.ts | 54 +++---------------------------- 3 files changed, 52 insertions(+), 53 deletions(-) create mode 100644 src/credentials/dynamic-hash.ts diff --git a/src/credential.ts b/src/credential.ts index 8a87e7a..419cd85 100644 --- a/src/credential.ts +++ b/src/credential.ts @@ -13,11 +13,9 @@ import { type InferNestedProvable, NestedProvable, type NestedProvableFor, - type NestedProvablePure, - type NestedProvablePureFor, } from './nested.ts'; import { zip } from './util.ts'; -import { hashRecord } from './credentials/dynamic-record.ts'; +import { hashRecord } from './credentials/dynamic-hash.ts'; export { type Credential, diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts new file mode 100644 index 0000000..ab6befa --- /dev/null +++ b/src/credentials/dynamic-hash.ts @@ -0,0 +1,47 @@ +/** + * Hashing of arbitrary data types compatible with dynamic-length schemas. + */ +import { Bytes, Field, Poseidon, Struct } from 'o1js'; +import { ProvableType, toFieldsPacked } from '../o1js-missing.ts'; +import { assert, hasProperty, mapEntries } from '../util.ts'; +import { NestedProvable } from '../nested.ts'; +import { GenericRecord, type UnknownRecord } from './dynamic-record.ts'; + +export { packStringToField, packToField, hashRecord }; + +// compatible hashing + +function packStringToField(string: string) { + let bytes = new TextEncoder().encode(string); + let B = Bytes(bytes.length); + let fields = toFieldsPacked(B, B.from(bytes)); + if (fields.length === 1) return fields[0]!; + return Poseidon.hash(fields); +} + +function packToField(type: ProvableType, value: T): Field { + // identify "record" types + if (isStruct(type) || value instanceof GenericRecord.Base) { + return hashRecord(value); + } + let fields = toFieldsPacked(type, value); + if (fields.length === 1) return fields[0]!; + return Poseidon.hash(fields); +} + +function hashRecord(data: unknown) { + if (data instanceof GenericRecord.Base) return data.hash(); + assert( + typeof data === 'object' && data !== null, + 'Expected DynamicRecord or plain object as data' + ); + let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { + let type = NestedProvable.get(NestedProvable.fromValue(value)); + return [packStringToField(key), packToField(type, value)]; + }); + return Poseidon.hash(entryHashes.flat()); +} + +function isStruct(type: ProvableType): type is Struct { + return hasProperty(type, '_isStruct') && type._isStruct === true; +} diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index 2623d70..5499bf4 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -2,7 +2,6 @@ * A dynamic record is a key-value list which can contain keys/values you are not aware of at compile time. */ import { - Bytes, Field, type From, type InferProvable, @@ -16,16 +15,12 @@ import { import { array, ProvableType, - toFieldsPacked, type ProvableHashableType, } from '../o1js-missing.ts'; import { TypeBuilder } from '../provable-type-builder.ts'; import { - assert, assertExtendsShape, assertHasProperty, - hasProperty, - mapEntries, mapObject, pad, zipObjects, @@ -37,15 +32,9 @@ import { serializeNestedProvable, serializeNestedProvableValue, } from '../serialize-provable.ts'; +import { packStringToField, packToField } from './dynamic-hash.ts'; -export { - DynamicRecord, - GenericRecord, - packStringToField, - packToField, - hashRecord, - extractProperty, -}; +export { DynamicRecord, GenericRecord, type UnknownRecord, extractProperty }; type DynamicRecord = DynamicRecordBase; @@ -210,6 +199,8 @@ class GenericRecordBase { } } +GenericRecord.Base = GenericRecordBase; + class DynamicRecordBase extends GenericRecordBase { get knownShape(): { [K in keyof TKnown]: ProvableHashableType } { throw Error('Need subclass'); @@ -232,43 +223,6 @@ type DynamicRecordRaw = { type UnknownRecord = Record; -// compatible hashing - -function packStringToField(string: string) { - let bytes = new TextEncoder().encode(string); - let B = Bytes(bytes.length); - let fields = toFieldsPacked(B, B.from(bytes)); - if (fields.length === 1) return fields[0]!; - return Poseidon.hash(fields); -} - -function packToField(type: ProvableType, value: T): Field { - // identify "record" types - if (isStruct(type) || value instanceof GenericRecordBase) { - return hashRecord(value); - } - let fields = toFieldsPacked(type, value); - if (fields.length === 1) return fields[0]!; - return Poseidon.hash(fields); -} - -function hashRecord(data: unknown) { - if (data instanceof GenericRecordBase) return data.hash(); - assert( - typeof data === 'object' && data !== null, - 'Expected DynamicRecord or plain object as data' - ); - let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { - let type = NestedProvable.get(NestedProvable.fromValue(value)); - return [packStringToField(key), packToField(type, value)]; - }); - return Poseidon.hash(entryHashes.flat()); -} - -function isStruct(type: ProvableType): type is Struct { - return hasProperty(type, '_isStruct') && type._isStruct === true; -} - // compatible key extraction function extractProperty(data: unknown, key: string): unknown { From 5842304092c6abc9e9cf3db255d5c29d55ba3026 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 4 Nov 2024 21:04:58 +0100 Subject: [PATCH 05/44] simplify test --- src/credentials/dynamic-record.test.ts | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/src/credentials/dynamic-record.test.ts b/src/credentials/dynamic-record.test.ts index c88cbe9..0e83774 100644 --- a/src/credentials/dynamic-record.test.ts +++ b/src/credentials/dynamic-record.test.ts @@ -3,25 +3,20 @@ import { Field, type From, type InferProvable, - Poseidon, Provable, ProvableType, Struct, UInt64, } from 'o1js'; -import { - DynamicRecord, - packToField, - packStringToField, - hashRecord, -} from './dynamic-record.ts'; +import { DynamicRecord } from './dynamic-record.ts'; import { DynamicString } from './dynamic-string.ts'; import { NestedProvable } from '../nested.ts'; -import { mapEntries, mapObject, zip, zipObjects } from '../util.ts'; +import { mapObject, zipObjects } from '../util.ts'; import { test } from 'node:test'; import assert from 'assert'; import { hashCredential } from '../credential.ts'; import { owner } from '../../tests/test-utils.ts'; +import { hashRecord } from './dynamic-hash.ts'; const String = DynamicString({ maxLength: 10 }); @@ -44,7 +39,7 @@ let input = { }; let original = OriginalSchema.from(input); -const expectedHash = OriginalSchema.hash(original); +const expectedHash = hashRecord(original); const OriginalWrappedInStruct = Struct(OriginalSchema.schema); let originalStruct = OriginalWrappedInStruct.fromValue(input); @@ -103,7 +98,6 @@ async function circuit() { record.hash().assertEquals(expectedHash, 'hash')); await test('hashRecord()', () => { - hashRecord(original).assertEquals(expectedHash); hashRecord(originalStruct).assertEquals(expectedHash); hashRecord(record).assertEquals(expectedHash); }); @@ -153,17 +147,5 @@ function Schema>(schema: A) { ); return actual; }, - - hash(value: { [K in keyof A]: From }) { - let normalized = this.from(value); - let entryHashes = mapEntries( - zipObjects(shape, normalized), - (key, [type, value]) => [ - packStringToField(key), - packToField(type, value), - ] - ); - return Poseidon.hash(entryHashes.flat()); - }, }; } From 33fe7a4e14bf28a17689fd0dc2e9ab6e3e4d8ee4 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 4 Nov 2024 21:23:40 +0100 Subject: [PATCH 06/44] dynamic array custom to input --- src/credentials/dynamic-array.ts | 42 ++++++++++++++++++++++++------- src/credentials/dynamic-bytes.ts | 4 +-- src/credentials/dynamic-string.ts | 4 +-- src/o1js-missing.ts | 1 + src/provable-type-builder.ts | 7 +++++- 5 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index 47d0b21..d894267 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -13,7 +13,12 @@ import { type IsPure, } from 'o1js'; import { assert, pad, zip } from '../util.ts'; -import { ProvableType } from '../o1js-missing.ts'; +import { + HashInput, + type ProvableHashablePure, + type ProvableHashableType, + ProvableType, +} from '../o1js-missing.ts'; import { assertInRange16, assertLessThan16, lessThan16 } from './gadgets.ts'; import { ProvableFactory } from '../provable-factory.ts'; import { @@ -65,7 +70,7 @@ type DynamicArrayClassPure = typeof DynamicArrayBase & * Instead, our methods ensure integrity of array operations _within_ the actual length. */ function DynamicArray< - A extends ProvableType, + A extends ProvableHashableType, T extends InferProvable = InferProvable, V extends InferValue = InferValue >( @@ -76,7 +81,7 @@ function DynamicArray< : DynamicArrayClass; function DynamicArray< - A extends ProvableType, + A extends ProvableHashableType, T extends InferProvable = InferProvable, V extends InferValue = InferValue >( @@ -126,7 +131,7 @@ class DynamicArrayBase { length: Field; // props to override - get innerType(): ProvableType { + get innerType(): ProvableHashableType { throw Error('Inner type must be defined in a subclass.'); } static get maxLength(): number { @@ -233,7 +238,7 @@ class DynamicArrayBase { * * **Warning**: The callback will be passed unconstrained dummy values. */ - map( + map( type: S, f: (t: T, i: number) => From ): DynamicArray, InferValue> { @@ -388,17 +393,17 @@ class DynamicArrayBase { DynamicArray.Base = DynamicArrayBase; function provable>( - type: ProvablePure, + type: ProvableHashablePure, Class: Class ): TypeBuilderPure, V[]>; function provable>( - type: Provable, + type: ProvableHashable, Class: Class ): TypeBuilder, V[]>; function provable>( - type: Provable, + type: ProvableHashable, Class: Class ) { let maxLength = Class.maxLength; @@ -427,6 +432,23 @@ function provable>( }, distinguish: (s) => s instanceof DynamicArrayBase, }) + + // custom hash input + .hashInput(({ array, length }) => { + let lengthInput: HashInput = { packed: [[length, 32]] }; + let arrayInput = array.map((x): HashInput => { + let { fields = [], packed = [] } = type.toInput(x); + return { + packed: fields + .map((x) => [x, 254] as [Field, number]) + .concat(packed), + }; + }); + return [lengthInput, ...arrayInput].reduce( + HashInput.append, + HashInput.empty + ); + }) ); } @@ -442,7 +464,9 @@ ProvableFactory.register(DynamicArray, { typeFromJSON(json) { let innerType = deserializeProvableType(json.innerType); - return DynamicArray(innerType, { maxLength: json.maxLength }); + return DynamicArray(innerType as ProvableHashableType, { + maxLength: json.maxLength, + }); }, valueToJSON(_, { array, length }) { diff --git a/src/credentials/dynamic-bytes.ts b/src/credentials/dynamic-bytes.ts index fc75dbe..6494c08 100644 --- a/src/credentials/dynamic-bytes.ts +++ b/src/credentials/dynamic-bytes.ts @@ -1,4 +1,4 @@ -import { Bool, Bytes, Field, Provable, UInt8 } from 'o1js'; +import { Bool, Bytes, Field, type ProvableHashable, UInt8 } from 'o1js'; import { DynamicArrayBase, provableDynamicArray } from './dynamic-array.ts'; import { ProvableFactory } from '../provable-factory.ts'; import { assert, chunk } from '../util.ts'; @@ -80,7 +80,7 @@ function DynamicBytes({ maxLength }: { maxLength: number }) { class DynamicBytesBase extends DynamicArrayBase { get innerType() { - return UInt8 as any as Provable; + return UInt8 as any as ProvableHashable; } /** diff --git a/src/credentials/dynamic-string.ts b/src/credentials/dynamic-string.ts index 58e6a21..31fc835 100644 --- a/src/credentials/dynamic-string.ts +++ b/src/credentials/dynamic-string.ts @@ -1,4 +1,4 @@ -import { Bool, Field, Provable, UInt8 } from 'o1js'; +import { Bool, Field, type ProvableHashable, UInt8 } from 'o1js'; import { DynamicArrayBase, provableDynamicArray } from './dynamic-array.ts'; import { ProvableFactory } from '../provable-factory.ts'; import { assert } from '../util.ts'; @@ -62,7 +62,7 @@ const dec = new TextDecoder(); class DynamicStringBase extends DynamicArrayBase { get innerType() { - return UInt8 as any as Provable; + return UInt8 as any as ProvableHashable; } /** diff --git a/src/o1js-missing.ts b/src/o1js-missing.ts index c9536a5..f51ef29 100644 --- a/src/o1js-missing.ts +++ b/src/o1js-missing.ts @@ -22,6 +22,7 @@ export { type ProvablePureType, type ProvableHashableType, type ProvableHashablePure, + type ProvableMaybeHashable, array, toFieldsPacked, HashInput, diff --git a/src/provable-type-builder.ts b/src/provable-type-builder.ts index 0730473..cab8ba4 100644 --- a/src/provable-type-builder.ts +++ b/src/provable-type-builder.ts @@ -10,7 +10,7 @@ import { type Field, } from 'o1js'; import type { NestedProvable } from './nested.ts'; -import type { ProvableHashablePure } from './o1js-missing.ts'; +import type { HashInput, ProvableHashablePure } from './o1js-missing.ts'; export { TypeBuilder, TypeBuilderPure }; @@ -87,6 +87,11 @@ class TypeBuilder { check(x); }); } + + hashInput(toInput: (x: T) => HashInput): TypeBuilder { + let type = this.type; + return new TypeBuilder({ ...type, toInput }); + } } class TypeBuilderPure extends TypeBuilder { From ea3eaa3dcbad2dec3d92703e12d1b0384f7286cd Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 07:55:49 +0100 Subject: [PATCH 07/44] minor refactor --- src/provable-factory.ts | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/provable-factory.ts b/src/provable-factory.ts index ad5ccd3..5329007 100644 --- a/src/provable-factory.ts +++ b/src/provable-factory.ts @@ -52,33 +52,33 @@ const ProvableFactory = { }, getRegistered(value: unknown) { - let key: string | undefined; - let factory: MapValue | undefined; - for (let [key_, factory_] of factories.entries()) { - if (value instanceof factory_.base) { - key = key_; - factory = factory_; + let entry: [string, MapValue] | undefined; + for (let [key, factory] of factories.entries()) { + if (value instanceof factory.base) { + entry = [key, factory]; } } - return [key, factory] as const; + return entry; }, tryToJSON(constructor: unknown): SerializedFactory | undefined { if (!hasProperty(constructor, 'prototype')) return undefined; - let [key, factory] = ProvableFactory.getRegistered(constructor.prototype); - if (factory === undefined) return undefined; + let entry = ProvableFactory.getRegistered(constructor.prototype); + if (entry === undefined) return undefined; + let [key, factory] = entry; let json = factory.typeToJSON(constructor as any); - return { _type: key!, ...json, _isFactory: true as const }; + return { _type: key, ...json, _isFactory: true as const }; }, tryValueToJSON( value: unknown ): (SerializedFactory & { value: any }) | undefined { - let [key, factory] = ProvableFactory.getRegistered(value); - if (factory === undefined) return undefined; + let entry = ProvableFactory.getRegistered(value); + if (entry === undefined) return undefined; + let [key, factory] = entry; let serializedType = factory.typeToJSON(value!.constructor as any); return { - _type: key!, + _type: key, ...serializedType, value: factory.valueToJSON(value!.constructor as any, value), _isFactory: true as const, From e932495a8723bc163058b9f7a503087bc48af6cb Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 09:50:54 +0100 Subject: [PATCH 08/44] general packing gadget --- src/credentials/gadgets.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/credentials/gadgets.ts b/src/credentials/gadgets.ts index 1378085..de4b3db 100644 --- a/src/credentials/gadgets.ts +++ b/src/credentials/gadgets.ts @@ -4,7 +4,26 @@ import { Bool, Field, Gadgets, Provable, UInt32 } from 'o1js'; import { assert } from '../util.ts'; -export { unsafeIf, seal, lessThan16, assertInRange16, assertLessThan16 }; +export { pack, unsafeIf, seal, lessThan16, assertInRange16, assertLessThan16 }; + +/** + * Pack a list of fields of bit size `chunkSize` each into a single field. + * Uses little-endian encoding. + * + * **Warning**: Assumes, but doesn't prove, that each chunk fits in the chunk size. + */ +function pack(chunks: Field[], chunkSize: number) { + let p = chunks.length * chunkSize; + assert( + p < Field.sizeInBits, + () => `pack(): too many chunks, got ${chunks.length} * ${chunkSize} = ${p}` + ); + let sum = Field(0); + chunks.forEach((chunk, i) => { + sum = sum.add(chunk.mul(1n << BigInt(i * chunkSize))); + }); + return sum.seal(); +} /** * Slightly more efficient version of Provable.if() which produces garbage if both t is a non-dummy and b is true. From 080e938e8b975b828f1f170f57ade1473a0b885a Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 10:51:03 +0100 Subject: [PATCH 09/44] switch to little endian packing --- examples/unique-hash.eg.ts | 5 +++-- src/index.ts | 1 + src/o1js-missing.ts | 37 +++++++++++++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/examples/unique-hash.eg.ts b/examples/unique-hash.eg.ts index 04ba7af..a894000 100644 --- a/examples/unique-hash.eg.ts +++ b/examples/unique-hash.eg.ts @@ -1,4 +1,4 @@ -import { Bytes, Field, Poseidon, UInt64 } from 'o1js'; +import { Bytes, Field, UInt64 } from 'o1js'; import { Spec, Operation, @@ -10,6 +10,7 @@ import { type InferSchema, DynamicString, DynamicArray, + hashPacked, } from '../src/index.ts'; import { issuer, @@ -136,7 +137,7 @@ let request = PresentationRequest.https( spec, { acceptedNations: FieldArray.from( - acceptedNations.map((s) => Poseidon.hashPacked(String, String.from(s))) + acceptedNations.map((s) => hashPacked(String, String.from(s))) ), acceptedIssuers: FieldArray.from(acceptedIssuers), currentDate: UInt64.from(Date.now()), diff --git a/src/index.ts b/src/index.ts index 38228cc..4a51451 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,5 +6,6 @@ export { DynamicArray } from './credentials/dynamic-array.ts'; export { StaticArray } from './credentials/static-array.ts'; export { DynamicBytes } from './credentials/dynamic-bytes.ts'; export { DynamicString } from './credentials/dynamic-string.ts'; +export { hashPacked } from './o1js-missing.ts'; export type { InferProvable as InferSchema } from 'o1js'; diff --git a/src/o1js-missing.ts b/src/o1js-missing.ts index f51ef29..0afacb7 100644 --- a/src/o1js-missing.ts +++ b/src/o1js-missing.ts @@ -6,7 +6,7 @@ import { Field, type InferProvable, type InferValue, - Packed, + Poseidon, Provable, type ProvableHashable, type ProvablePure, @@ -25,6 +25,7 @@ export { type ProvableMaybeHashable, array, toFieldsPacked, + hashPacked, HashInput, type WithProvable, }; @@ -152,6 +153,8 @@ type ProvableHashablePure = ProvablePure & /** * Pack a value to as few field elements as possible using `toInput()`, falling back to `toFields()` if that's not available. + * + * Note: Different than `Packed` in o1js, this uses little-endian packing. */ function toFieldsPacked( type_: WithProvable>, @@ -159,7 +162,37 @@ function toFieldsPacked( ): Field[] { let type = ProvableType.get(type_); if (type.toInput === undefined) return type.toFields(value); - return Packed.create(type as ProvableHashable).pack(value).packed; + + let { fields = [], packed = [] } = toInput(type, value); + let result = [...fields]; + let current = Field(0); + let currentSize = 0; + + for (let [field, size] of packed) { + if (currentSize + size < Field.sizeInBits) { + current = current.add(field.mul(1n << BigInt(currentSize))); + currentSize += size; + } else { + result.push(current.seal()); + current = field; + currentSize = size; + } + } + if (currentSize > 0) result.push(current.seal()); + return result; +} + +/** + * Hash a provable value efficiently, by first packing it into as few field elements as possible. + * + * Note: Different than `Poseidon.hashPacked()` and `Hashed` (by default) in o1js, this uses little-endian packing. + */ +function hashPacked( + type: WithProvable>, + value: T +): Field { + let fields = toFieldsPacked(type, value); + return Poseidon.hash(fields); } // temporary, until we land `StaticArray` From d41a1357c67fa4d895a0e3a79e7d84b5b68a4c99 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 10:52:11 +0100 Subject: [PATCH 10/44] compatible dynamic array hashing, first version --- src/credentials/dynamic-array.ts | 86 ++++++++++++++++++++++++++-- src/credentials/dynamic-hash.test.ts | 18 ++++++ src/credentials/dynamic-hash.ts | 69 ++++++++++++++++++++-- src/credentials/dynamic-record.ts | 11 ++-- src/credentials/static-array.ts | 9 +-- 5 files changed, 173 insertions(+), 20 deletions(-) create mode 100644 src/credentials/dynamic-hash.test.ts diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index d894267..8e7c3e9 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -11,15 +11,21 @@ import { type From, type ProvablePure, type IsPure, + Poseidon, } from 'o1js'; -import { assert, pad, zip } from '../util.ts'; +import { assert, assertHasProperty, chunk, pad, zip } from '../util.ts'; import { HashInput, type ProvableHashablePure, type ProvableHashableType, ProvableType, } from '../o1js-missing.ts'; -import { assertInRange16, assertLessThan16, lessThan16 } from './gadgets.ts'; +import { + assertInRange16, + assertLessThan16, + lessThan16, + pack, +} from './gadgets.ts'; import { ProvableFactory } from '../provable-factory.ts'; import { deserializeProvable, @@ -28,6 +34,8 @@ import { serializeProvableType, } from '../serialize-provable.ts'; import { TypeBuilder, TypeBuilderPure } from '../provable-type-builder.ts'; +import { StaticArray } from './static-array.ts'; +import { bitSize, packedFieldSize, packToField } from './dynamic-hash.ts'; export { DynamicArray }; @@ -282,6 +290,75 @@ class DynamicArrayBase { return state; } + /** + * Dynamic array hash that only depends on the actual values, not the padding. + */ + hash() { + let type = ProvableType.get(this.innerType); + + // assert that all padding elements are 0. this allows us to pack values into blocks + let NULL = ProvableType.synthesize(type); + this.forEach((x, isPadding) => { + Provable.assertEqualIf(isPadding, this.innerType, x, NULL); + }); + + // create blocks of 2 field elements each + // TODO abstract this into a `chunk()` method that returns a DynamicArray> + let mustPack = packedFieldSize(type) > 1; + let elementSize = bitSize(type); + let elementsPerHalfBlock = Math.floor(254 / elementSize); + if (elementsPerHalfBlock < 1) elementsPerHalfBlock = 1; // larger types are compressed + + let elementsPerBlock = 2 * elementsPerHalfBlock; + assert(!mustPack, 'TODO'); // this should get a separate branch here + + // TODO pack the `length` here as well, into the minimum (whole) number of elements + // we can just put zeros in front for the length and finally add it to the first block + + let Block = StaticArray(type, elementsPerBlock); + let maxBlocks = Math.ceil(this.maxLength / elementsPerBlock); + let Blocks = DynamicArray(Block, { maxLength: maxBlocks }); + + // nBlocks = ceil(length / elementsPerBlock) = floor((length + elementsPerBlock - 1) / elementsPerBlock) + let nBlocks = UInt32.Unsafe.fromField( + this.length.add(elementsPerBlock - 1) + ).div(elementsPerBlock).value; + let padded = pad(this.array, maxBlocks * elementsPerBlock, NULL); + let chunked = chunk(padded, elementsPerBlock).map(Block.from); + let blocks = new Blocks(chunked, nBlocks).map( + StaticArray(Field, 2), + (block) => { + let firstHalf = block.array + .slice(0, elementsPerHalfBlock) + .map((el) => packToField(type, el)); + let secondHalf = block.array + .slice(elementsPerHalfBlock) + .map((el) => packToField(type, el)); + return [pack(firstHalf, elementSize), pack(secondHalf, elementSize)]; + } + ); + + // TODO remove + // Provable.log({ + // elementSize, + // elementsPerBlock, + // maxBlocks, + // hash: blocks.array.flatMap((x) => x.array), + // }); + + // now hash the 2-field elements blocks, on permutation at a time + // TODO: first we hash the length, but this should be included in the rest + let state = Poseidon.initialState(); + state = Poseidon.update(state, [this.length, Field(0)]); + blocks.forEach((block, isPadding) => { + let newState = Poseidon.update(state, block.array); + state[0] = Provable.if(isPadding, state[0], newState[0]); + state[1] = Provable.if(isPadding, state[1], newState[1]); + state[2] = Provable.if(isPadding, state[2], newState[2]); + }); + return state[0]; + } + /** * Push a value, without changing the maxLength. * @@ -381,9 +458,8 @@ class DynamicArrayBase { } toValue() { - return ( - this.constructor as any as { provable: Provable } - ).provable.toValue(this); + assertHasProperty(this.constructor, 'provable', 'Need subclass'); + return (this.constructor.provable as Provable).toValue(this); } } diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts new file mode 100644 index 0000000..dc47547 --- /dev/null +++ b/src/credentials/dynamic-hash.test.ts @@ -0,0 +1,18 @@ +import { DynamicArray } from './dynamic-array.ts'; +import { DynamicString } from './dynamic-string.ts'; +import { hashString } from './dynamic-hash.ts'; + +let shortString = 'hi'; +let ShortString = DynamicString({ maxLength: 5 }); + +let longString = + 'Poseidon (/pəˈsaɪdən, pɒ-, poʊ-/;[1] Greek: Ποσειδῶν) is one of the Twelve Olympians in ancient Greek religion and mythology,' + + ' presiding over the sea, storms, earthquakes and horses.[2]'; +let LongString = DynamicString({ maxLength: 300 }); + +ShortString.from(shortString) + .hash() + .assertEquals(hashString(shortString), 'hash mismatch (short)'); +LongString.from(longString) + .hash() + .assertEquals(hashString(longString), 'hash mismatch (long)'); diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index ab6befa..dc5e223 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -1,25 +1,49 @@ /** * Hashing of arbitrary data types compatible with dynamic-length schemas. */ -import { Bytes, Field, Poseidon, Struct } from 'o1js'; -import { ProvableType, toFieldsPacked } from '../o1js-missing.ts'; +import { Bytes, Field, Poseidon, Struct, UInt8 } from 'o1js'; +import { + type ProvableHashableType, + ProvableType, + toFieldsPacked, +} from '../o1js-missing.ts'; import { assert, hasProperty, mapEntries } from '../util.ts'; import { NestedProvable } from '../nested.ts'; import { GenericRecord, type UnknownRecord } from './dynamic-record.ts'; -export { packStringToField, packToField, hashRecord }; +export { + hashString, + packStringToField, + packToField, + hashRecord, + bitSize, + packedFieldSize, +}; // compatible hashing +const enc = new TextEncoder(); + +function hashString(string: string) { + // encode length + bytes + let bytes = enc.encode(string); + let length = bytes.length; + let B = Bytes(length); + let fields = toFieldsPacked(B, B.from(bytes)); + return Poseidon.hash([Field(length), Field(0), ...fields]); +} + function packStringToField(string: string) { - let bytes = new TextEncoder().encode(string); + let bytes = enc.encode(string); let B = Bytes(bytes.length); let fields = toFieldsPacked(B, B.from(bytes)); if (fields.length === 1) return fields[0]!; return Poseidon.hash(fields); } -function packToField(type: ProvableType, value: T): Field { +function packToField(type: ProvableType | undefined, value: T): Field { + type ??= NestedProvable.get(NestedProvable.fromValue(value)); + // identify "record" types if (isStruct(type) || value instanceof GenericRecord.Base) { return hashRecord(value); @@ -42,6 +66,39 @@ function hashRecord(data: unknown) { return Poseidon.hash(entryHashes.flat()); } +// helpers + function isStruct(type: ProvableType): type is Struct { - return hasProperty(type, '_isStruct') && type._isStruct === true; + return ( + hasProperty(type, '_isStruct') && + type._isStruct === true && + // this shouldn't have been implemented as struct, it's just 1 field + type !== UInt8 + ); +} + +function bitSize(type: ProvableHashableType): number { + let provable = ProvableType.get(type); + let { fields = [], packed = [] } = provable.toInput(provable.empty()); + let nBits = fields.length * Field.sizeInBits; + for (let [, size] of packed) { + nBits += size; + } + return nBits; +} + +function packedFieldSize(type: ProvableHashableType): number { + let provable = ProvableType.get(type); + let { fields = [], packed = [] } = provable.toInput(provable.empty()); + let nFields = fields.length; + let pendingBits = 0; + for (let [, size] of packed) { + pendingBits += size; + if (pendingBits >= Field.sizeInBits) { + nFields++; + pendingBits -= Field.sizeInBits; + } + } + if (pendingBits > 0) nFields++; + return nFields; } diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index 5499bf4..90e4788 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -86,14 +86,15 @@ function DynamicRecord< assertExtendsShape(actual, knownShape); let entries = Object.entries(actual).map(([key, value]) => { - let type = NestedProvable.get( + let type = key in knownShape - ? (knownShape[key] as any) - : NestedProvable.fromValue(value) - ); + ? NestedProvable.get(knownShape[key]!) + : undefined; + let actualValue = + type === undefined ? value : type.fromValue(value); return { key: packStringToField(key).toBigInt(), - value: packToField(type, type.fromValue(value)).toBigInt(), + value: packToField(type, actualValue).toBigInt(), }; }); return { entries: pad(entries, maxEntries, undefined), actual }; diff --git a/src/credentials/static-array.ts b/src/credentials/static-array.ts index 9653b08..a71a15d 100644 --- a/src/credentials/static-array.ts +++ b/src/credentials/static-array.ts @@ -9,7 +9,7 @@ import { Gadgets, type ProvableHashable, } from 'o1js'; -import { assert, chunk, zip } from '../util.ts'; +import { assert, assertHasProperty, chunk, zip } from '../util.ts'; import { ProvableType } from '../o1js-missing.ts'; import { assertLessThan16, lessThan16 } from './gadgets.ts'; import { TypeBuilder } from '../provable-type-builder.ts'; @@ -125,6 +125,8 @@ class StaticArrayBase { /** * Gets value at index i, and proves that the index is in the array. * + * Handles constant indices without creating constraints. + * * Cost: TN + 1.5 */ get(i: UInt32 | number): T { @@ -276,9 +278,8 @@ class StaticArrayBase { } toValue() { - return ( - this.constructor as any as { provable: Provable } - ).provable.toValue(this); + assertHasProperty(this.constructor, 'provable', 'Need subclass'); + return (this.constructor.provable as Provable).toValue(this); } } From 5f693480ee9f3499d2f32b2b81fc4a452b6bdf94 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 11:17:20 +0100 Subject: [PATCH 11/44] more efficient length encoding --- src/credentials/dynamic-array.ts | 27 ++++++++++++++++++--------- src/credentials/dynamic-hash.test.ts | 16 ++++++++++------ src/credentials/dynamic-hash.ts | 12 ++++++++---- src/util.ts | 7 +++++++ 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index 8e7c3e9..b46d436 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -13,7 +13,7 @@ import { type IsPure, Poseidon, } from 'o1js'; -import { assert, assertHasProperty, chunk, pad, zip } from '../util.ts'; +import { assert, assertHasProperty, chunk, fill, pad, zip } from '../util.ts'; import { HashInput, type ProvableHashablePure, @@ -291,7 +291,7 @@ class DynamicArrayBase { } /** - * Dynamic array hash that only depends on the actual values, not the padding. + * Dynamic array hash that only depends on the actual values (not the padding). */ hash() { let type = ProvableType.get(this.innerType); @@ -307,23 +307,28 @@ class DynamicArrayBase { let mustPack = packedFieldSize(type) > 1; let elementSize = bitSize(type); let elementsPerHalfBlock = Math.floor(254 / elementSize); - if (elementsPerHalfBlock < 1) elementsPerHalfBlock = 1; // larger types are compressed + if (elementsPerHalfBlock === 0) elementsPerHalfBlock = 1; // larger types are compressed let elementsPerBlock = 2 * elementsPerHalfBlock; assert(!mustPack, 'TODO'); // this should get a separate branch here - // TODO pack the `length` here as well, into the minimum (whole) number of elements - // we can just put zeros in front for the length and finally add it to the first block + // we pack the length at the beginning of the first block + // for efficiency (to avoid unpacking the length), we first put zeros at the beginning + // and later just add the length to the first block + let elementsPerUint32 = Math.max(Math.floor(32 / elementSize), 1); + let array = fill(elementsPerUint32, NULL).concat(this.array); let Block = StaticArray(type, elementsPerBlock); - let maxBlocks = Math.ceil(this.maxLength / elementsPerBlock); + let maxBlocks = Math.ceil( + (elementsPerUint32 + this.maxLength) / elementsPerBlock + ); let Blocks = DynamicArray(Block, { maxLength: maxBlocks }); // nBlocks = ceil(length / elementsPerBlock) = floor((length + elementsPerBlock - 1) / elementsPerBlock) let nBlocks = UInt32.Unsafe.fromField( - this.length.add(elementsPerBlock - 1) + this.length.add(elementsPerUint32 + elementsPerBlock - 1) ).div(elementsPerBlock).value; - let padded = pad(this.array, maxBlocks * elementsPerBlock, NULL); + let padded = pad(array, maxBlocks * elementsPerBlock, NULL); let chunked = chunk(padded, elementsPerBlock).map(Block.from); let blocks = new Blocks(chunked, nBlocks).map( StaticArray(Field, 2), @@ -338,8 +343,13 @@ class DynamicArrayBase { } ); + // add length to the first block + let firstBlock = blocks.array[0]!; + firstBlock.set(0, firstBlock.get(0).add(this.length).seal()); + // TODO remove // Provable.log({ + // elementsPerUint32, // elementSize, // elementsPerBlock, // maxBlocks, @@ -349,7 +359,6 @@ class DynamicArrayBase { // now hash the 2-field elements blocks, on permutation at a time // TODO: first we hash the length, but this should be included in the rest let state = Poseidon.initialState(); - state = Poseidon.update(state, [this.length, Field(0)]); blocks.forEach((block, isPadding) => { let newState = Poseidon.update(state, block.array); state[0] = Provable.if(isPadding, state[0], newState[0]); diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index dc47547..ca56fc8 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -1,6 +1,7 @@ import { DynamicArray } from './dynamic-array.ts'; import { DynamicString } from './dynamic-string.ts'; import { hashString } from './dynamic-hash.ts'; +import { test } from 'node:test'; let shortString = 'hi'; let ShortString = DynamicString({ maxLength: 5 }); @@ -10,9 +11,12 @@ let longString = ' presiding over the sea, storms, earthquakes and horses.[2]'; let LongString = DynamicString({ maxLength: 300 }); -ShortString.from(shortString) - .hash() - .assertEquals(hashString(shortString), 'hash mismatch (short)'); -LongString.from(longString) - .hash() - .assertEquals(hashString(longString), 'hash mismatch (long)'); +test('hash strings', () => { + ShortString.from(shortString) + .hash() + .assertEquals(hashString(shortString), 'hash mismatch (short)'); + + LongString.from(longString) + .hash() + .assertEquals(hashString(longString), 'hash mismatch (long)'); +}); diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index dc5e223..9421561 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -26,11 +26,15 @@ const enc = new TextEncoder(); function hashString(string: string) { // encode length + bytes - let bytes = enc.encode(string); - let length = bytes.length; - let B = Bytes(length); + let stringBytes = enc.encode(string); + let length = stringBytes.length; + let bytes = new Uint8Array(4 + length); + new DataView(bytes.buffer).setUint32(0, length, true); + bytes.set(stringBytes, 4); + let B = Bytes(4 + length); let fields = toFieldsPacked(B, B.from(bytes)); - return Poseidon.hash([Field(length), Field(0), ...fields]); + // console.log({ hashString: fields.map((x) => x.toBigInt()), bytes }); + return Poseidon.hash(fields); } function packStringToField(string: string) { diff --git a/src/util.ts b/src/util.ts index 9e631d2..0e6a230 100644 --- a/src/util.ts +++ b/src/util.ts @@ -8,6 +8,7 @@ export { zip, chunk, pad, + fill, mapObject, mapEntries, zipObjects, @@ -95,6 +96,12 @@ function pad(array: T[], size: number, value: T | (() => T)): T[] { return array.concat(Array.from({ length: size - array.length }, cb)); } +function fill(size: number, value: T | (() => T)): T[] { + let cb: () => T = + typeof value === 'function' ? (value as () => T) : () => value; + return Array.from({ length: size }, cb); +} + function mapObject< T extends Record, S extends Record From d7a66ee6285f3881654400ede2d0ced8eb0c46db Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 12:43:20 +0100 Subject: [PATCH 12/44] convert between dynamic arrays of different sizes --- src/credentials/dynamic-array.ts | 8 +++++-- src/credentials/dynamic-string.ts | 8 +++++-- src/provable-type-builder.ts | 37 ++++++++++++++++++++++--------- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index b46d436..000ca48 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -511,11 +511,15 @@ function provable>( there({ array, length }) { return array.slice(0, Number(length)); }, - back(array) { + backAndDistinguish(array) { + // gracefully handle different maxLength + if (array instanceof DynamicArrayBase) { + if (array.maxLength === maxLength) return array; + array = array.toValue(); + } let padded = pad(array, maxLength, NULL); return { array: padded, length: BigInt(array.length) }; }, - distinguish: (s) => s instanceof DynamicArrayBase, }) // custom hash input diff --git a/src/credentials/dynamic-string.ts b/src/credentials/dynamic-string.ts index 31fc835..704bfa3 100644 --- a/src/credentials/dynamic-string.ts +++ b/src/credentials/dynamic-string.ts @@ -47,10 +47,14 @@ function DynamicString({ maxLength }: { maxLength: number }) { there(s) { return dec.decode(Uint8Array.from(s, ({ value }) => Number(value))); }, - back(s) { + backAndDistinguish(s) { + // gracefully handle different maxLength + if (s instanceof DynamicStringBase) { + if (s.maxLength === maxLength) return s; + s = s.toString(); + } return [...enc.encode(s)].map((t) => ({ value: BigInt(t) })); }, - distinguish: (s) => s instanceof DynamicStringBase, }) .build(); diff --git a/src/provable-type-builder.ts b/src/provable-type-builder.ts index cab8ba4..e7222ae 100644 --- a/src/provable-type-builder.ts +++ b/src/provable-type-builder.ts @@ -58,11 +58,18 @@ class TypeBuilder { }); } - mapValue(transform: { - there: (x: V) => W; - back: (x: W) => V; - distinguish: (x: T | W) => x is T; - }): TypeBuilder { + mapValue( + transform: + | { + there: (x: V) => W; + back: (x: W) => V; + distinguish: (x: T | W) => x is T; + } + | { + there: (x: V) => W; + backAndDistinguish: (x: W | T) => V | T; + } + ): TypeBuilder { let type = this.type; return new TypeBuilder({ ...type, @@ -71,6 +78,9 @@ class TypeBuilder { return transform.there(type.toValue(value)); }, fromValue(value) { + if ('backAndDistinguish' in transform) { + return type.fromValue(transform.backAndDistinguish(value)); + } if (transform.distinguish(value)) return value; return type.fromValue(transform.back(value)); }, @@ -109,11 +119,18 @@ class TypeBuilderPure extends TypeBuilder { return super.forConstructor(constructor) as TypeBuilderPure; } - mapValue(transform: { - there: (x: V) => W; - back: (x: W) => V; - distinguish: (x: T | W) => x is T; - }): TypeBuilderPure { + mapValue( + transform: + | { + there: (x: V) => W; + back: (x: W) => V; + distinguish: (x: T | W) => x is T; + } + | { + there: (x: V) => W; + backAndDistinguish: (x: W | T) => V | T; + } + ): TypeBuilderPure { return super.mapValue(transform) as TypeBuilderPure; } From 25b6f80cecd3ccc006a7bef2e90c6e26f950cd6a Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 12:44:48 +0100 Subject: [PATCH 13/44] use dynamic array hashing when packing attributes --- examples/unique-hash.eg.ts | 5 ++++- src/credentials/dynamic-hash.test.ts | 18 ++++++++++++++++-- src/credentials/dynamic-hash.ts | 7 ++++++- src/credentials/dynamic-record.test.ts | 22 +++++++++++++++++----- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/examples/unique-hash.eg.ts b/examples/unique-hash.eg.ts index a894000..daa3159 100644 --- a/examples/unique-hash.eg.ts +++ b/examples/unique-hash.eg.ts @@ -81,9 +81,12 @@ console.log('✅ WALLET: imported and validated credential'); // VERIFIER: request a presentation // it's enough to know a subset of the schema to create the request +// and we don't have to use the original string lengths +const NewString = DynamicString({ maxLength: 30 }); + const Subschema = DynamicRecord( { - nationality: String, + nationality: NewString, expiresAt: UInt64, // we don't have to match the original order of keys id: Bytes16, }, diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index ca56fc8..db7d552 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -2,6 +2,7 @@ import { DynamicArray } from './dynamic-array.ts'; import { DynamicString } from './dynamic-string.ts'; import { hashString } from './dynamic-hash.ts'; import { test } from 'node:test'; +import * as nodeAssert from 'node:assert'; let shortString = 'hi'; let ShortString = DynamicString({ maxLength: 5 }); @@ -12,11 +13,24 @@ let longString = let LongString = DynamicString({ maxLength: 300 }); test('hash strings', () => { + let shortHash = hashString(shortString); ShortString.from(shortString) .hash() - .assertEquals(hashString(shortString), 'hash mismatch (short)'); + .assertEquals(shortHash, 'hash mismatch (short)'); + let longHash = hashString(longString); LongString.from(longString) .hash() - .assertEquals(hashString(longString), 'hash mismatch (long)'); + .assertEquals(longHash, 'hash mismatch (long)'); + + // we can even convert the `ShortString` into a `LongString` + LongString.provable + .fromValue(ShortString.from(shortString)) + .hash() + .assertEquals(shortHash, 'hash mismatch (short -> long)'); + + // (the other way round doesn't work because the string is too long) + nodeAssert.throws(() => { + ShortString.provable.fromValue(LongString.from(longString)); + }, /larger than target size/); }); diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index 9421561..00bc407 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -10,6 +10,7 @@ import { import { assert, hasProperty, mapEntries } from '../util.ts'; import { NestedProvable } from '../nested.ts'; import { GenericRecord, type UnknownRecord } from './dynamic-record.ts'; +import { DynamicArray } from './dynamic-array.ts'; export { hashString, @@ -48,10 +49,14 @@ function packStringToField(string: string) { function packToField(type: ProvableType | undefined, value: T): Field { type ??= NestedProvable.get(NestedProvable.fromValue(value)); - // identify "record" types + // record types if (isStruct(type) || value instanceof GenericRecord.Base) { return hashRecord(value); } + // dynamic array types + if (value instanceof DynamicArray.Base) { + return value.hash(); + } let fields = toFieldsPacked(type, value); if (fields.length === 1) return fields[0]!; return Poseidon.hash(fields); diff --git a/src/credentials/dynamic-record.test.ts b/src/credentials/dynamic-record.test.ts index 0e83774..4d34993 100644 --- a/src/credentials/dynamic-record.test.ts +++ b/src/credentials/dynamic-record.test.ts @@ -46,15 +46,22 @@ let originalStruct = OriginalWrappedInStruct.fromValue(input); // subset schema and circuit that doesn't know the full original layout +// not necessarily matches the length of the original schema +const MyString = DynamicString({ maxLength: 20 }); + const Fifth = DynamicRecord( - { field: Field, string: String }, + { + field: Field, + // different max length here as well + string: DynamicString({ maxLength: 5 }), + }, { maxEntries: 5 } ); const Subschema = DynamicRecord( { // not necessarily in order - third: DynamicString({ maxLength: 10 }), + third: MyString, fifth: Fifth, first: Field, }, @@ -70,7 +77,11 @@ async function circuit() { await test('DynamicRecord.get()', () => { record.get('first').assertEquals(1, 'first'); - Provable.assertEqual(String, record.get('third'), String.from('something')); + Provable.assertEqual( + MyString, + record.get('third'), + MyString.from('something') + ); Provable.assertEqual( Fifth, @@ -83,8 +94,9 @@ async function circuit() { record.getAny(Bool, 'second').assertEquals(true, 'second'); record.getAny(UInt64, 'fourth').assertEquals(UInt64.from(123n)); - // this works because structs are hashed in dynamic record form - const FifthStruct = Struct({ field: Field, string: String }); + // this works because structs are hashed in dynamic record style, + // and the string is hashed in dynamic array style + const FifthStruct = Struct({ field: Field, string: MyString }); Provable.assertEqual( FifthStruct, record.getAny(FifthStruct, 'fifth'), From 1c1b9118ba1ddba31bb3d3f9891655c627d58394 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 13:39:14 +0100 Subject: [PATCH 14/44] test nested subschema --- src/credentials/dynamic-record.test.ts | 33 +++++++++++++------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/credentials/dynamic-record.test.ts b/src/credentials/dynamic-record.test.ts index 4d34993..f47e908 100644 --- a/src/credentials/dynamic-record.test.ts +++ b/src/credentials/dynamic-record.test.ts @@ -18,16 +18,16 @@ import { hashCredential } from '../credential.ts'; import { owner } from '../../tests/test-utils.ts'; import { hashRecord } from './dynamic-hash.ts'; -const String = DynamicString({ maxLength: 10 }); +const String10 = DynamicString({ maxLength: 10 }); // original schema, data and hash from known layout const OriginalSchema = Schema({ first: Field, second: Bool, - third: String, + third: String10, fourth: UInt64, - fifth: { field: Field, string: String }, + fifth: { field: Field, string: String10 }, }); let input = { @@ -47,13 +47,13 @@ let originalStruct = OriginalWrappedInStruct.fromValue(input); // subset schema and circuit that doesn't know the full original layout // not necessarily matches the length of the original schema -const MyString = DynamicString({ maxLength: 20 }); +const String20 = DynamicString({ maxLength: 20 }); +const String5 = DynamicString({ maxLength: 5 }); const Fifth = DynamicRecord( { - field: Field, - // different max length here as well - string: DynamicString({ maxLength: 5 }), + // _nested_ subset of original schema + string: String5, // different max length here as well }, { maxEntries: 5 } ); @@ -61,7 +61,7 @@ const Fifth = DynamicRecord( const Subschema = DynamicRecord( { // not necessarily in order - third: MyString, + third: String20, fifth: Fifth, first: Field, }, @@ -76,18 +76,19 @@ async function circuit() { let record = Provable.witness(Subschema, () => original); await test('DynamicRecord.get()', () => { + // static field record.get('first').assertEquals(1, 'first'); + + // dynamic string with different max length Provable.assertEqual( - MyString, + String20, record.get('third'), - MyString.from('something') + String20.from('something') ); - Provable.assertEqual( - Fifth, - record.get('fifth'), - Fifth.from({ field: 2, string: '...' }) - ); + // nested subschema + let fifthString = record.get('fifth').get('string'); + Provable.assertEqual(String5, fifthString, String5.from('...')); }); await test('DynamicRecord.getAny()', () => { @@ -96,7 +97,7 @@ async function circuit() { // this works because structs are hashed in dynamic record style, // and the string is hashed in dynamic array style - const FifthStruct = Struct({ field: Field, string: MyString }); + const FifthStruct = Struct({ field: Field, string: String20 }); Provable.assertEqual( FifthStruct, record.getAny(FifthStruct, 'fifth'), From 191eaf0d1c19e98f414eae9a2e86c03744c0c071 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 17:14:04 +0100 Subject: [PATCH 15/44] switch order in packToFields --- src/credentials/dynamic-array.ts | 4 ++-- src/credentials/dynamic-hash.ts | 4 ++-- src/credentials/dynamic-record.ts | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index 000ca48..be17390 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -335,10 +335,10 @@ class DynamicArrayBase { (block) => { let firstHalf = block.array .slice(0, elementsPerHalfBlock) - .map((el) => packToField(type, el)); + .map((el) => packToField(el, type)); let secondHalf = block.array .slice(elementsPerHalfBlock) - .map((el) => packToField(type, el)); + .map((el) => packToField(el, type)); return [pack(firstHalf, elementSize), pack(secondHalf, elementSize)]; } ); diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index 00bc407..ac71808 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -46,7 +46,7 @@ function packStringToField(string: string) { return Poseidon.hash(fields); } -function packToField(type: ProvableType | undefined, value: T): Field { +function packToField(value: T, type?: ProvableType): Field { type ??= NestedProvable.get(NestedProvable.fromValue(value)); // record types @@ -70,7 +70,7 @@ function hashRecord(data: unknown) { ); let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { let type = NestedProvable.get(NestedProvable.fromValue(value)); - return [packStringToField(key), packToField(type, value)]; + return [packStringToField(key), packToField(value, type)]; }); return Poseidon.hash(entryHashes.flat()); } diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index 90e4788..db28e0f 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -94,7 +94,7 @@ function DynamicRecord< type === undefined ? value : type.fromValue(value); return { key: packStringToField(key).toBigInt(), - value: packToField(type, actualValue).toBigInt(), + value: packToField(actualValue, type).toBigInt(), }; }); return { entries: pad(entries, maxEntries, undefined), actual }; @@ -146,7 +146,7 @@ class GenericRecordBase { let type = NestedProvable.get(NestedProvable.fromValue(value)); return { key: packStringToField(key), - value: packToField(type, type.fromValue(value)), + value: packToField(type.fromValue(value), type), }; }); let options = pad( @@ -176,7 +176,7 @@ class GenericRecordBase { ); // assert that value matches hash, and return it - packToField(valueType, value).assertEquals( + packToField(value, valueType).assertEquals( valueHash, `Bug: Invalid value for key "${key}"` ); From fe21ccd03fcfa7325a0ec437506c7ca3db18ea0b Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 17:14:20 +0100 Subject: [PATCH 16/44] handle full field case in dynamic array hash --- src/credentials/dynamic-array.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index be17390..16afb09 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -307,6 +307,7 @@ class DynamicArrayBase { let mustPack = packedFieldSize(type) > 1; let elementSize = bitSize(type); let elementsPerHalfBlock = Math.floor(254 / elementSize); + let fullField = elementsPerHalfBlock === 0; if (elementsPerHalfBlock === 0) elementsPerHalfBlock = 1; // larger types are compressed let elementsPerBlock = 2 * elementsPerHalfBlock; @@ -339,6 +340,7 @@ class DynamicArrayBase { let secondHalf = block.array .slice(elementsPerHalfBlock) .map((el) => packToField(el, type)); + if (fullField) return [firstHalf[0]!, secondHalf[1]!]; return [pack(firstHalf, elementSize), pack(secondHalf, elementSize)]; } ); From 772cb2f14d1d040a588897d2ccf608cd40bfa3f7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 17:14:32 +0100 Subject: [PATCH 17/44] add two failing tests for array hashing --- src/credentials/dynamic-hash.test.ts | 16 +++++++++++++++- src/credentials/dynamic-record.test.ts | 11 +++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index db7d552..9719767 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -1,6 +1,6 @@ import { DynamicArray } from './dynamic-array.ts'; import { DynamicString } from './dynamic-string.ts'; -import { hashString } from './dynamic-hash.ts'; +import { hashString, packToField } from './dynamic-hash.ts'; import { test } from 'node:test'; import * as nodeAssert from 'node:assert'; @@ -34,3 +34,17 @@ test('hash strings', () => { ShortString.provable.fromValue(LongString.from(longString)); }, /larger than target size/); }); + +let ShortArray = DynamicArray(ShortString, { maxLength: 5 }); + +test('hash arrays', () => { + let shortArrayHash = packToField([shortString, shortString]); + ShortArray.from([shortString, shortString]) + .hash() + .assertEquals(shortArrayHash, 'hash mismatch (short array)'); + + let longArrayHash = packToField([longString, longString]); + ShortArray.from([longString, longString]) + .hash() + .assertEquals(longArrayHash, 'hash mismatch (long array)'); +}); diff --git a/src/credentials/dynamic-record.test.ts b/src/credentials/dynamic-record.test.ts index f47e908..d2b9512 100644 --- a/src/credentials/dynamic-record.test.ts +++ b/src/credentials/dynamic-record.test.ts @@ -17,6 +17,8 @@ import assert from 'assert'; import { hashCredential } from '../credential.ts'; import { owner } from '../../tests/test-utils.ts'; import { hashRecord } from './dynamic-hash.ts'; +import { array } from '../o1js-missing.ts'; +import { DynamicArray } from './dynamic-array.ts'; const String10 = DynamicString({ maxLength: 10 }); @@ -28,6 +30,7 @@ const OriginalSchema = Schema({ third: String10, fourth: UInt64, fifth: { field: Field, string: String10 }, + sixth: array(Field, 3), }); let input = { @@ -36,6 +39,7 @@ let input = { third: 'something', fourth: 123n, fifth: { field: 2, string: '...' }, + sixth: [1n, 2n, 3n], }; let original = OriginalSchema.from(input); @@ -104,6 +108,13 @@ async function circuit() { FifthStruct.fromValue({ field: 2, string: '...' }) ); + const SixthDynamic = DynamicArray(Field, { maxLength: 7 }); + Provable.assertEqual( + SixthDynamic, + record.getAny(SixthDynamic, 'sixth'), + SixthDynamic.from([1n, 2n, 3n]) + ); + assert.throws(() => record.getAny(Bool, 'missing'), /Key not found/); }); From 4b8b5c62c1be1872f708b388ac14e01dc1ad69f4 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 18:11:00 +0100 Subject: [PATCH 18/44] fix import cycles --- src/credentials/dynamic-array.ts | 2 ++ src/credentials/dynamic-base-types.ts | 39 +++++++++++++++++++++++++++ src/credentials/dynamic-record.ts | 3 +++ src/credentials/dynamic-string.ts | 2 ++ 4 files changed, 46 insertions(+) create mode 100644 src/credentials/dynamic-base-types.ts diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index 16afb09..ad58e8b 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -36,6 +36,7 @@ import { import { TypeBuilder, TypeBuilderPure } from '../provable-type-builder.ts'; import { StaticArray } from './static-array.ts'; import { bitSize, packedFieldSize, packToField } from './dynamic-hash.ts'; +import { BaseType } from './dynamic-base-types.ts'; export { DynamicArray }; @@ -126,6 +127,7 @@ function DynamicArray< return DynamicArray_; } +BaseType.set('DynamicArray', DynamicArray); class DynamicArrayBase { /** diff --git a/src/credentials/dynamic-base-types.ts b/src/credentials/dynamic-base-types.ts new file mode 100644 index 0000000..06b5564 --- /dev/null +++ b/src/credentials/dynamic-base-types.ts @@ -0,0 +1,39 @@ +/** + * This file is just a hack to break import cycles + */ +import type { DynamicArray } from './dynamic-array.ts'; +import type { DynamicString } from './dynamic-string.ts'; +import type { DynamicRecord, GenericRecord } from './dynamic-record.ts'; +import { assertDefined } from '../util.ts'; + +export { BaseType }; + +let baseType: { + DynamicArray?: typeof DynamicArray; + DynamicString?: typeof DynamicString; + DynamicRecord?: typeof DynamicRecord; + GenericRecord?: typeof GenericRecord; +} = {}; +type BaseType = typeof baseType; + +const BaseType = { + set(key: K, value: BaseType[K]) { + baseType[key] = value; + }, + get DynamicArray() { + assertDefined(baseType.DynamicArray); + return baseType.DynamicArray; + }, + get DynamicString() { + assertDefined(baseType.DynamicString); + return baseType.DynamicString; + }, + get DynamicRecord() { + assertDefined(baseType.DynamicRecord); + return baseType.DynamicRecord; + }, + get GenericRecord() { + assertDefined(baseType.GenericRecord); + return baseType.GenericRecord; + }, +}; diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index db28e0f..5af99b1 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -33,6 +33,7 @@ import { serializeNestedProvableValue, } from '../serialize-provable.ts'; import { packStringToField, packToField } from './dynamic-hash.ts'; +import { BaseType } from './dynamic-base-types.ts'; export { DynamicRecord, GenericRecord, type UnknownRecord, extractProperty }; @@ -113,6 +114,7 @@ function DynamicRecord< } }; } +BaseType.set('DynamicRecord', DynamicRecord); const OptionField = Option(Field); const OptionKeyValue = Option(Struct({ key: Field, value: Field })); @@ -127,6 +129,7 @@ function GenericRecord({ maxEntries }: { maxEntries: number }) { } }; } +BaseType.set('GenericRecord', GenericRecord); class GenericRecordBase { entries: Option<{ key: Field; value: Field }>[]; diff --git a/src/credentials/dynamic-string.ts b/src/credentials/dynamic-string.ts index 704bfa3..9f4fb67 100644 --- a/src/credentials/dynamic-string.ts +++ b/src/credentials/dynamic-string.ts @@ -2,6 +2,7 @@ import { Bool, Field, type ProvableHashable, UInt8 } from 'o1js'; import { DynamicArrayBase, provableDynamicArray } from './dynamic-array.ts'; import { ProvableFactory } from '../provable-factory.ts'; import { assert } from '../util.ts'; +import { BaseType } from './dynamic-base-types.ts'; export { DynamicString }; @@ -60,6 +61,7 @@ function DynamicString({ maxLength }: { maxLength: number }) { return DynamicString; } +BaseType.set('DynamicString', DynamicString); const enc = new TextEncoder(); const dec = new TextDecoder(); From e889027ca998d02e8a77c6b36cca5284d6e6b8cb Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 18:11:56 +0100 Subject: [PATCH 19/44] generic hash --- src/credentials/dynamic-hash.ts | 110 ++++++++++++++++++++++++++++---- 1 file changed, 99 insertions(+), 11 deletions(-) diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index ac71808..49960b4 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -1,18 +1,29 @@ /** * Hashing of arbitrary data types compatible with dynamic-length schemas. */ -import { Bytes, Field, Poseidon, Struct, UInt8 } from 'o1js'; +import { + Bool, + Bytes, + Field, + Poseidon, + Struct, + UInt64, + UInt8, + Undefined, +} from 'o1js'; import { type ProvableHashableType, ProvableType, toFieldsPacked, } from '../o1js-missing.ts'; -import { assert, hasProperty, mapEntries } from '../util.ts'; +import { assert, hasProperty, isSubclass, mapEntries } from '../util.ts'; import { NestedProvable } from '../nested.ts'; -import { GenericRecord, type UnknownRecord } from './dynamic-record.ts'; -import { DynamicArray } from './dynamic-array.ts'; +import type { UnknownRecord } from './dynamic-record.ts'; +import { BaseType } from './dynamic-base-types.ts'; export { + hashDynamic, + hashArray, hashString, packStringToField, packToField, @@ -23,6 +34,82 @@ export { // compatible hashing +type HashableValue = + | undefined + | string + | number + | boolean + | bigint + | HashableValue[] + | { [key in string]: HashableValue }; + +function hashDynamic(value: HashableValue) { + if (typeof value === 'string') return hashString(value); + if (typeof value === 'number') return packToField(UInt64.from(value), UInt64); + if (typeof value === 'boolean') return packToField(Bool(value), Bool); + if (typeof value === 'bigint') return packToField(Field(value), Field); + if (Array.isArray(value)) return hashArray(value); + return hashRecord(value); +} + +const simpleTypes = new Set(['number', 'boolean', 'bigint', 'undefined']); + +function isSimple(value: HashableValue) { + return simpleTypes.has(typeof value); +} + +function provableTypeOf(value: HashableValue): ProvableHashableType { + if (value === undefined) return Undefined; + if (typeof value === 'string') { + return BaseType.DynamicString({ maxLength: value.length }); + } + if (typeof value === 'number') return UInt64; + if (typeof value === 'boolean') return Bool; + if (typeof value === 'bigint') return Field; + if (Array.isArray(value)) { + return BaseType.DynamicArray(innerArrayType(value), { + maxLength: value.length, + }); + } + return BaseType.DynamicRecord({}, { maxEntries: Object.keys(value).length }); +} + +function provableTypeEquals( + value: HashableValue, + type: ProvableHashableType +): boolean { + if (isSimple(value)) return provableTypeOf(value) === type; + if (typeof value === 'string') { + return isSubclass(type, BaseType.DynamicString.Base); + } + if (Array.isArray(value)) { + if (!isSubclass(type, BaseType.DynamicArray.Base)) return false; + let innerType = type.prototype.innerType; + return value.every((v) => provableTypeEquals(v, innerType)); + } + return isSubclass(type, BaseType.GenericRecord.Base); +} + +function innerArrayType(array: HashableValue[]): ProvableHashableType { + let type = provableTypeOf(array[0]); + assert( + array.every((v) => { + console.log(v, type, provableTypeOf(v)); + return provableTypeEquals(v, type); + }), + 'Array elements must be homogenous' + ); + return type; +} + +function hashArray(array: HashableValue[]) { + let type = innerArrayType(array); + let Array = BaseType.DynamicArray(type, { maxLength: array.length }); + let as = Array.from(array); + console.log(as); + return as.hash(); +} + const enc = new TextEncoder(); function hashString(string: string) { @@ -34,7 +121,6 @@ function hashString(string: string) { bytes.set(stringBytes, 4); let B = Bytes(4 + length); let fields = toFieldsPacked(B, B.from(bytes)); - // console.log({ hashString: fields.map((x) => x.toBigInt()), bytes }); return Poseidon.hash(fields); } @@ -47,23 +133,25 @@ function packStringToField(string: string) { } function packToField(value: T, type?: ProvableType): Field { + // dynamic array types + if (value instanceof BaseType.DynamicArray.Base) { + return value.hash(); + } + type ??= NestedProvable.get(NestedProvable.fromValue(value)); // record types - if (isStruct(type) || value instanceof GenericRecord.Base) { + if (isStruct(type) || value instanceof BaseType.GenericRecord.Base) { return hashRecord(value); } - // dynamic array types - if (value instanceof DynamicArray.Base) { - return value.hash(); - } + let fields = toFieldsPacked(type, value); if (fields.length === 1) return fields[0]!; return Poseidon.hash(fields); } function hashRecord(data: unknown) { - if (data instanceof GenericRecord.Base) return data.hash(); + if (data instanceof BaseType.GenericRecord.Base) return data.hash(); assert( typeof data === 'object' && data !== null, 'Expected DynamicRecord or plain object as data' From cedf584abaf7da527b13eb47644a472671262ec0 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 18:17:22 +0100 Subject: [PATCH 20/44] more dynamic hashing --- src/credentials/dynamic-hash.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index 49960b4..4478b97 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -54,7 +54,9 @@ function hashDynamic(value: HashableValue) { const simpleTypes = new Set(['number', 'boolean', 'bigint', 'undefined']); -function isSimple(value: HashableValue) { +function isSimple( + value: unknown +): value is number | boolean | bigint | undefined { return simpleTypes.has(typeof value); } @@ -94,7 +96,6 @@ function innerArrayType(array: HashableValue[]): ProvableHashableType { let type = provableTypeOf(array[0]); assert( array.every((v) => { - console.log(v, type, provableTypeOf(v)); return provableTypeEquals(v, type); }), 'Array elements must be homogenous' @@ -106,7 +107,8 @@ function hashArray(array: HashableValue[]) { let type = innerArrayType(array); let Array = BaseType.DynamicArray(type, { maxLength: array.length }); let as = Array.from(array); - console.log(as); + // TODO remove + console.dir(as, { depth: 4 }); return as.hash(); } @@ -133,6 +135,11 @@ function packStringToField(string: string) { } function packToField(value: T, type?: ProvableType): Field { + // hashable values + if (isSimple(value) || typeof value === 'string' || Array.isArray(value)) { + return hashDynamic(value); + } + // dynamic array types if (value instanceof BaseType.DynamicArray.Base) { return value.hash(); From 20c049ebdd7316aedb336628241b7ec92750d353 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Nov 2024 18:17:36 +0100 Subject: [PATCH 21/44] failing test --- src/credentials/dynamic-hash.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 9719767..7c20585 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -1,6 +1,7 @@ import { DynamicArray } from './dynamic-array.ts'; import { DynamicString } from './dynamic-string.ts'; -import { hashString, packToField } from './dynamic-hash.ts'; +import './dynamic-record.ts'; +import { hashDynamic, hashString } from './dynamic-hash.ts'; import { test } from 'node:test'; import * as nodeAssert from 'node:assert'; @@ -36,15 +37,16 @@ test('hash strings', () => { }); let ShortArray = DynamicArray(ShortString, { maxLength: 5 }); +let LongArray = DynamicArray(LongString, { maxLength: 5 }); test('hash arrays', () => { - let shortArrayHash = packToField([shortString, shortString]); + let shortArrayHash = hashDynamic([shortString, shortString]); ShortArray.from([shortString, shortString]) .hash() .assertEquals(shortArrayHash, 'hash mismatch (short array)'); - let longArrayHash = packToField([longString, longString]); - ShortArray.from([longString, longString]) + let longArrayHash = hashDynamic([longString, longString]); + LongArray.from([longString, longString]) .hash() .assertEquals(longArrayHash, 'hash mismatch (long array)'); }); From 61e227fcec5da5a75763f8d151bf08fbd747be31 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 13:33:42 +0100 Subject: [PATCH 22/44] fix a few issues, make one test work --- src/credentials/dynamic-array.ts | 39 ++++++++++++++++---------------- src/credentials/dynamic-hash.ts | 15 +++++++----- src/o1js-missing.ts | 2 ++ src/util.ts | 13 +++++++++++ 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index ad58e8b..8c929cc 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -13,9 +13,16 @@ import { type IsPure, Poseidon, } from 'o1js'; -import { assert, assertHasProperty, chunk, fill, pad, zip } from '../util.ts'; import { - HashInput, + assert, + assertHasProperty, + chunk, + defined, + fill, + pad, + zip, +} from '../util.ts'; +import { type ProvableHashablePure, type ProvableHashableType, ProvableType, @@ -342,7 +349,9 @@ class DynamicArrayBase { let secondHalf = block.array .slice(elementsPerHalfBlock) .map((el) => packToField(el, type)); - if (fullField) return [firstHalf[0]!, secondHalf[1]!]; + if (fullField) { + return [defined(firstHalf[0]), defined(secondHalf[0])]; + } return [pack(firstHalf, elementSize), pack(secondHalf, elementSize)]; } ); @@ -496,7 +505,7 @@ function provable>( Class: Class ) { let maxLength = Class.maxLength; - let NULL = type.toValue(ProvableType.synthesize(type)); + let NULL = ProvableType.synthesize(type); return ( TypeBuilder.shape({ @@ -521,26 +530,16 @@ function provable>( if (array.maxLength === maxLength) return array; array = array.toValue(); } - let padded = pad(array, maxLength, NULL); - return { array: padded, length: BigInt(array.length) }; + // fully convert back so that we can pad with NULL + let converted = array.map((x) => type.fromValue(x)); + let padded = pad(converted, maxLength, NULL); + return new Class(padded, Field(array.length)); }, }) // custom hash input - .hashInput(({ array, length }) => { - let lengthInput: HashInput = { packed: [[length, 32]] }; - let arrayInput = array.map((x): HashInput => { - let { fields = [], packed = [] } = type.toInput(x); - return { - packed: fields - .map((x) => [x, 254] as [Field, number]) - .concat(packed), - }; - }); - return [lengthInput, ...arrayInput].reduce( - HashInput.append, - HashInput.empty - ); + .hashInput((array) => { + return { fields: [array.hash()] }; }) ); } diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index 4478b97..f10282b 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -16,7 +16,13 @@ import { ProvableType, toFieldsPacked, } from '../o1js-missing.ts'; -import { assert, hasProperty, isSubclass, mapEntries } from '../util.ts'; +import { + assert, + hasProperty, + isSubclass, + mapEntries, + stringLength, +} from '../util.ts'; import { NestedProvable } from '../nested.ts'; import type { UnknownRecord } from './dynamic-record.ts'; import { BaseType } from './dynamic-base-types.ts'; @@ -63,7 +69,7 @@ function isSimple( function provableTypeOf(value: HashableValue): ProvableHashableType { if (value === undefined) return Undefined; if (typeof value === 'string') { - return BaseType.DynamicString({ maxLength: value.length }); + return BaseType.DynamicString({ maxLength: stringLength(value) }); } if (typeof value === 'number') return UInt64; if (typeof value === 'boolean') return Bool; @@ -106,10 +112,7 @@ function innerArrayType(array: HashableValue[]): ProvableHashableType { function hashArray(array: HashableValue[]) { let type = innerArrayType(array); let Array = BaseType.DynamicArray(type, { maxLength: array.length }); - let as = Array.from(array); - // TODO remove - console.dir(as, { depth: 4 }); - return as.hash(); + return Array.from(array).hash(); } const enc = new TextEncoder(); diff --git a/src/o1js-missing.ts b/src/o1js-missing.ts index 0afacb7..2868e75 100644 --- a/src/o1js-missing.ts +++ b/src/o1js-missing.ts @@ -26,6 +26,8 @@ export { array, toFieldsPacked, hashPacked, + empty, + toInput, HashInput, type WithProvable, }; diff --git a/src/util.ts b/src/util.ts index 0e6a230..b38281b 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,6 +1,7 @@ export { assert, assertDefined, + defined, assertHasProperty, hasProperty, assertIsObject, @@ -14,6 +15,7 @@ export { zipObjects, assertExtendsShape, isSubclass, + stringLength, }; function assert( @@ -35,6 +37,11 @@ function assertDefined( } } +function defined(input: T | undefined, message?: string): T { + assertDefined(input, message); + return input; +} + function assertIsObject( obj: unknown, message?: string @@ -151,3 +158,9 @@ function isSubclass>( if (!hasProperty(constructor, 'prototype')) return false; return constructor.prototype instanceof base; } + +let enc = new TextEncoder(); + +function stringLength(str: string): number { + return enc.encode(str).length; +} From 73a079371134bb3c75ba7645fff2b904f08c21da Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 14:06:00 +0100 Subject: [PATCH 23/44] tweak nested provable get type --- src/credential.ts | 2 +- src/nested.ts | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/credential.ts b/src/credential.ts index 419cd85..1fcc6b2 100644 --- a/src/credential.ts +++ b/src/credential.ts @@ -260,7 +260,7 @@ function withOwner(data: DataType) { function HashableCredential( dataType: NestedProvableFor ): ProvableHashable> { - return NestedProvable.get(withOwner(dataType)) as any; + return NestedProvable.get(withOwner(dataType)); } function HashedCredential( diff --git a/src/nested.ts b/src/nested.ts index 5401360..dc31154 100644 --- a/src/nested.ts +++ b/src/nested.ts @@ -2,8 +2,18 @@ * Allows us to represent nested Provable types, to save us from always having to * wrap types in `Struct` and similar. */ -import { type InferProvable, Provable, type ProvablePure, Struct } from 'o1js'; -import { array, type ProvablePureType, ProvableType } from './o1js-missing.ts'; +import { + type InferProvable, + Provable, + type ProvableHashable, + Struct, +} from 'o1js'; +import { + array, + type ProvableHashablePure, + type ProvablePureType, + ProvableType, +} from './o1js-missing.ts'; import { assertIsObject } from './util.ts'; export { NestedProvable }; @@ -15,18 +25,16 @@ export type { InferNestedProvable, }; -// TODO!! NestedProvable should include Hashable type as well - const NestedProvable = { get: ((type: NestedProvableFor): Provable => { return ProvableType.isProvableType(type) ? ProvableType.get(type) : Struct(type); }) as { - (type: NestedProvablePureFor): ProvablePure; - (type: NestedProvableFor): Provable; - (type: NestedProvablePure): ProvablePure; - (type: NestedProvable): Provable; + (type: NestedProvablePureFor): ProvableHashablePure; + (type: NestedProvableFor): ProvableHashable; + (type: NestedProvablePure): ProvableHashablePure; + (type: NestedProvable): ProvableHashable; }, fromValue(value: T): NestedProvableFor { @@ -51,6 +59,8 @@ const NestedProvable = { }, }; +// TODO!! NestedProvable should accurately requre hashable type + type NestedProvable = ProvableType | { [key: string]: NestedProvable }; type NestedProvablePure = | ProvablePureType From 472e2ba8277658ba46ee731739685fbe5303a6cd Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 14:11:11 +0100 Subject: [PATCH 24/44] handle general types in hashArray to fix other test --- src/credentials/dynamic-hash.ts | 47 ++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index f10282b..0bdd81c 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -18,6 +18,7 @@ import { } from '../o1js-missing.ts'; import { assert, + assertIsObject, hasProperty, isSubclass, mapEntries, @@ -58,6 +59,12 @@ function hashDynamic(value: HashableValue) { return hashRecord(value); } +function hashArray(array: unknown[]) { + let type = innerArrayType(array); + let Array = BaseType.DynamicArray(type, { maxLength: array.length }); + return Array.from(array).hash(); +} + const simpleTypes = new Set(['number', 'boolean', 'bigint', 'undefined']); function isSimple( @@ -66,7 +73,7 @@ function isSimple( return simpleTypes.has(typeof value); } -function provableTypeOf(value: HashableValue): ProvableHashableType { +function provableTypeOf(value: unknown): ProvableHashableType { if (value === undefined) return Undefined; if (typeof value === 'string') { return BaseType.DynamicString({ maxLength: stringLength(value) }); @@ -79,11 +86,22 @@ function provableTypeOf(value: HashableValue): ProvableHashableType { maxLength: value.length, }); } - return BaseType.DynamicRecord({}, { maxEntries: Object.keys(value).length }); + if (value instanceof BaseType.GenericRecord.Base) + return ProvableType.fromValue(value); + + let type = NestedProvable.get(NestedProvable.fromValue(value)); + if (isStruct(type)) { + assertIsObject(value); + return BaseType.DynamicRecord( + {}, + { maxEntries: Object.keys(value).length } + ); + } + return type; } function provableTypeEquals( - value: HashableValue, + value: unknown, type: ProvableHashableType ): boolean { if (isSimple(value)) return provableTypeOf(value) === type; @@ -95,10 +113,17 @@ function provableTypeEquals( let innerType = type.prototype.innerType; return value.every((v) => provableTypeEquals(v, innerType)); } - return isSubclass(type, BaseType.GenericRecord.Base); + if (value instanceof BaseType.GenericRecord.Base) + return isSubclass(type, BaseType.GenericRecord.Base); + + let valueType = NestedProvable.get(NestedProvable.fromValue(value)); + if (isStruct(type)) { + return isSubclass(valueType, BaseType.DynamicRecord.Base); + } + return valueType === ProvableType.get(type); } -function innerArrayType(array: HashableValue[]): ProvableHashableType { +function innerArrayType(array: unknown[]): ProvableHashableType { let type = provableTypeOf(array[0]); assert( array.every((v) => { @@ -109,12 +134,6 @@ function innerArrayType(array: HashableValue[]): ProvableHashableType { return type; } -function hashArray(array: HashableValue[]) { - let type = innerArrayType(array); - let Array = BaseType.DynamicArray(type, { maxLength: array.length }); - return Array.from(array).hash(); -} - const enc = new TextEncoder(); function hashString(string: string) { @@ -139,11 +158,14 @@ function packStringToField(string: string) { function packToField(value: T, type?: ProvableType): Field { // hashable values - if (isSimple(value) || typeof value === 'string' || Array.isArray(value)) { + if (isSimple(value) || typeof value === 'string') { return hashDynamic(value); } // dynamic array types + if (Array.isArray(value)) { + return hashArray(value); + } if (value instanceof BaseType.DynamicArray.Base) { return value.hash(); } @@ -167,6 +189,7 @@ function hashRecord(data: unknown) { 'Expected DynamicRecord or plain object as data' ); let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { + // TODO does it have any benefit here to get the type? let type = NestedProvable.get(NestedProvable.fromValue(value)); return [packStringToField(key), packToField(value, type)]; }); From c4dc13c31555d18c1cb35eb7e8f346f9afac7b08 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 15:12:15 +0100 Subject: [PATCH 25/44] test in circuit + analyze constraint efficiency --- src/credentials/dynamic-hash.test.ts | 102 +++++++++++++++++---------- src/credentials/dynamic-string.ts | 2 +- 2 files changed, 66 insertions(+), 38 deletions(-) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 7c20585..115c797 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -4,49 +4,77 @@ import './dynamic-record.ts'; import { hashDynamic, hashString } from './dynamic-hash.ts'; import { test } from 'node:test'; import * as nodeAssert from 'node:assert'; +import { Bytes, MerkleList, Poseidon, Provable, UInt8 } from 'o1js'; let shortString = 'hi'; let ShortString = DynamicString({ maxLength: 5 }); +let shortHash = hashString(shortString); let longString = - 'Poseidon (/pəˈsaɪdən, pɒ-, poʊ-/;[1] Greek: Ποσειδῶν) is one of the Twelve Olympians in ancient Greek religion and mythology,' + - ' presiding over the sea, storms, earthquakes and horses.[2]'; -let LongString = DynamicString({ maxLength: 300 }); - -test('hash strings', () => { - let shortHash = hashString(shortString); - ShortString.from(shortString) - .hash() - .assertEquals(shortHash, 'hash mismatch (short)'); - - let longHash = hashString(longString); - LongString.from(longString) - .hash() - .assertEquals(longHash, 'hash mismatch (long)'); - - // we can even convert the `ShortString` into a `LongString` - LongString.provable - .fromValue(ShortString.from(shortString)) - .hash() - .assertEquals(shortHash, 'hash mismatch (short -> long)'); - - // (the other way round doesn't work because the string is too long) - nodeAssert.throws(() => { - ShortString.provable.fromValue(LongString.from(longString)); - }, /larger than target size/); -}); + 'Poseidon (/pəˈsaɪdən, pɒ-, poʊ-/;[1] Greek: Ποσειδῶν) is one of the Twelve Olympians'; + +let LongString = DynamicString({ maxLength: 100 }); +let longHash = hashString(longString); + +async function main() { + await test('hash strings', () => { + Provable.witness(ShortString, () => shortString) + .hash() + .assertEquals(shortHash, 'hash mismatch (short)'); + + LongString.from(longString) + .hash() + .assertEquals(longHash, 'hash mismatch (long)'); + + // we can even convert the `ShortString` into a `LongString` + Provable.witness(LongString, () => ShortString.from(shortString)) + .hash() + .assertEquals(shortHash, 'hash mismatch (short -> long)'); + + // (the other way round doesn't work because the string is too long) + nodeAssert.throws(() => { + ShortString.from(LongString.from(longString)); + }, /larger than target size/); + }); + + let shortArray = [shortString, shortString]; + let ShortArray = DynamicArray(ShortString, { maxLength: 5 }); + let longArray = Array(8).fill(longString); + let LongArray = DynamicArray(LongString, { maxLength: 10 }); -let ShortArray = DynamicArray(ShortString, { maxLength: 5 }); -let LongArray = DynamicArray(LongString, { maxLength: 5 }); + let shortArrayHash = hashDynamic(shortArray); + let longArrayHash = hashDynamic(longArray); + + await test('hash arrays of strings', () => { + Provable.witness(ShortArray, () => [shortString, shortString]) + .hash() + .assertEquals(shortArrayHash, 'hash mismatch (short array)'); + + Provable.witness(LongArray, () => Array(8).fill(longString)) + .hash() + .assertEquals(longArrayHash, 'hash mismatch (long array)'); + }); +} + +await test('outside circuit', () => main()); +await test('inside circuit', () => Provable.runAndCheck(main)); + +// comparison of constraint efficiency of different approaches + +let cs = await Provable.constraintSystem(() => { + Provable.witness(LongString, () => longString).hash(); +}); +console.log('constraints: string hash (100)', cs.rows); -test('hash arrays', () => { - let shortArrayHash = hashDynamic([shortString, shortString]); - ShortArray.from([shortString, shortString]) - .hash() - .assertEquals(shortArrayHash, 'hash mismatch (short array)'); +// merkle list of characters +// list is represented as a single hash, so the equivalent of hashing is unpacking the entire list +let CharList = MerkleList.create(UInt8, (hash, { value }) => + Poseidon.hash([hash, value]) +); - let longArrayHash = hashDynamic([longString, longString]); - LongArray.from([longString, longString]) - .hash() - .assertEquals(longArrayHash, 'hash mismatch (long array)'); +let cs2 = await Provable.constraintSystem(() => { + Provable.witness(CharList, () => + CharList.from(Bytes.fromString(longString).bytes) + ).forEach(100, (_item, _isDummy) => {}); }); +console.log('constraints: merkle list of chars (100)', cs2.rows); diff --git a/src/credentials/dynamic-string.ts b/src/credentials/dynamic-string.ts index 9f4fb67..d8f2d59 100644 --- a/src/credentials/dynamic-string.ts +++ b/src/credentials/dynamic-string.ts @@ -34,7 +34,7 @@ function DynamicString({ maxLength }: { maxLength: number }) { /** * Create DynamicBytes from a string. */ - static from(s: string) { + static from(s: string | DynamicStringBase) { return provableString.fromValue(s); } } From 4e8ea3f0602cd1bb736d0cee8602bcc4284f0e06 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 15:28:36 +0100 Subject: [PATCH 26/44] some test cleanup --- src/credentials/dynamic-hash.test.ts | 24 +++++++++++------------- src/credentials/dynamic-string.ts | 13 ++++++++++++- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 115c797..80a26f0 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -18,20 +18,18 @@ let longHash = hashString(longString); async function main() { await test('hash strings', () => { - Provable.witness(ShortString, () => shortString) - .hash() - .assertEquals(shortHash, 'hash mismatch (short)'); + let shortStringVar = Provable.witness(ShortString, () => shortString); + shortStringVar.hash().assertEquals(shortHash, 'short string'); - LongString.from(longString) - .hash() - .assertEquals(longHash, 'hash mismatch (long)'); + let longStringVar = Provable.witness(LongString, () => longString); + longStringVar.hash().assertEquals(longHash, 'long string'); // we can even convert the `ShortString` into a `LongString` - Provable.witness(LongString, () => ShortString.from(shortString)) + LongString.from(shortStringVar) .hash() - .assertEquals(shortHash, 'hash mismatch (short -> long)'); + .assertEquals(shortHash, 'short -> long string'); - // (the other way round doesn't work because the string is too long) + // the other way round doesn't work because the string is too long nodeAssert.throws(() => { ShortString.from(LongString.from(longString)); }, /larger than target size/); @@ -46,13 +44,13 @@ async function main() { let longArrayHash = hashDynamic(longArray); await test('hash arrays of strings', () => { - Provable.witness(ShortArray, () => [shortString, shortString]) + Provable.witness(ShortArray, () => shortArray) .hash() - .assertEquals(shortArrayHash, 'hash mismatch (short array)'); + .assertEquals(shortArrayHash, 'short array'); - Provable.witness(LongArray, () => Array(8).fill(longString)) + Provable.witness(LongArray, () => longArray) .hash() - .assertEquals(longArrayHash, 'hash mismatch (long array)'); + .assertEquals(longArrayHash, 'long array'); }); } diff --git a/src/credentials/dynamic-string.ts b/src/credentials/dynamic-string.ts index d8f2d59..7867223 100644 --- a/src/credentials/dynamic-string.ts +++ b/src/credentials/dynamic-string.ts @@ -1,7 +1,7 @@ import { Bool, Field, type ProvableHashable, UInt8 } from 'o1js'; import { DynamicArrayBase, provableDynamicArray } from './dynamic-array.ts'; import { ProvableFactory } from '../provable-factory.ts'; -import { assert } from '../util.ts'; +import { assert, pad } from '../util.ts'; import { BaseType } from './dynamic-base-types.ts'; export { DynamicString }; @@ -52,6 +52,8 @@ function DynamicString({ maxLength }: { maxLength: number }) { // gracefully handle different maxLength if (s instanceof DynamicStringBase) { if (s.maxLength === maxLength) return s; + if (s.maxLength < maxLength) return s.growMaxLengthTo(maxLength); + // shrinking max length will only work outside circuit s = s.toString(); } return [...enc.encode(s)].map((t) => ({ value: BigInt(t) })); @@ -77,6 +79,15 @@ class DynamicStringBase extends DynamicArrayBase { toString() { return this.toValue() as any as string; } + + growMaxLengthTo(maxLength: number): DynamicStringBase { + assert( + maxLength >= this.maxLength, + 'new maxLength must be greater or equal' + ); + let array = pad(this.array, maxLength, UInt8.from(0)); + return new (DynamicString({ maxLength }))(array, this.length); + } } DynamicString.Base = DynamicStringBase; From 53fc580106796d760f446fd16d999d86ad21d7f7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 16:03:27 +0100 Subject: [PATCH 27/44] add more tests, and separate packing --- src/credentials/dynamic-hash.test.ts | 31 ++++++++++++++-- src/credentials/dynamic-hash.ts | 53 +++++++++++++++++----------- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 80a26f0..a4f577a 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -1,10 +1,11 @@ import { DynamicArray } from './dynamic-array.ts'; import { DynamicString } from './dynamic-string.ts'; import './dynamic-record.ts'; -import { hashDynamic, hashString } from './dynamic-hash.ts'; +import { hashDynamic, hashString, packDynamic } from './dynamic-hash.ts'; import { test } from 'node:test'; import * as nodeAssert from 'node:assert'; -import { Bytes, MerkleList, Poseidon, Provable, UInt8 } from 'o1js'; +import { Bytes, Field, MerkleList, Poseidon, Provable, UInt8 } from 'o1js'; +import { DynamicRecord } from './dynamic-record.ts'; let shortString = 'hi'; let ShortString = DynamicString({ maxLength: 5 }); @@ -35,6 +36,7 @@ async function main() { }, /larger than target size/); }); + // arrays of strings let shortArray = [shortString, shortString]; let ShortArray = DynamicArray(ShortString, { maxLength: 5 }); let longArray = Array(8).fill(longString); @@ -52,6 +54,31 @@ async function main() { .hash() .assertEquals(longArrayHash, 'long array'); }); + + // single-field values + await test('plain values', () => { + // stay the same when packing + packDynamic(-1n).assertEquals(Field(-1n), 'pack bigint'); + packDynamic(true).assertEquals(Field(1), 'pack boolean'); + packDynamic(123).assertEquals(Field(123), 'pack number'); + + // hash is plain poseidon hash + hashDynamic(-1n).assertEquals(Poseidon.hash([Field(-1n)]), 'hash bigint'); + hashDynamic(true).assertEquals(Poseidon.hash([Field(1)]), 'hash boolean'); + hashDynamic(123).assertEquals(Poseidon.hash([Field(123)]), 'hash number'); + }); + + // records of plain values + let record = { a: shortString, b: 1, c: true, d: -1n }; + let recordHash = hashDynamic(record); + + let Record = DynamicRecord({}, { maxEntries: 5 }); + + await test('hash records', () => { + Provable.witness(Record, () => record) + .hash() + .assertEquals(recordHash, 'record'); + }); } await test('outside circuit', () => main()); diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index 0bdd81c..42b2ca3 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -30,6 +30,7 @@ import { BaseType } from './dynamic-base-types.ts'; export { hashDynamic, + packDynamic, hashArray, hashString, packStringToField, @@ -51,10 +52,16 @@ type HashableValue = | { [key in string]: HashableValue }; function hashDynamic(value: HashableValue) { + return packDynamic(value, { mustHash: true }); +} + +function packDynamic(value: HashableValue, config?: { mustHash: boolean }) { if (typeof value === 'string') return hashString(value); - if (typeof value === 'number') return packToField(UInt64.from(value), UInt64); - if (typeof value === 'boolean') return packToField(Bool(value), Bool); - if (typeof value === 'bigint') return packToField(Field(value), Field); + if (typeof value === 'number') + return packToField(UInt64.from(value), UInt64, config); + if (typeof value === 'boolean') return packToField(Bool(value), Bool, config); + if (typeof value === 'bigint') + return packToField(Field(value), Field, config); if (Array.isArray(value)) return hashArray(value); return hashRecord(value); } @@ -148,18 +155,14 @@ function hashString(string: string) { return Poseidon.hash(fields); } -function packStringToField(string: string) { - let bytes = enc.encode(string); - let B = Bytes(bytes.length); - let fields = toFieldsPacked(B, B.from(bytes)); - if (fields.length === 1) return fields[0]!; - return Poseidon.hash(fields); -} - -function packToField(value: T, type?: ProvableType): Field { +function packToField( + value: T, + type?: ProvableType, + config?: { mustHash: boolean } +): Field { // hashable values if (isSimple(value) || typeof value === 'string') { - return hashDynamic(value); + return packDynamic(value, config); } // dynamic array types @@ -170,15 +173,18 @@ function packToField(value: T, type?: ProvableType): Field { return value.hash(); } - type ??= NestedProvable.get(NestedProvable.fromValue(value)); - // record types - if (isStruct(type) || value instanceof BaseType.GenericRecord.Base) { + if (value instanceof BaseType.GenericRecord.Base) { + return hashRecord(value); + } + + type ??= NestedProvable.get(NestedProvable.fromValue(value)); + if (isStruct(type)) { return hashRecord(value); } let fields = toFieldsPacked(type, value); - if (fields.length === 1) return fields[0]!; + if (fields.length === 1 && !config?.mustHash) return fields[0]!; return Poseidon.hash(fields); } @@ -189,13 +195,20 @@ function hashRecord(data: unknown) { 'Expected DynamicRecord or plain object as data' ); let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { - // TODO does it have any benefit here to get the type? - let type = NestedProvable.get(NestedProvable.fromValue(value)); - return [packStringToField(key), packToField(value, type)]; + return [packStringToField(key), packToField(value)]; }); return Poseidon.hash(entryHashes.flat()); } +// for packing keys -- not compatible with dynamic string hash! (as keys will be known at compile time) +function packStringToField(string: string) { + let bytes = enc.encode(string); + let B = Bytes(bytes.length); + let fields = toFieldsPacked(B, B.from(bytes)); + if (fields.length === 1) return fields[0]!; + return Poseidon.hash(fields); +} + // helpers function isStruct(type: ProvableType): type is Struct { From c613ab6e50998cfad91b9bfb0e0b06f805df176b Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 16:34:26 +0100 Subject: [PATCH 28/44] add failing test --- src/credentials/dynamic-hash.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index a4f577a..4cf2b44 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -79,6 +79,18 @@ async function main() { .hash() .assertEquals(recordHash, 'record'); }); + + // arrays of records + let array = [record, record, record]; + let arrayHash = hashDynamic(array); + + let RecordArray = DynamicArray(Record, { maxLength: 5 }); + + // await test('hash arrays of records', () => { + Provable.witness(RecordArray, () => array) + .hash() + .assertEquals(arrayHash, 'array'); + // }); } await test('outside circuit', () => main()); From 13a895af860a45fdbedefc63fd1d5a7f3adf75f4 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 17:23:49 +0100 Subject: [PATCH 29/44] support arrays of records of plain values --- src/credentials/dynamic-hash.ts | 69 ++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index 42b2ca3..9625cda 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -96,15 +96,28 @@ function provableTypeOf(value: unknown): ProvableHashableType { if (value instanceof BaseType.GenericRecord.Base) return ProvableType.fromValue(value); - let type = NestedProvable.get(NestedProvable.fromValue(value)); - if (isStruct(type)) { - assertIsObject(value); - return BaseType.DynamicRecord( - {}, - { maxEntries: Object.keys(value).length } - ); - } - return type; + // now let's simply try to get the type from the value + try { + // this may throw, in that case we continue below + let type = ProvableType.fromValue(value); + + // handle structs as dynamic records + if (isStruct(type)) { + assertIsObject(value); + let length = Object.keys(value).length; + return BaseType.DynamicRecord({}, { maxEntries: length }); + } + + // other types use directly + return type; + } catch {} + + // at this point, the only valid types are records + assert( + typeof value === 'object' && value !== null, + `Failed to get type for value ${value}` + ); + return BaseType.DynamicRecord({}, { maxEntries: Object.keys(value).length }); } function provableTypeEquals( @@ -121,13 +134,39 @@ function provableTypeEquals( return value.every((v) => provableTypeEquals(v, innerType)); } if (value instanceof BaseType.GenericRecord.Base) - return isSubclass(type, BaseType.GenericRecord.Base); + return ( + isSubclass(type, BaseType.GenericRecord.Base) && + value.maxEntries <= type.prototype.maxEntries + ); - let valueType = NestedProvable.get(NestedProvable.fromValue(value)); - if (isStruct(type)) { - return isSubclass(valueType, BaseType.DynamicRecord.Base); - } - return valueType === ProvableType.get(type); + try { + // this may throw, in that case we continue below + let valueType = ProvableType.fromValue(value); + + // handle structs as dynamic records + if (isStruct(valueType)) { + assertIsObject(value); + let length = Object.keys(value).length; + return ( + isSubclass(type, BaseType.DynamicRecord.Base) && + length <= type.prototype.maxEntries + ); + } + + // other types check directly + return valueType === ProvableType.get(type); + } catch {} + + // at this point, the only valid types are records + assert( + typeof value === 'object' && value !== null, + `Failed to get type for value ${value}` + ); + let length = Object.keys(value).length; + return ( + isSubclass(type, BaseType.DynamicRecord.Base) && + length <= type.prototype.maxEntries + ); } function innerArrayType(array: unknown[]): ProvableHashableType { From 3af1a0854e39b1e27586221dc31213afcdf72696 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 17:24:04 +0100 Subject: [PATCH 30/44] fix test --- src/credentials/dynamic-array.ts | 11 +++-------- src/credentials/dynamic-hash.test.ts | 10 +++++----- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index 8c929cc..1d84be8 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -313,14 +313,12 @@ class DynamicArrayBase { // create blocks of 2 field elements each // TODO abstract this into a `chunk()` method that returns a DynamicArray> - let mustPack = packedFieldSize(type) > 1; let elementSize = bitSize(type); let elementsPerHalfBlock = Math.floor(254 / elementSize); let fullField = elementsPerHalfBlock === 0; if (elementsPerHalfBlock === 0) elementsPerHalfBlock = 1; // larger types are compressed let elementsPerBlock = 2 * elementsPerHalfBlock; - assert(!mustPack, 'TODO'); // this should get a separate branch here // we pack the length at the beginning of the first block // for efficiency (to avoid unpacking the length), we first put zeros at the beginning @@ -343,12 +341,9 @@ class DynamicArrayBase { let blocks = new Blocks(chunked, nBlocks).map( StaticArray(Field, 2), (block) => { - let firstHalf = block.array - .slice(0, elementsPerHalfBlock) - .map((el) => packToField(el, type)); - let secondHalf = block.array - .slice(elementsPerHalfBlock) - .map((el) => packToField(el, type)); + let fields = block.array.map((el) => packToField(el, type)); + let firstHalf = fields.slice(0, elementsPerHalfBlock); + let secondHalf = fields.slice(elementsPerHalfBlock); if (fullField) { return [defined(firstHalf[0]), defined(secondHalf[0])]; } diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 4cf2b44..8f581bc 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -86,11 +86,11 @@ async function main() { let RecordArray = DynamicArray(Record, { maxLength: 5 }); - // await test('hash arrays of records', () => { - Provable.witness(RecordArray, () => array) - .hash() - .assertEquals(arrayHash, 'array'); - // }); + await test('hash arrays of records', () => { + Provable.witness(RecordArray, () => array) + .hash() + .assertEquals(arrayHash, 'array'); + }); } await test('outside circuit', () => main()); From 4ffd49240225cd3343f80256c138b83bb8d33367 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 17:50:42 +0100 Subject: [PATCH 31/44] more cleanup --- src/credentials/dynamic-array.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index 1d84be8..d2d9317 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -315,7 +315,6 @@ class DynamicArrayBase { // TODO abstract this into a `chunk()` method that returns a DynamicArray> let elementSize = bitSize(type); let elementsPerHalfBlock = Math.floor(254 / elementSize); - let fullField = elementsPerHalfBlock === 0; if (elementsPerHalfBlock === 0) elementsPerHalfBlock = 1; // larger types are compressed let elementsPerBlock = 2 * elementsPerHalfBlock; @@ -344,9 +343,6 @@ class DynamicArrayBase { let fields = block.array.map((el) => packToField(el, type)); let firstHalf = fields.slice(0, elementsPerHalfBlock); let secondHalf = fields.slice(elementsPerHalfBlock); - if (fullField) { - return [defined(firstHalf[0]), defined(secondHalf[0])]; - } return [pack(firstHalf, elementSize), pack(secondHalf, elementSize)]; } ); @@ -355,15 +351,6 @@ class DynamicArrayBase { let firstBlock = blocks.array[0]!; firstBlock.set(0, firstBlock.get(0).add(this.length).seal()); - // TODO remove - // Provable.log({ - // elementsPerUint32, - // elementSize, - // elementsPerBlock, - // maxBlocks, - // hash: blocks.array.flatMap((x) => x.array), - // }); - // now hash the 2-field elements blocks, on permutation at a time // TODO: first we hash the length, but this should be included in the rest let state = Poseidon.initialState(); From 92670e5595e9dd8602aadbea1ebfe342cb6a6d86 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 21:06:36 +0100 Subject: [PATCH 32/44] simplify --- src/credentials/dynamic-array.ts | 45 ++++++++++++++++---------------- src/credentials/gadgets.ts | 2 +- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index d2d9317..b566599 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -305,10 +305,13 @@ class DynamicArrayBase { hash() { let type = ProvableType.get(this.innerType); + // pack all elements into a single field element + let fields = this.array.map((x) => packToField(x, type)); + let NULL = packToField(ProvableType.synthesize(type), type); + // assert that all padding elements are 0. this allows us to pack values into blocks - let NULL = ProvableType.synthesize(type); - this.forEach((x, isPadding) => { - Provable.assertEqualIf(isPadding, this.innerType, x, NULL); + zip(fields, this._dummyMask()).forEach(([x, isPadding]) => { + Provable.assertEqualIf(isPadding, Field, x, NULL); }); // create blocks of 2 field elements each @@ -323,38 +326,34 @@ class DynamicArrayBase { // for efficiency (to avoid unpacking the length), we first put zeros at the beginning // and later just add the length to the first block let elementsPerUint32 = Math.max(Math.floor(32 / elementSize), 1); - let array = fill(elementsPerUint32, NULL).concat(this.array); + let array = fill(elementsPerUint32, Field(0)).concat(fields); - let Block = StaticArray(type, elementsPerBlock); let maxBlocks = Math.ceil( (elementsPerUint32 + this.maxLength) / elementsPerBlock ); - let Blocks = DynamicArray(Block, { maxLength: maxBlocks }); + let padded = pad(array, maxBlocks * elementsPerBlock, NULL); + let chunked = chunk(padded, elementsPerBlock); + let blocks = chunked.map((block): [Field, Field] => { + let firstHalf = block.slice(0, elementsPerHalfBlock); + let secondHalf = block.slice(elementsPerHalfBlock); + return [pack(firstHalf, elementSize), pack(secondHalf, elementSize)]; + }); + + // add length to the first block + let firstBlock = blocks[0]!; + firstBlock[0] = firstBlock[0].add(this.length).seal(); // nBlocks = ceil(length / elementsPerBlock) = floor((length + elementsPerBlock - 1) / elementsPerBlock) + let Fieldx2 = StaticArray(Field, 2); + let Blocks = DynamicArray(Fieldx2, { maxLength: maxBlocks }); let nBlocks = UInt32.Unsafe.fromField( this.length.add(elementsPerUint32 + elementsPerBlock - 1) ).div(elementsPerBlock).value; - let padded = pad(array, maxBlocks * elementsPerBlock, NULL); - let chunked = chunk(padded, elementsPerBlock).map(Block.from); - let blocks = new Blocks(chunked, nBlocks).map( - StaticArray(Field, 2), - (block) => { - let fields = block.array.map((el) => packToField(el, type)); - let firstHalf = fields.slice(0, elementsPerHalfBlock); - let secondHalf = fields.slice(elementsPerHalfBlock); - return [pack(firstHalf, elementSize), pack(secondHalf, elementSize)]; - } - ); - - // add length to the first block - let firstBlock = blocks.array[0]!; - firstBlock.set(0, firstBlock.get(0).add(this.length).seal()); + let dynBlocks = new Blocks(blocks.map(Fieldx2.from), nBlocks); // now hash the 2-field elements blocks, on permutation at a time - // TODO: first we hash the length, but this should be included in the rest let state = Poseidon.initialState(); - blocks.forEach((block, isPadding) => { + dynBlocks.forEach((block, isPadding) => { let newState = Poseidon.update(state, block.array); state[0] = Provable.if(isPadding, state[0], newState[0]); state[1] = Provable.if(isPadding, state[1], newState[1]); diff --git a/src/credentials/gadgets.ts b/src/credentials/gadgets.ts index de4b3db..547732c 100644 --- a/src/credentials/gadgets.ts +++ b/src/credentials/gadgets.ts @@ -15,7 +15,7 @@ export { pack, unsafeIf, seal, lessThan16, assertInRange16, assertLessThan16 }; function pack(chunks: Field[], chunkSize: number) { let p = chunks.length * chunkSize; assert( - p < Field.sizeInBits, + chunks.length <= 1 || p < Field.sizeInBits, () => `pack(): too many chunks, got ${chunks.length} * ${chunkSize} = ${p}` ); let sum = Field(0); From d7d02ebc8c4ee5c2e7e88dadf78ef447a9a415b2 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 21:14:00 +0100 Subject: [PATCH 33/44] move comment --- src/credentials/dynamic-array.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index b566599..142af94 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -343,9 +343,9 @@ class DynamicArrayBase { let firstBlock = blocks[0]!; firstBlock[0] = firstBlock[0].add(this.length).seal(); - // nBlocks = ceil(length / elementsPerBlock) = floor((length + elementsPerBlock - 1) / elementsPerBlock) let Fieldx2 = StaticArray(Field, 2); let Blocks = DynamicArray(Fieldx2, { maxLength: maxBlocks }); + // nBlocks = ceil(length / elementsPerBlock) = floor((length + elementsPerBlock - 1) / elementsPerBlock) let nBlocks = UInt32.Unsafe.fromField( this.length.add(elementsPerUint32 + elementsPerBlock - 1) ).div(elementsPerBlock).value; From 095e95704860374f7161028a936ad97968dd700d Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 21:36:21 +0100 Subject: [PATCH 34/44] consolidate packing methods --- src/credentials/dynamic-hash.test.ts | 9 +- src/credentials/dynamic-hash.ts | 153 ++++++++++++++------------- 2 files changed, 83 insertions(+), 79 deletions(-) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 8f581bc..75773b2 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -1,7 +1,7 @@ import { DynamicArray } from './dynamic-array.ts'; import { DynamicString } from './dynamic-string.ts'; import './dynamic-record.ts'; -import { hashDynamic, hashString, packDynamic } from './dynamic-hash.ts'; +import { hashDynamic, hashString, packToField } from './dynamic-hash.ts'; import { test } from 'node:test'; import * as nodeAssert from 'node:assert'; import { Bytes, Field, MerkleList, Poseidon, Provable, UInt8 } from 'o1js'; @@ -58,9 +58,10 @@ async function main() { // single-field values await test('plain values', () => { // stay the same when packing - packDynamic(-1n).assertEquals(Field(-1n), 'pack bigint'); - packDynamic(true).assertEquals(Field(1), 'pack boolean'); - packDynamic(123).assertEquals(Field(123), 'pack number'); + packToField(-1n).assertEquals(Field(-1n), 'pack bigint'); + packToField(true).assertEquals(Field(1), 'pack boolean'); + packToField(123).assertEquals(Field(123), 'pack number'); + // packDynamic(undefined).assertEquals(Field(0), 'pack undefined'); // hash is plain poseidon hash hashDynamic(-1n).assertEquals(Poseidon.hash([Field(-1n)]), 'hash bigint'); diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index 9625cda..c5bc023 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -24,13 +24,11 @@ import { mapEntries, stringLength, } from '../util.ts'; -import { NestedProvable } from '../nested.ts'; import type { UnknownRecord } from './dynamic-record.ts'; import { BaseType } from './dynamic-base-types.ts'; export { hashDynamic, - packDynamic, hashArray, hashString, packStringToField, @@ -52,17 +50,55 @@ type HashableValue = | { [key in string]: HashableValue }; function hashDynamic(value: HashableValue) { - return packDynamic(value, { mustHash: true }); + return packToField(value, undefined, { mustHash: true }); } -function packDynamic(value: HashableValue, config?: { mustHash: boolean }) { +function packToField( + value: T, + type?: ProvableType, + config?: { mustHash: boolean } +): Field { + // hashable values if (typeof value === 'string') return hashString(value); if (typeof value === 'number') return packToField(UInt64.from(value), UInt64, config); if (typeof value === 'boolean') return packToField(Bool(value), Bool, config); if (typeof value === 'bigint') return packToField(Field(value), Field, config); - if (Array.isArray(value)) return hashArray(value); + + // dynamic array types + if (Array.isArray(value)) { + return hashArray(value); + } + if (value instanceof BaseType.DynamicArray.Base) { + return value.hash(); + } + // dynamic records + if (value instanceof BaseType.GenericRecord.Base) { + return hashRecord(value); + } + + // now let's simply try to get the type from the value + try { + // this may throw, in that case we continue below + type ??= ProvableType.fromValue(value); + + // handle structs as dynamic records + if (isStruct(type)) { + return hashRecord(value); + } + + // other provable types use directly + let fields = toFieldsPacked(type, value); + if (fields.length === 1 && !config?.mustHash) return fields[0]!; + return Poseidon.hash(fields); + } catch {} + + // at this point, the only valid types are records + assert( + typeof value === 'object' && value !== null, + `Failed to get type for value ${value}` + ); return hashRecord(value); } @@ -72,6 +108,32 @@ function hashArray(array: unknown[]) { return Array.from(array).hash(); } +function hashRecord(data: unknown) { + if (data instanceof BaseType.GenericRecord.Base) return data.hash(); + assert( + typeof data === 'object' && data !== null, + 'Expected DynamicRecord or plain object as data' + ); + let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { + return [packStringToField(key), packToField(value)]; + }); + return Poseidon.hash(entryHashes.flat()); +} + +const enc = new TextEncoder(); + +function hashString(string: string) { + // encode length + bytes + let stringBytes = enc.encode(string); + let length = stringBytes.length; + let bytes = new Uint8Array(4 + length); + new DataView(bytes.buffer).setUint32(0, length, true); + bytes.set(stringBytes, 4); + let B = Bytes(4 + length); + let fields = toFieldsPacked(B, B.from(bytes)); + return Poseidon.hash(fields); +} + const simpleTypes = new Set(['number', 'boolean', 'bigint', 'undefined']); function isSimple( @@ -80,6 +142,17 @@ function isSimple( return simpleTypes.has(typeof value); } +function innerArrayType(array: unknown[]): ProvableHashableType { + let type = provableTypeOf(array[0]); + assert( + array.every((v) => { + return provableTypeEquals(v, type); + }), + 'Array elements must be homogenous' + ); + return type; +} + function provableTypeOf(value: unknown): ProvableHashableType { if (value === undefined) return Undefined; if (typeof value === 'string') { @@ -169,76 +242,6 @@ function provableTypeEquals( ); } -function innerArrayType(array: unknown[]): ProvableHashableType { - let type = provableTypeOf(array[0]); - assert( - array.every((v) => { - return provableTypeEquals(v, type); - }), - 'Array elements must be homogenous' - ); - return type; -} - -const enc = new TextEncoder(); - -function hashString(string: string) { - // encode length + bytes - let stringBytes = enc.encode(string); - let length = stringBytes.length; - let bytes = new Uint8Array(4 + length); - new DataView(bytes.buffer).setUint32(0, length, true); - bytes.set(stringBytes, 4); - let B = Bytes(4 + length); - let fields = toFieldsPacked(B, B.from(bytes)); - return Poseidon.hash(fields); -} - -function packToField( - value: T, - type?: ProvableType, - config?: { mustHash: boolean } -): Field { - // hashable values - if (isSimple(value) || typeof value === 'string') { - return packDynamic(value, config); - } - - // dynamic array types - if (Array.isArray(value)) { - return hashArray(value); - } - if (value instanceof BaseType.DynamicArray.Base) { - return value.hash(); - } - - // record types - if (value instanceof BaseType.GenericRecord.Base) { - return hashRecord(value); - } - - type ??= NestedProvable.get(NestedProvable.fromValue(value)); - if (isStruct(type)) { - return hashRecord(value); - } - - let fields = toFieldsPacked(type, value); - if (fields.length === 1 && !config?.mustHash) return fields[0]!; - return Poseidon.hash(fields); -} - -function hashRecord(data: unknown) { - if (data instanceof BaseType.GenericRecord.Base) return data.hash(); - assert( - typeof data === 'object' && data !== null, - 'Expected DynamicRecord or plain object as data' - ); - let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { - return [packStringToField(key), packToField(value)]; - }); - return Poseidon.hash(entryHashes.flat()); -} - // for packing keys -- not compatible with dynamic string hash! (as keys will be known at compile time) function packStringToField(string: string) { let bytes = enc.encode(string); From 34608a1ff5b378ab45af4d25059d39a666ce4455 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 21:48:32 +0100 Subject: [PATCH 35/44] simplify --- src/credential.ts | 4 ++-- src/credentials/dynamic-hash.ts | 10 +++++++--- src/credentials/dynamic-record.test.ts | 9 +++++---- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/credential.ts b/src/credential.ts index 1fcc6b2..5389d63 100644 --- a/src/credential.ts +++ b/src/credential.ts @@ -15,7 +15,7 @@ import { type NestedProvableFor, } from './nested.ts'; import { zip } from './util.ts'; -import { hashRecord } from './credentials/dynamic-hash.ts'; +import { hashDynamic } from './credentials/dynamic-hash.ts'; export { type Credential, @@ -271,6 +271,6 @@ function HashedCredential( function credentialHash({ owner, data }: Credential) { let ownerHash = Poseidon.hash(owner.toFields()); - let dataHash = hashRecord(data); + let dataHash = hashDynamic(data); return Poseidon.hash([ownerHash, dataHash]); } diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index c5bc023..f30ee8e 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -49,7 +49,12 @@ type HashableValue = | HashableValue[] | { [key in string]: HashableValue }; -function hashDynamic(value: HashableValue) { +/** + * Hash an input that is either a simple JSON-with-bigints object or a provable type. + * + * The hashing algorithm is compatible with dynamic-length schemas. + */ +function hashDynamic(value: HashableValue | unknown) { return packToField(value, undefined, { mustHash: true }); } @@ -75,7 +80,7 @@ function packToField( } // dynamic records if (value instanceof BaseType.GenericRecord.Base) { - return hashRecord(value); + return value.hash(); } // now let's simply try to get the type from the value @@ -109,7 +114,6 @@ function hashArray(array: unknown[]) { } function hashRecord(data: unknown) { - if (data instanceof BaseType.GenericRecord.Base) return data.hash(); assert( typeof data === 'object' && data !== null, 'Expected DynamicRecord or plain object as data' diff --git a/src/credentials/dynamic-record.test.ts b/src/credentials/dynamic-record.test.ts index d2b9512..4152134 100644 --- a/src/credentials/dynamic-record.test.ts +++ b/src/credentials/dynamic-record.test.ts @@ -16,7 +16,7 @@ import { test } from 'node:test'; import assert from 'assert'; import { hashCredential } from '../credential.ts'; import { owner } from '../../tests/test-utils.ts'; -import { hashRecord } from './dynamic-hash.ts'; +import { hashDynamic, hashRecord } from './dynamic-hash.ts'; import { array } from '../o1js-missing.ts'; import { DynamicArray } from './dynamic-array.ts'; @@ -121,9 +121,10 @@ async function circuit() { await test('DynamicRecord.hash()', () => record.hash().assertEquals(expectedHash, 'hash')); - await test('hashRecord()', () => { - hashRecord(originalStruct).assertEquals(expectedHash); - hashRecord(record).assertEquals(expectedHash); + await test('hashDynamic()', () => { + hashDynamic(original).assertEquals(expectedHash); + hashDynamic(originalStruct).assertEquals(expectedHash); + hashDynamic(record).assertEquals(expectedHash); }); await test('hashCredential()', () => { From 9d995c4e337413734d198dd1ebb4c6056da00943 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Nov 2024 21:52:54 +0100 Subject: [PATCH 36/44] more tests --- src/credentials/dynamic-hash.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 75773b2..d9e84c5 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -34,6 +34,12 @@ async function main() { nodeAssert.throws(() => { ShortString.from(LongString.from(longString)); }, /larger than target size/); + + // for strings, hashDynamic === packToField === hashString + hashDynamic(shortString).assertEquals(shortHash, 'short string'); + packToField(shortString).assertEquals(shortHash, 'short string'); + hashDynamic(shortStringVar).assertEquals(shortHash, 'short string'); + packToField(shortStringVar).assertEquals(shortHash, 'short string'); }); // arrays of strings @@ -53,6 +59,10 @@ async function main() { Provable.witness(LongArray, () => longArray) .hash() .assertEquals(longArrayHash, 'long array'); + + // for arrays, hashDynamic === packToField === hashArray + hashDynamic(shortArray).assertEquals(shortArrayHash, 'short array'); + packToField(shortArray).assertEquals(shortArrayHash, 'short array'); }); // single-field values @@ -61,12 +71,13 @@ async function main() { packToField(-1n).assertEquals(Field(-1n), 'pack bigint'); packToField(true).assertEquals(Field(1), 'pack boolean'); packToField(123).assertEquals(Field(123), 'pack number'); - // packDynamic(undefined).assertEquals(Field(0), 'pack undefined'); + packToField(undefined).assertEquals(Poseidon.hash([]), 'pack undefined'); // hash is plain poseidon hash hashDynamic(-1n).assertEquals(Poseidon.hash([Field(-1n)]), 'hash bigint'); hashDynamic(true).assertEquals(Poseidon.hash([Field(1)]), 'hash boolean'); hashDynamic(123).assertEquals(Poseidon.hash([Field(123)]), 'hash number'); + hashDynamic(undefined).assertEquals(Poseidon.hash([]), 'pack undefined'); }); // records of plain values From 4554597f1bcdf655d25cf365f7b22d2dc17dd6c7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 7 Nov 2024 10:10:43 +0100 Subject: [PATCH 37/44] a few more tests --- src/credentials/dynamic-hash.test.ts | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index d9e84c5..2bb14c8 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -1,7 +1,13 @@ import { DynamicArray } from './dynamic-array.ts'; import { DynamicString } from './dynamic-string.ts'; import './dynamic-record.ts'; -import { hashDynamic, hashString, packToField } from './dynamic-hash.ts'; +import { + hashArray, + hashDynamic, + hashRecord, + hashString, + packToField, +} from './dynamic-hash.ts'; import { test } from 'node:test'; import * as nodeAssert from 'node:assert'; import { Bytes, Field, MerkleList, Poseidon, Provable, UInt8 } from 'o1js'; @@ -52,17 +58,18 @@ async function main() { let longArrayHash = hashDynamic(longArray); await test('hash arrays of strings', () => { - Provable.witness(ShortArray, () => shortArray) - .hash() - .assertEquals(shortArrayHash, 'short array'); + let shortArrayVar = Provable.witness(ShortArray, () => shortArray); + shortArrayVar.hash().assertEquals(shortArrayHash, 'short array'); Provable.witness(LongArray, () => longArray) .hash() .assertEquals(longArrayHash, 'long array'); // for arrays, hashDynamic === packToField === hashArray - hashDynamic(shortArray).assertEquals(shortArrayHash, 'short array'); + hashArray(shortArray).assertEquals(shortArrayHash, 'short array'); packToField(shortArray).assertEquals(shortArrayHash, 'short array'); + hashDynamic(shortArrayVar).assertEquals(shortArrayHash, 'short array'); + packToField(shortArrayVar).assertEquals(shortArrayHash, 'short array'); }); // single-field values @@ -87,9 +94,11 @@ async function main() { let Record = DynamicRecord({}, { maxEntries: 5 }); await test('hash records', () => { - Provable.witness(Record, () => record) - .hash() - .assertEquals(recordHash, 'record'); + let recordVar = Provable.witness(Record, () => record); + recordVar.hash().assertEquals(recordHash, 'record'); + + packToField(recordVar).assertEquals(recordHash, 'record'); + hashRecord(record).assertEquals(recordHash, 'record'); }); // arrays of records From 8ce8c70f7af259556c6b8c2e79aadbefcbd450e8 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 7 Nov 2024 11:53:45 +0100 Subject: [PATCH 38/44] improve logical flow of dynamic hashing --- src/credentials/dynamic-hash.ts | 177 +++++++++++++++--------------- src/credentials/dynamic-record.ts | 2 +- src/o1js-missing.ts | 19 +++- 3 files changed, 102 insertions(+), 96 deletions(-) diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index f30ee8e..6cef8a3 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -12,13 +12,13 @@ import { Undefined, } from 'o1js'; import { + hashPacked, type ProvableHashableType, ProvableType, toFieldsPacked, } from '../o1js-missing.ts'; import { assert, - assertIsObject, hasProperty, isSubclass, mapEntries, @@ -60,7 +60,7 @@ function hashDynamic(value: HashableValue | unknown) { function packToField( value: T, - type?: ProvableType, + type?: ProvableHashableType, config?: { mustHash: boolean } ): Field { // hashable values @@ -70,6 +70,8 @@ function packToField( if (typeof value === 'boolean') return packToField(Bool(value), Bool, config); if (typeof value === 'bigint') return packToField(Field(value), Field, config); + if (value === undefined || value === null) + return hashPacked(Undefined, undefined); // dynamic array types if (Array.isArray(value)) { @@ -83,27 +85,22 @@ function packToField( return value.hash(); } - // now let's simply try to get the type from the value - try { - // this may throw, in that case we continue below - type ??= ProvableType.fromValue(value); + // now let's try to get the type from the value + type ??= provableTypeOfConstructor(value); + if (type !== undefined) { // handle structs as dynamic records - if (isStruct(type)) { - return hashRecord(value); - } + if (isStruct(type)) return hashRecord(value); // other provable types use directly let fields = toFieldsPacked(type, value); if (fields.length === 1 && !config?.mustHash) return fields[0]!; return Poseidon.hash(fields); - } catch {} + } // at this point, the only valid types are records - assert( - typeof value === 'object' && value !== null, - `Failed to get type for value ${value}` - ); + // functions are a hint that something went wrong, so throw a descriptive error + assert(typeof value === 'object', `Failed to get type for value ${value}`); return hashRecord(value); } @@ -113,11 +110,8 @@ function hashArray(array: unknown[]) { return Array.from(array).hash(); } -function hashRecord(data: unknown) { - assert( - typeof data === 'object' && data !== null, - 'Expected DynamicRecord or plain object as data' - ); +function hashRecord(data: {}) { + assert(typeof data === 'object', 'Expected plain object'); let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { return [packStringToField(key), packToField(value)]; }); @@ -126,6 +120,11 @@ function hashRecord(data: unknown) { const enc = new TextEncoder(); +/** + * Hash a string using Poseidon on packed UInt8s. + * + * Avoids hash collisions by encoding the length of the string at the beginning. + */ function hashString(string: string) { // encode length + bytes let stringBytes = enc.encode(string); @@ -138,112 +137,112 @@ function hashString(string: string) { return Poseidon.hash(fields); } -const simpleTypes = new Set(['number', 'boolean', 'bigint', 'undefined']); - -function isSimple( - value: unknown -): value is number | boolean | bigint | undefined { - return simpleTypes.has(typeof value); -} - -function innerArrayType(array: unknown[]): ProvableHashableType { - let type = provableTypeOf(array[0]); - assert( - array.every((v) => { - return provableTypeEquals(v, type); - }), - 'Array elements must be homogenous' - ); - return type; -} - +/** + * Gets a provable type from any value. + * + * The fallback type for unknown objects is DynamicRecord. + */ function provableTypeOf(value: unknown): ProvableHashableType { - if (value === undefined) return Undefined; if (typeof value === 'string') { return BaseType.DynamicString({ maxLength: stringLength(value) }); } if (typeof value === 'number') return UInt64; if (typeof value === 'boolean') return Bool; if (typeof value === 'bigint') return Field; + if (value === undefined || value === null) return Undefined; if (Array.isArray(value)) { return BaseType.DynamicArray(innerArrayType(value), { maxLength: value.length, }); } - if (value instanceof BaseType.GenericRecord.Base) - return ProvableType.fromValue(value); + let type = provableTypeOfConstructor(value); - // now let's simply try to get the type from the value - try { - // this may throw, in that case we continue below - let type = ProvableType.fromValue(value); + // handle structs and unknown objects as dynamic records + if (type === undefined || isStruct(type)) { + let length = Object.keys(value).length; + return BaseType.DynamicRecord({}, { maxEntries: length }); + } - // handle structs as dynamic records - if (isStruct(type)) { - assertIsObject(value); - let length = Object.keys(value).length; - return BaseType.DynamicRecord({}, { maxEntries: length }); - } + // other types use directly + return type; +} - // other types use directly - return type; - } catch {} +/** + * Gets a provable type from value.constructor, otherwise returns undefined. + */ +function provableTypeOfConstructor( + value: T +): ProvableHashableType | undefined { + if (!hasProperty(value, 'constructor')) return undefined; + + // special checks for Field, Bool because their constructor doesn't match the function that wraps it + if (value instanceof Field) return Field as any; + if (value instanceof Bool) return Bool as any; + + let constructor = value.constructor; + if (!ProvableType.isProvableHashableType(constructor)) return undefined; + return constructor; +} - // at this point, the only valid types are records +/** + * Gets the inner type of an array, asserting that it is unique. + * + * Throws an error for inhomogeneous arrays like [1, 'a']. + * These should be represented as records i.e. { first: 1, second: 'a' }. + */ +function innerArrayType(array: unknown[]): ProvableHashableType { + let type = provableTypeOf(array[0]); assert( - typeof value === 'object' && value !== null, - `Failed to get type for value ${value}` + array.every((v) => provableTypeEquals(v, type)), + 'Array elements must be homogenous' ); - return BaseType.DynamicRecord({}, { maxEntries: Object.keys(value).length }); + return type; } function provableTypeEquals( value: unknown, type: ProvableHashableType ): boolean { - if (isSimple(value)) return provableTypeOf(value) === type; if (typeof value === 'string') { return isSubclass(type, BaseType.DynamicString.Base); } + if (typeof value === 'number') return type === UInt64; + if (typeof value === 'boolean') return type === Bool; + if (typeof value === 'bigint') return type === Field; + if (value === undefined || value === null) return type === Undefined; + if (Array.isArray(value)) { if (!isSubclass(type, BaseType.DynamicArray.Base)) return false; let innerType = type.prototype.innerType; return value.every((v) => provableTypeEquals(v, innerType)); } - if (value instanceof BaseType.GenericRecord.Base) + // dynamic types only have to be compatible + if (value instanceof BaseType.DynamicArray.Base) return ( - isSubclass(type, BaseType.GenericRecord.Base) && - value.maxEntries <= type.prototype.maxEntries + isSubclass(type, BaseType.DynamicArray.Base) && + value.maxLength <= type.prototype.maxLength ); - try { - // this may throw, in that case we continue below - let valueType = ProvableType.fromValue(value); - - // handle structs as dynamic records - if (isStruct(valueType)) { - assertIsObject(value); - let length = Object.keys(value).length; - return ( - isSubclass(type, BaseType.DynamicRecord.Base) && - length <= type.prototype.maxEntries - ); - } - - // other types check directly - return valueType === ProvableType.get(type); - } catch {} + let valueType = provableTypeOfConstructor(value); + + // handle structs and unknown objects as dynamic records + if ( + valueType === undefined || + isStruct(valueType) || + value instanceof BaseType.GenericRecord.Base + ) { + let length = + value instanceof BaseType.GenericRecord.Base + ? value.maxEntries + : Object.keys(value).length; + return ( + isSubclass(type, BaseType.GenericRecord.Base) && + length <= type.prototype.maxEntries + ); + } - // at this point, the only valid types are records - assert( - typeof value === 'object' && value !== null, - `Failed to get type for value ${value}` - ); - let length = Object.keys(value).length; - return ( - isSubclass(type, BaseType.DynamicRecord.Base) && - length <= type.prototype.maxEntries - ); + // other types check directly + return valueType === type; } // for packing keys -- not compatible with dynamic string hash! (as keys will be known at compile time) diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index 5af99b1..07a8f0a 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -160,7 +160,7 @@ class GenericRecordBase { return new this({ entries: options, actual: Unconstrained.from(actual) }); } - getAny(valueType: A, key: string) { + getAny(valueType: A, key: string) { // find valueHash for key let keyHash = packStringToField(key); let current = OptionField.none(); diff --git a/src/o1js-missing.ts b/src/o1js-missing.ts index 2868e75..8f8fc6f 100644 --- a/src/o1js-missing.ts +++ b/src/o1js-missing.ts @@ -35,11 +35,7 @@ export { const ProvableType = { get>(type: A): ToProvable { return ( - (typeof type === 'object' || typeof type === 'function') && - type !== null && - 'provable' in type - ? type.provable - : type + hasProperty(type, 'provable') ? type.provable : type ) as ToProvable; }, @@ -72,6 +68,15 @@ const ProvableType = { return hasProperty(type_, 'toFields') && hasProperty(type_, 'fromFields'); }, + isProvableHashableType(type: unknown): type is ProvableHashableType { + let type_ = ProvableType.get(type); + return ( + ProvableType.isProvableType(type_) && + hasProperty(type_, 'toInput') && + hasProperty(type_, 'empty') + ); + }, + constant(value: T): ProvablePure & { serialize(): any } { return { serialize() { @@ -113,7 +118,9 @@ function lengthRecursive(array: NestedArray): number { return length; } -function assertIsProvable(type: unknown): asserts type is Provable { +function assertIsProvable( + type: unknown +): asserts type is ProvableMaybeHashable { assertHasProperty( type, 'toFields', From 46f816eb493b0a0515ba755188c25f0efafe4f80 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 7 Nov 2024 11:53:55 +0100 Subject: [PATCH 39/44] add documentation, also of hash collisions --- src/credentials/dynamic-array.ts | 1 + src/credentials/dynamic-hash.test.ts | 13 +++++++++++ src/credentials/dynamic-hash.ts | 34 +++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index 142af94..ed9c478 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -317,6 +317,7 @@ class DynamicArrayBase { // create blocks of 2 field elements each // TODO abstract this into a `chunk()` method that returns a DynamicArray> let elementSize = bitSize(type); + if (elementSize === 0) elementSize = 1; // edge case for empty types like `Undefined` let elementsPerHalfBlock = Math.floor(254 / elementSize); if (elementsPerHalfBlock === 0) elementsPerHalfBlock = 1; // larger types are compressed diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 2bb14c8..3d8ec1d 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -6,6 +6,7 @@ import { hashDynamic, hashRecord, hashString, + packStringToField, packToField, } from './dynamic-hash.ts'; import { test } from 'node:test'; @@ -13,6 +14,18 @@ import * as nodeAssert from 'node:assert'; import { Bytes, Field, MerkleList, Poseidon, Provable, UInt8 } from 'o1js'; import { DynamicRecord } from './dynamic-record.ts'; +// some hash collisions to be aware of +hashDynamic(5).assertEquals(hashDynamic(5n), '1'); +hashDynamic(undefined).assertEquals(hashDynamic(null), '2'); +hashDynamic([0]).assertEquals(hashDynamic('\x00'), '3'); +hashDynamic([1, 2].map(UInt8.from)).assertEquals(hashDynamic('\x01\x02'), '4'); + +// TODO: this is a hash collision we need to fix +let emptyString = packStringToField('\x00').toBigInt(); +let emptyValue = packToField(0).toBigInt(); +console.log({ emptyString, emptyValue }); +hashRecord({ '\x00': 0 }).assertEquals(hashRecord({})); + let shortString = 'hi'; let ShortString = DynamicString({ maxLength: 5 }); let shortHash = hashString(shortString); diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index 6cef8a3..991ad2c 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -53,11 +53,34 @@ type HashableValue = * Hash an input that is either a simple JSON-with-bigints object or a provable type. * * The hashing algorithm is compatible with dynamic-length schemas. + * + * Note: There are expected hash collisions between _different_ types, like + * ```ts + * hashDynamic(5) === hashDynamic(5n); + * hashDynamic(undefined) === hashDynamic(null); + * hashDynamic([1]) === hashDynamic("\x01"); + * ``` */ function hashDynamic(value: HashableValue | unknown) { return packToField(value, undefined, { mustHash: true }); } +/** + * Pack an arbitrary value into a field element. + * + * The packing algorithm is compatible with dynamic-length schemas. + * + * This is the same as `hashDynamic()`, with the (default) option to not hash + * types that are single field elements after packing, but return them directly. + * + * e.g. + * ```ts + * packToField(5) === Field(5); + * hashDynamic(5) === Poseidon.hash([Field(5)]); + * ``` + * + * The fallback algorithm for unknown objects is to call `hashRecord()` on them. + */ function packToField( value: T, type?: ProvableHashableType, @@ -104,12 +127,21 @@ function packToField( return hashRecord(value); } +/** + * Hash an array, packing the elements if possible. + * + * Avoids hash collisions by encoding the length of the array at the beginning. + */ function hashArray(array: unknown[]) { let type = innerArrayType(array); let Array = BaseType.DynamicArray(type, { maxLength: array.length }); return Array.from(array).hash(); } +/** + * Hash an arbitrary object, by first packing keys and values into 1 field element each, + * and then using Poseidon on the concatenated elements (which are a multiple of 2, so we avoid collisions). + */ function hashRecord(data: {}) { assert(typeof data === 'object', 'Expected plain object'); let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { @@ -191,7 +223,7 @@ function provableTypeOfConstructor( * These should be represented as records i.e. { first: 1, second: 'a' }. */ function innerArrayType(array: unknown[]): ProvableHashableType { - let type = provableTypeOf(array[0]); + let type = provableTypeOf(array[0]); // empty array => Undefined assert( array.every((v) => provableTypeEquals(v, type)), 'Array elements must be homogenous' From eb2674bd10f580accf11f93b561cfaffaa0f6cc4 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 7 Nov 2024 12:14:28 +0100 Subject: [PATCH 40/44] fix a bad hash collision and document a few others --- src/credentials/dynamic-hash.test.ts | 45 ++++++++++++++++++++-------- src/credentials/dynamic-hash.ts | 21 +++++-------- src/credentials/dynamic-record.ts | 8 ++--- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 3d8ec1d..93c829b 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -6,25 +6,44 @@ import { hashDynamic, hashRecord, hashString, - packStringToField, packToField, } from './dynamic-hash.ts'; import { test } from 'node:test'; import * as nodeAssert from 'node:assert'; -import { Bytes, Field, MerkleList, Poseidon, Provable, UInt8 } from 'o1js'; +import { + Bytes, + Field, + MerkleList, + Poseidon, + Provable, + UInt32, + UInt8, +} from 'o1js'; import { DynamicRecord } from './dynamic-record.ts'; -// some hash collisions to be aware of -hashDynamic(5).assertEquals(hashDynamic(5n), '1'); -hashDynamic(undefined).assertEquals(hashDynamic(null), '2'); -hashDynamic([0]).assertEquals(hashDynamic('\x00'), '3'); -hashDynamic([1, 2].map(UInt8.from)).assertEquals(hashDynamic('\x01\x02'), '4'); - -// TODO: this is a hash collision we need to fix -let emptyString = packStringToField('\x00').toBigInt(); -let emptyValue = packToField(0).toBigInt(); -console.log({ emptyString, emptyValue }); -hashRecord({ '\x00': 0 }).assertEquals(hashRecord({})); +// some hash collisions to be aware of, that are also quite natural + +hashDynamic(5n).assertEquals(hashDynamic(5), 'bigint ~ number'); +hashDynamic(true).assertEquals(hashDynamic(1), 'bool ~ number'); +hashDynamic({ a: 0n }).assertEquals( + hashDynamic({ a: false }), + 'record ~ record with equivalent fields' +); +hashDynamic(undefined).assertEquals( + hashDynamic(null), + 'undefined ~ null ~ any empty type' +); +hashDynamic('\x01\x02').assertEquals( + hashDynamic([1, 2].map(UInt8.from)), + 'strings ~ dynamic arrays of bytes' +); +hashDynamic([UInt32.from(0)]).assertEquals( + hashDynamic([Field(0)]), + 'dynamic arrays of zeros of same length, regardless of packing density of type (because padding is zeros)' +); + +// this used to be an unexpected hash collision that was fixed +hashRecord({ '\x00': 0 }).assertNotEquals(hashRecord({})); let shortString = 'hi'; let ShortString = DynamicString({ maxLength: 5 }); diff --git a/src/credentials/dynamic-hash.ts b/src/credentials/dynamic-hash.ts index 991ad2c..cfeef75 100644 --- a/src/credentials/dynamic-hash.ts +++ b/src/credentials/dynamic-hash.ts @@ -31,7 +31,6 @@ export { hashDynamic, hashArray, hashString, - packStringToField, packToField, hashRecord, bitSize, @@ -54,11 +53,14 @@ type HashableValue = * * The hashing algorithm is compatible with dynamic-length schemas. * - * Note: There are expected hash collisions between _different_ types, like + * Note: There are expected hash collisions between different types + * - that have the same overall shape in terms of dynamic-length types, and + * - individual atomic pieces have the same representation as field elements * ```ts - * hashDynamic(5) === hashDynamic(5n); + * hashDynamic(true) === hashDynamic(1); + * hashDynamic({ a: 5 }) === hashDynamic({ a: 5n }); * hashDynamic(undefined) === hashDynamic(null); - * hashDynamic([1]) === hashDynamic("\x01"); + * hashDynamic("\x01") === hashDynamic([UInt8.from(1)]); * ``` */ function hashDynamic(value: HashableValue | unknown) { @@ -145,7 +147,7 @@ function hashArray(array: unknown[]) { function hashRecord(data: {}) { assert(typeof data === 'object', 'Expected plain object'); let entryHashes = mapEntries(data as UnknownRecord, (key, value) => { - return [packStringToField(key), packToField(value)]; + return [hashString(key), packToField(value)]; }); return Poseidon.hash(entryHashes.flat()); } @@ -277,15 +279,6 @@ function provableTypeEquals( return valueType === type; } -// for packing keys -- not compatible with dynamic string hash! (as keys will be known at compile time) -function packStringToField(string: string) { - let bytes = enc.encode(string); - let B = Bytes(bytes.length); - let fields = toFieldsPacked(B, B.from(bytes)); - if (fields.length === 1) return fields[0]!; - return Poseidon.hash(fields); -} - // helpers function isStruct(type: ProvableType): type is Struct { diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index 07a8f0a..e4a205b 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -32,7 +32,7 @@ import { serializeNestedProvable, serializeNestedProvableValue, } from '../serialize-provable.ts'; -import { packStringToField, packToField } from './dynamic-hash.ts'; +import { hashString, packToField } from './dynamic-hash.ts'; import { BaseType } from './dynamic-base-types.ts'; export { DynamicRecord, GenericRecord, type UnknownRecord, extractProperty }; @@ -94,7 +94,7 @@ function DynamicRecord< let actualValue = type === undefined ? value : type.fromValue(value); return { - key: packStringToField(key).toBigInt(), + key: hashString(key).toBigInt(), value: packToField(actualValue, type).toBigInt(), }; }); @@ -148,7 +148,7 @@ class GenericRecordBase { let entries = Object.entries(actual).map(([key, value]) => { let type = NestedProvable.get(NestedProvable.fromValue(value)); return { - key: packStringToField(key), + key: hashString(key), value: packToField(type.fromValue(value), type), }; }); @@ -162,7 +162,7 @@ class GenericRecordBase { getAny(valueType: A, key: string) { // find valueHash for key - let keyHash = packStringToField(key); + let keyHash = hashString(key); let current = OptionField.none(); for (let { isSome, value: entry } of this.entries) { From ffb18e8ea62ba108b1fc87aacb89165682d257f9 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 7 Nov 2024 12:42:01 +0100 Subject: [PATCH 41/44] add method to check array equality, use to clean up tests --- src/credentials/dynamic-array.ts | 18 ++++++++- src/credentials/dynamic-hash.test.ts | 7 ++-- src/credentials/dynamic-record.test.ts | 55 ++++++++++---------------- src/credentials/dynamic-record.ts | 10 ++++- 4 files changed, 49 insertions(+), 41 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index ed9c478..b7d7d56 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -42,7 +42,7 @@ import { } from '../serialize-provable.ts'; import { TypeBuilder, TypeBuilderPure } from '../provable-type-builder.ts'; import { StaticArray } from './static-array.ts'; -import { bitSize, packedFieldSize, packToField } from './dynamic-hash.ts'; +import { bitSize, packToField } from './dynamic-hash.ts'; import { BaseType } from './dynamic-base-types.ts'; export { DynamicArray }; @@ -301,6 +301,8 @@ class DynamicArrayBase { /** * Dynamic array hash that only depends on the actual values (not the padding). + * + * Avoids hash collisions by encoding the number of actual elements at the beginning of the hash input. */ hash() { let type = ProvableType.get(this.innerType); @@ -363,6 +365,20 @@ class DynamicArrayBase { return state[0]; } + /** + * Assert that the array is exactly equal, in its representation in field elements, to another array. + * + * Warning: Also checks equality of the padding and maxLength, which don't contribute to the "meaningful" part of the array. + * Therefore, this method is mainly intended for testing. + */ + assertEqualsStrict(other: DynamicArray) { + assert(this.maxLength === other.maxLength, 'max length mismatch'); + this.length.assertEquals(other.length, 'length mismatch'); + zip(this.array, other.array).forEach(([a, b]) => { + Provable.assertEqual(this.innerType, a, b); + }); + } + /** * Push a value, without changing the maxLength. * diff --git a/src/credentials/dynamic-hash.test.ts b/src/credentials/dynamic-hash.test.ts index 93c829b..1f494dc 100644 --- a/src/credentials/dynamic-hash.test.ts +++ b/src/credentials/dynamic-hash.test.ts @@ -140,9 +140,10 @@ async function main() { let RecordArray = DynamicArray(Record, { maxLength: 5 }); await test('hash arrays of records', () => { - Provable.witness(RecordArray, () => array) - .hash() - .assertEquals(arrayHash, 'array'); + let arrayVar = Provable.witness(RecordArray, () => array); + + arrayVar.hash().assertEquals(arrayHash, 'array'); + hashDynamic(array).assertEquals(arrayHash, 'array'); }); } diff --git a/src/credentials/dynamic-record.test.ts b/src/credentials/dynamic-record.test.ts index 4152134..aa04739 100644 --- a/src/credentials/dynamic-record.test.ts +++ b/src/credentials/dynamic-record.test.ts @@ -20,7 +20,9 @@ import { hashDynamic, hashRecord } from './dynamic-hash.ts'; import { array } from '../o1js-missing.ts'; import { DynamicArray } from './dynamic-array.ts'; +const String5 = DynamicString({ maxLength: 5 }); const String10 = DynamicString({ maxLength: 10 }); +const String20 = DynamicString({ maxLength: 20 }); // original schema, data and hash from known layout @@ -33,41 +35,35 @@ const OriginalSchema = Schema({ sixth: array(Field, 3), }); -let input = { +let original = OriginalSchema.from({ first: 1, second: true, third: 'something', fourth: 123n, fifth: { field: 2, string: '...' }, sixth: [1n, 2n, 3n], -}; - -let original = OriginalSchema.from(input); +}); const expectedHash = hashRecord(original); const OriginalWrappedInStruct = Struct(OriginalSchema.schema); -let originalStruct = OriginalWrappedInStruct.fromValue(input); +let originalStruct = OriginalWrappedInStruct.fromValue(original); // subset schema and circuit that doesn't know the full original layout -// not necessarily matches the length of the original schema -const String20 = DynamicString({ maxLength: 20 }); -const String5 = DynamicString({ maxLength: 5 }); - -const Fifth = DynamicRecord( - { - // _nested_ subset of original schema - string: String5, // different max length here as well - }, - { maxEntries: 5 } -); - const Subschema = DynamicRecord( { - // not necessarily in order + // different max length of string/array properties third: String20, - fifth: Fifth, + // not necessarily in original order first: Field, + // subset in nested schema + fifth: DynamicRecord( + { + // different max length here as well + string: String5, + }, + { maxEntries: 5 } + ), }, { maxEntries: 10 } ); @@ -84,15 +80,11 @@ async function circuit() { record.get('first').assertEquals(1, 'first'); // dynamic string with different max length - Provable.assertEqual( - String20, - record.get('third'), - String20.from('something') - ); + record.get('third').assertEqualsStrict(String20.from('something')); // nested subschema let fifthString = record.get('fifth').get('string'); - Provable.assertEqual(String5, fifthString, String5.from('...')); + fifthString.assertEqualsStrict(String5.from('...')); }); await test('DynamicRecord.getAny()', () => { @@ -109,11 +101,8 @@ async function circuit() { ); const SixthDynamic = DynamicArray(Field, { maxLength: 7 }); - Provable.assertEqual( - SixthDynamic, - record.getAny(SixthDynamic, 'sixth'), - SixthDynamic.from([1n, 2n, 3n]) - ); + let sixth = record.getAny(SixthDynamic, 'sixth'); + sixth.assertEqualsStrict(SixthDynamic.from([1n, 2n, 3n])); assert.throws(() => record.getAny(Bool, 'missing'), /Key not found/); }); @@ -140,11 +129,7 @@ async function circuit() { originalStructHash.assertEquals(originalHash, 'hashCredential() (struct)'); - let subschemaHash = hashCredential(Subschema, { - owner, - data: record, - }).hash; - + let subschemaHash = hashCredential(Subschema, { owner, data: record }).hash; subschemaHash.assertEquals( originalHash, 'hashCredential() (dynamic record)' diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index e4a205b..0728f53 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -73,6 +73,12 @@ function DynamicRecord< return DynamicRecord.provable.fromValue(actual); } + static get shape(): { + [K in keyof TKnown]: ProvableHashableType; + } { + return shape; + } + static provable = TypeBuilder.shape({ entries: array(Option(Struct({ key: Field, value: Field })), maxEntries), actual: Unconstrained.withEmpty(emptyTKnown), @@ -114,7 +120,6 @@ function DynamicRecord< } }; } -BaseType.set('DynamicRecord', DynamicRecord); const OptionField = Option(Field); const OptionKeyValue = Option(Struct({ key: Field, value: Field })); @@ -129,7 +134,6 @@ function GenericRecord({ maxEntries }: { maxEntries: number }) { } }; } -BaseType.set('GenericRecord', GenericRecord); class GenericRecordBase { entries: Option<{ key: Field; value: Field }>[]; @@ -203,6 +207,7 @@ class GenericRecordBase { } } +BaseType.set('GenericRecord', GenericRecord); GenericRecord.Base = GenericRecordBase; class DynamicRecordBase extends GenericRecordBase { @@ -218,6 +223,7 @@ class DynamicRecordBase extends GenericRecordBase { } } +BaseType.set('DynamicRecord', DynamicRecord); DynamicRecord.Base = DynamicRecordBase; type DynamicRecordRaw = { From 02df06b436f4af7e1811319c07e9387940292a12 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 7 Nov 2024 12:49:05 +0100 Subject: [PATCH 42/44] one more comment --- src/credentials/dynamic-array.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index b7d7d56..d6e7160 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -348,13 +348,16 @@ class DynamicArrayBase { let Fieldx2 = StaticArray(Field, 2); let Blocks = DynamicArray(Fieldx2, { maxLength: maxBlocks }); + // nBlocks = ceil(length / elementsPerBlock) = floor((length + elementsPerBlock - 1) / elementsPerBlock) let nBlocks = UInt32.Unsafe.fromField( this.length.add(elementsPerUint32 + elementsPerBlock - 1) ).div(elementsPerBlock).value; let dynBlocks = new Blocks(blocks.map(Fieldx2.from), nBlocks); - // now hash the 2-field elements blocks, on permutation at a time + // now hash the 2-field elements blocks, one permutation at a time + // note: there's a padding element included at the end in the case of uneven number of blocks + // however, this doesn't cause hash collisions because we encoded the length at the beginning let state = Poseidon.initialState(); dynBlocks.forEach((block, isPadding) => { let newState = Poseidon.update(state, block.array); From 0f3e09684b9bccd072cde95dfe3bf575f2c0581c Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 7 Nov 2024 14:16:50 +0100 Subject: [PATCH 43/44] a nice abstraction --- src/credentials/dynamic-array.ts | 12 ++---------- src/credentials/dynamic-base-types.ts | 25 ++----------------------- src/credentials/dynamic-record.ts | 4 ++-- src/credentials/dynamic-string.ts | 2 +- src/util.ts | 16 ++++++++++++++++ 5 files changed, 23 insertions(+), 36 deletions(-) diff --git a/src/credentials/dynamic-array.ts b/src/credentials/dynamic-array.ts index d6e7160..8d7d065 100644 --- a/src/credentials/dynamic-array.ts +++ b/src/credentials/dynamic-array.ts @@ -13,15 +13,7 @@ import { type IsPure, Poseidon, } from 'o1js'; -import { - assert, - assertHasProperty, - chunk, - defined, - fill, - pad, - zip, -} from '../util.ts'; +import { assert, assertHasProperty, chunk, fill, pad, zip } from '../util.ts'; import { type ProvableHashablePure, type ProvableHashableType, @@ -134,7 +126,7 @@ function DynamicArray< return DynamicArray_; } -BaseType.set('DynamicArray', DynamicArray); +BaseType.DynamicArray = DynamicArray; class DynamicArrayBase { /** diff --git a/src/credentials/dynamic-base-types.ts b/src/credentials/dynamic-base-types.ts index 06b5564..98aa99c 100644 --- a/src/credentials/dynamic-base-types.ts +++ b/src/credentials/dynamic-base-types.ts @@ -4,7 +4,7 @@ import type { DynamicArray } from './dynamic-array.ts'; import type { DynamicString } from './dynamic-string.ts'; import type { DynamicRecord, GenericRecord } from './dynamic-record.ts'; -import { assertDefined } from '../util.ts'; +import { Required } from '../util.ts'; export { BaseType }; @@ -14,26 +14,5 @@ let baseType: { DynamicRecord?: typeof DynamicRecord; GenericRecord?: typeof GenericRecord; } = {}; -type BaseType = typeof baseType; -const BaseType = { - set(key: K, value: BaseType[K]) { - baseType[key] = value; - }, - get DynamicArray() { - assertDefined(baseType.DynamicArray); - return baseType.DynamicArray; - }, - get DynamicString() { - assertDefined(baseType.DynamicString); - return baseType.DynamicString; - }, - get DynamicRecord() { - assertDefined(baseType.DynamicRecord); - return baseType.DynamicRecord; - }, - get GenericRecord() { - assertDefined(baseType.GenericRecord); - return baseType.GenericRecord; - }, -}; +const BaseType = Required(baseType); diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index 0728f53..ed9af8c 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -207,7 +207,7 @@ class GenericRecordBase { } } -BaseType.set('GenericRecord', GenericRecord); +BaseType.GenericRecord = GenericRecord; GenericRecord.Base = GenericRecordBase; class DynamicRecordBase extends GenericRecordBase { @@ -223,7 +223,7 @@ class DynamicRecordBase extends GenericRecordBase { } } -BaseType.set('DynamicRecord', DynamicRecord); +BaseType.DynamicRecord = DynamicRecord; DynamicRecord.Base = DynamicRecordBase; type DynamicRecordRaw = { diff --git a/src/credentials/dynamic-string.ts b/src/credentials/dynamic-string.ts index 7867223..cbc171f 100644 --- a/src/credentials/dynamic-string.ts +++ b/src/credentials/dynamic-string.ts @@ -63,7 +63,7 @@ function DynamicString({ maxLength }: { maxLength: number }) { return DynamicString; } -BaseType.set('DynamicString', DynamicString); +BaseType.DynamicString = DynamicString; const enc = new TextEncoder(); const dec = new TextDecoder(); diff --git a/src/util.ts b/src/util.ts index b38281b..2d02138 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2,6 +2,7 @@ export { assert, assertDefined, defined, + Required, assertHasProperty, hasProperty, assertIsObject, @@ -42,6 +43,21 @@ function defined(input: T | undefined, message?: string): T { return input; } +function Required( + t: T +): { + [P in keyof T]-?: T[P]; +} { + return new Proxy(t, { + get(target, key) { + return defined( + (target as any)[key], + `Property "${String(key)}" is undefined` + ); + }, + }) as Required; +} + function assertIsObject( obj: unknown, message?: string From c1521a0a5c8c541e0b0c8b4e56989856e1f07efb Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 7 Nov 2024 14:40:49 +0100 Subject: [PATCH 44/44] few more tests & cleanup --- src/credentials/dynamic-record.test.ts | 25 ++++++++++++++++++++----- src/credentials/dynamic-record.ts | 18 +++++++----------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/credentials/dynamic-record.test.ts b/src/credentials/dynamic-record.test.ts index aa04739..565cb22 100644 --- a/src/credentials/dynamic-record.test.ts +++ b/src/credentials/dynamic-record.test.ts @@ -88,22 +88,37 @@ async function circuit() { }); await test('DynamicRecord.getAny()', () => { + // we can get the other fields as well, if we know their type record.getAny(Bool, 'second').assertEquals(true, 'second'); record.getAny(UInt64, 'fourth').assertEquals(UInt64.from(123n)); - // this works because structs are hashed in dynamic record style, - // and the string is hashed in dynamic array style + // `packToField()` collisions mean that we can also reinterpret fields into types with equivalent packing + // (if the new type's `fromValue()` allows the original value) + record.getAny(Bool, 'first').assertEquals(true, 'first'); + record.getAny(UInt64, 'first').assertEquals(UInt64.one); + + // we can get a nested record as struct (and nested strings can have different length) + // this works because structs are hashed in dynamic record style const FifthStruct = Struct({ field: Field, string: String20 }); + let fifth = record.getAny(FifthStruct, 'fifth'); Provable.assertEqual( FifthStruct, - record.getAny(FifthStruct, 'fifth'), - FifthStruct.fromValue({ field: 2, string: '...' }) + fifth, + FifthStruct.fromValue(original.fifth) ); + // can get an array as dynamic array, as long as the maxLength is >= the actual length const SixthDynamic = DynamicArray(Field, { maxLength: 7 }); let sixth = record.getAny(SixthDynamic, 'sixth'); - sixth.assertEqualsStrict(SixthDynamic.from([1n, 2n, 3n])); + sixth.assertEqualsStrict(SixthDynamic.from(original.sixth)); + + const SixthDynamicShort = DynamicArray(Field, { maxLength: 2 }); + assert.throws( + () => record.getAny(SixthDynamicShort, 'sixth'), + /larger than target size/ + ); + // can't get a missing key assert.throws(() => record.getAny(Bool, 'missing'), /Key not found/); }); diff --git a/src/credentials/dynamic-record.ts b/src/credentials/dynamic-record.ts index ed9af8c..0ef7576 100644 --- a/src/credentials/dynamic-record.ts +++ b/src/credentials/dynamic-record.ts @@ -149,19 +149,15 @@ class GenericRecordBase { } static from(actual: UnknownRecord): GenericRecordBase { - let entries = Object.entries(actual).map(([key, value]) => { - let type = NestedProvable.get(NestedProvable.fromValue(value)); - return { + let entries = Object.entries(actual).map(([key, value]) => { + return OptionKeyValue.from({ key: hashString(key), - value: packToField(type.fromValue(value), type), - }; + value: packToField(value), + }); }); - let options = pad( - entries.map((entry) => OptionKeyValue.from(entry)), - this.prototype.maxEntries, - OptionKeyValue.none() - ); - return new this({ entries: options, actual: Unconstrained.from(actual) }); + let maxEntries = this.prototype.maxEntries; + let padded = pad(entries, maxEntries, OptionKeyValue.none()); + return new this({ entries: padded, actual: Unconstrained.from(actual) }); } getAny(valueType: A, key: string) {