From 5b65e02f4b29115898f2d4c561a644671d65b3ea Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sun, 22 Dec 2024 21:09:47 +0100 Subject: [PATCH] keccak: Pass fuzz tests vs OpenSSL --- constantine.nimble | 3 +- constantine/hashes.nim | 14 +- constantine/hashes/h_keccak.nim | 39 ++---- constantine/hashes/h_sha256.nim | 6 +- constantine/hashes/keccak/keccak_generic.nim | 16 +-- tests/t_hash_sha256_vs_openssl.nim | 79 +++++------ tests/t_hash_sha3_vs_openssl.nim | 131 +++++++++++++++++++ 7 files changed, 196 insertions(+), 92 deletions(-) create mode 100644 tests/t_hash_sha3_vs_openssl.nim diff --git a/constantine.nimble b/constantine.nimble index 1f6cd0e00..325431e27 100644 --- a/constantine.nimble +++ b/constantine.nimble @@ -362,7 +362,8 @@ const testDesc: seq[tuple[path: string, useGMP: bool]] = @[ # Hashing vs OpenSSL # ---------------------------------------------------------- - ("tests/t_hash_sha256_vs_openssl.nim", false), # skip OpenSSL tests on Windows + ("tests/t_hash_sha256_vs_openssl.nim", false), + ("tests/t_hash_sha3_vs_openssl.nim", false), # Ciphers # ---------------------------------------------------------- diff --git a/constantine/hashes.nim b/constantine/hashes.nim index 5569fbb04..55da1a6fd 100644 --- a/constantine/hashes.nim +++ b/constantine/hashes.nim @@ -63,7 +63,15 @@ func hash*( # Exports # ----------------------------------------------------------------------- -import ./hashes/h_sha256 -export h_sha256 +import ./hashes/[ + h_keccak, + h_sha256 +] +export + h_keccak, + h_sha256 -static: doAssert sha256 is CryptoHash +static: + doAssert keccak256 is CryptoHash + doAssert sha256 is CryptoHash + doAssert sha3_256 is CryptoHash diff --git a/constantine/hashes/h_keccak.nim b/constantine/hashes/h_keccak.nim index 3c3312cc4..49e996144 100644 --- a/constantine/hashes/h_keccak.nim +++ b/constantine/hashes/h_keccak.nim @@ -6,10 +6,8 @@ # * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -import ../zoo_exports - import - ../platforms/[abstractions, views], + constantine/platforms/[abstractions, views], ./keccak/keccak_generic # Keccak, the hash function underlying SHA3 @@ -93,9 +91,8 @@ type # Similarly after a squeeze, absorb_offset is incremented by the sponge rate. # The real offset can be recovered with a substraction # to properly update the state. - H {.align: 64.}: KeccakState - buf {.align: 64.}: array[bits div 8, byte] + buf {.align: 64.}: array[200 - 2*(bits div 8), byte] absorb_offset: int32 squeeze_offset: int32 @@ -125,11 +122,13 @@ func absorbBuffer(ctx: var KeccakContext) {.inline.} = template digestSize*(H: type KeccakContext): int = ## Returns the output size in bytes - KeccakContext.bits shr 3 + # hardcoded for now or concept match issue with CryptoHash + 32 template internalBlockSize*(H: type KeccakContext): int = ## Returns the byte size of the hash function ingested blocks - 2 * (KeccakContext.bits shr 3) + # hardcoded for now or concept match issue with CryptoHash + 200 func init*(ctx: var KeccakContext) {.inline.} = ## Initialize or reinitialize a Keccak context @@ -145,9 +144,6 @@ func absorb*(ctx: var KeccakContext, message: openArray[byte]) = ## Additionally ensure that the message(s) passed were stored ## in memory considered secure for your threat model. - if message.len == 0: - return - var pos = int ctx.absorb_offset var cur = 0 var bytesLeft = message.len @@ -183,13 +179,11 @@ func absorb*(ctx: var KeccakContext, message: openArray[byte]) = ctx.buf.rawCopy(dStart = pos, message, sStart = cur, len = bytesLeft) # Epilogue - ctx.absorb_offset = int32 bytesLeft + ctx.absorb_offset = int32(pos+bytesLeft) # Signal that the next squeeze transition needs a permute ctx.squeeze_offset = int32 ctx.rate() func squeeze*(ctx: var KeccakContext, digest: var openArray[byte]) = - if digest.len == 0: - return var pos = ctx.squeeze_offset var cur = 0 @@ -246,9 +240,11 @@ func update*(ctx: var KeccakContext, message: openArray[byte]) = ## in memory considered secure for your threat model. ctx.absorb(message) -func finish*[N: static int](ctx: var KeccakContext, digest: var array[N, byte]) = +func finish*(ctx: var KeccakContext, digest: var array[32, byte]) = ## Finalize a Keccak computation and output the - ## message digest to the `digest` buffer + ## message digest to the `digest` buffer. + ## + ## An `update` MUST be called before finish even with empty message. ## ## Security note: this does not clear the internal buffer. ## if sensitive content is used, use "ctx.clear()" @@ -260,16 +256,3 @@ func clear*(ctx: var KeccakContext) = ## Clear the context internal buffers # TODO: ensure compiler cannot optimize the code away ctx.reset() - -when isMainModule: - import constantine/serialization/codecs - - var msg: array[32, byte] - var digest: array[32, byte] - var ctx: keccak256 - - ctx.init() - ctx.update(msg) - ctx.finish(digest) - - echo digest.toHex() \ No newline at end of file diff --git a/constantine/hashes/h_sha256.nim b/constantine/hashes/h_sha256.nim index ccd46da86..85a9a93a1 100644 --- a/constantine/hashes/h_sha256.nim +++ b/constantine/hashes/h_sha256.nim @@ -6,11 +6,11 @@ # * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -import ../zoo_exports +import constantine/zoo_exports import - ../platforms/[abstractions, views], - ../serialization/endians, + constantine/platforms/[abstractions, views], + constantine/serialization/endians, ./sha256/sha256_generic when UseASM_X86_32: diff --git a/constantine/hashes/keccak/keccak_generic.nim b/constantine/hashes/keccak/keccak_generic.nim index 1835d5bd3..9f1eda50c 100644 --- a/constantine/hashes/keccak/keccak_generic.nim +++ b/constantine/hashes/keccak/keccak_generic.nim @@ -231,10 +231,10 @@ func xorInSingle(H: var KeccakState, val: byte, offset: int) {.inline.} = let lane = uint64(val) shl slot # All bits but the one set in `val` are 0, and 0 is neutral element of xor H.state[offset shr 3] ^= lane -func xorInBlock_generic(H: var KeccakState, msg: array[64, byte]) {.inline.} = +func xorInBlock_generic(H: var KeccakState, msg: array[200 - 2*32, byte]) {.inline.} = ## Add new data into the Keccak state # This can benefit from vectorized instructions - for i in 0 ..< 8: + for i in 0 ..< msg.len div 8: H.state[i] ^= uint64.fromBytes(msg, i*8, littleEndian) func xorInPartial*(H: var KeccakState, msg: openArray[byte]) = @@ -254,7 +254,7 @@ func xorInPartial*(H: var KeccakState, msg: openArray[byte]) = # Lastly, this is only called when transitioning # between absorbing and squeezing, for hashing # this means once, however long a message to hash is. - var blck: array[64, byte] # zero-init + var blck: array[200 - 2*32, byte] # zero-init rawCopy(blck, 0, msg, 0, msg.len) H.xorInBlock_generic(blck) @@ -283,7 +283,7 @@ func copyOutPartial*( # Implementation details: # we could avoid a temporary block # see `xorInPartial` for rationale - var blck {.noInit.}: array[64, byte] + var blck {.noInit.}: array[200 - 2*32, byte] H.copyOutWords(blck) rawCopy(dst, 0, blck, hByteOffset, dst.len) @@ -303,8 +303,8 @@ func hashMessageBlocks_generic*( ## a permutation is needed in-between var message = message - const rate = 64 # TODO: make a generic Keccak state with auto-derived rate - const numRounds = 24 # TODO: auto derive number of rounds + const rate = 200 - 2*32 # TODO: make a generic Keccak state with auto-derived rate + const numRounds = 24 # TODO: auto derive number of rounds for _ in 0 ..< numBlocks: let msg = cast[ptr array[rate, byte]](message) H.xorInBlock_generic(msg[]) @@ -321,8 +321,8 @@ func squeezeDigestBlocks_generic*( ## i.e. previous operation cannot be an absorb ## a permutation is needed in-between var digest = digest - const rate = 64 # TODO: make a generic Keccak state with auto-derived rate - const numRounds = 24 # TODO: auto derive number of rounds + const rate = 200 - 2*32 # TODO: make a generic Keccak state with auto-derived rate + const numRounds = 24 # TODO: auto derive number of rounds for _ in 0 ..< numBlocks: let msg = cast[ptr array[rate, byte]](digest) H.copyOutWords(msg[]) diff --git a/tests/t_hash_sha256_vs_openssl.nim b/tests/t_hash_sha256_vs_openssl.nim index 84e388e6c..80d43c4ae 100644 --- a/tests/t_hash_sha256_vs_openssl.nim +++ b/tests/t_hash_sha256_vs_openssl.nim @@ -28,30 +28,19 @@ else: # OpenSSL wrapper # -------------------------------------------------------------------- -# OpenSSL removed direct use of their SHA256 function. https://github.com/openssl/openssl/commit/4d49b68504cc494e552bce8e0b82ec8b501d5abe -# It isn't accessible anymore in Windows CI on Github Action. -# But the new API isn't expose on Linux :/ - -# TODO: fix Windows -when not defined(windows): - proc SHA256[T: byte|char]( - msg: openarray[T], - digest: ptr array[32, byte] = nil - ): ptr array[32, byte] {.noconv, dynlib: DLLSSLName, importc.} - - # proc EVP_Q_digest[T: byte|char]( - # ossl_libctx: pointer, - # algoName: cstring, - # propq: cstring, - # data: openArray[T], - # digest: var array[32, byte], - # size: ptr uint): int32 {.noconv, dynlib: DLLSSLName, importc.} - - proc SHA256_OpenSSL[T: byte|char]( - digest: var array[32, byte], - s: openArray[T]) = - discard SHA256(s, digest.addr) - # discard EVP_Q_digest(nil, "SHA256", nil, s, digest, nil) +proc EVP_Q_digest[T: byte|char]( + ossl_libctx: pointer, + algoName: cstring, + propq: cstring, + data: openArray[T], + digest: var array[32, byte], + size: ptr uint): int32 {.noconv, dynlib: DLLSSLName, importc.} + +proc SHA256_OpenSSL[T: byte|char]( + digest: var array[32, byte], + s: openArray[T]) = + # discard SHA256(s, digest.addr) + discard EVP_Q_digest(nil, "SHA256", nil, s, digest, nil) # Test # -------------------------------------------------------------------- @@ -84,16 +73,15 @@ proc sanityABC2 = doAssert bufCt == hashed -when not defined(windows): - proc innerTest(rng: var RngState, sizeRange: Slice[int]) = - let size = rng.random_unsafe(sizeRange) - let msg = rng.random_byte_seq(size) +proc innerTest(rng: var RngState, sizeRange: Slice[int]) = + let size = rng.random_unsafe(sizeRange) + let msg = rng.random_byte_seq(size) - var bufCt, bufOssl: array[32, byte] + var bufCt, bufOssl: array[32, byte] - sha256.hash(bufCt, msg) - SHA256_OpenSSL(bufOssl, msg) - doAssert bufCt == bufOssl, "Test failed with message of length " & $size + sha256.hash(bufCt, msg) + SHA256_OpenSSL(bufOssl, msg) + doAssert bufCt == bufOssl, "Test failed with message of length " & $size proc chunkTest(rng: var RngState, sizeRange: Slice[int]) = let size = rng.random_unsafe(sizeRange) @@ -131,12 +119,9 @@ proc main() = var rng: RngState rng.seed(0xFACADE) - when not defined(windows): - echo "SHA256 - 0 <= size < 64 - exhaustive" - for i in 0 ..< 64: - rng.innerTest(i .. i) - else: - echo "SHA256 - 0 <= size < 64 - exhaustive [SKIPPED]" + echo "SHA256 - 0 <= size < 64 - exhaustive" + for i in 0 ..< 64: + rng.innerTest(i .. i) echo "SHA256 - 0 <= size < 64 - exhaustive chunked" for i in 0 ..< 64: @@ -146,18 +131,14 @@ proc main() = for _ in 0 ..< SmallSizeIters: rng.chunkTest(0 ..< 1024) - when not defined(windows): - echo "SHA256 - 64 <= size < 1024B" - for _ in 0 ..< SmallSizeIters: - rng.innerTest(0 ..< 1024) - - echo "SHA256 - 1MB <= size < 50MB" - for _ in 0 ..< LargeSizeIters: - rng.innerTest(1_000_000 ..< 50_000_000) + echo "SHA256 - 64 <= size < 1024B" + for _ in 0 ..< SmallSizeIters: + rng.innerTest(0 ..< 1024) - echo "SHA256 - Differential testing vs OpenSSL - SUCCESS" + echo "SHA256 - 1MB <= size < 50MB" + for _ in 0 ..< LargeSizeIters: + rng.innerTest(1_000_000 ..< 50_000_000) - else: - echo "SHA256 - Differential testing vs OpenSSL - [SKIPPED]" + echo "SHA256 - Differential testing vs OpenSSL - SUCCESS" main() diff --git a/tests/t_hash_sha3_vs_openssl.nim b/tests/t_hash_sha3_vs_openssl.nim new file mode 100644 index 000000000..2c4f11d9d --- /dev/null +++ b/tests/t_hash_sha3_vs_openssl.nim @@ -0,0 +1,131 @@ +import + # Internals + constantine/hashes, + # Helpers + helpers/prng_unsafe + +# Deal with platform mess +# -------------------------------------------------------------------- +when defined(windows): + when sizeof(int) == 8: + const DLLSSLName* = "(libssl-1_1-x64|ssleay64|libssl64).dll" + else: + const DLLSSLName* = "(libssl-1_1|ssleay32|libssl32).dll" +else: + when defined(macosx) or defined(macos) or defined(ios): + const versions = "(.1.1|.38|.39|.41|.43|.44|.45|.46|.47|.48|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" + else: + const versions = "(.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.48|.47|.46|.45|.44|.43|.41|.39|.38|.10|)" + + when defined(macosx) or defined(macos) or defined(ios): + const DLLSSLName* = "libssl" & versions & ".dylib" + elif defined(genode): + const DLLSSLName* = "libssl.lib.so" + else: + const DLLSSLName* = "libssl.so" & versions + +# OpenSSL wrapper +# -------------------------------------------------------------------- + +proc EVP_Q_digest[T: byte|char]( + ossl_libctx: pointer, + algoName: cstring, + propq: cstring, + data: openArray[T], + digest: var array[32, byte], + size: ptr uint): int32 {.noconv, dynlib: DLLSSLName, importc.} + +proc SHA3_256_OpenSSL[T: byte|char]( + digest: var array[32, byte], + s: openArray[T]) = + discard EVP_Q_digest(nil, "SHA3-256", nil, s, digest, nil) + +# Test +# -------------------------------------------------------------------- + +echo "\n------------------------------------------------------\n" +const SmallSizeIters = 64 +const LargeSizeIters = 1 + +proc innerTest(rng: var RngState, sizeRange: Slice[int]) = + let size = rng.random_unsafe(sizeRange) + let msg = rng.random_byte_seq(size) + + var bufCt, bufOssl: array[32, byte] + + sha3_256.hash(bufCt, msg) + SHA3_256_OpenSSL(bufOssl, msg) + doAssert bufCt == bufOssl, "Test failed with message of length " & $size + +template doWhile(a: bool, b: untyped): untyped = + ## For Keccak / SHA-3, an update MUST be called + ## before finish, hence we need do while loop + ## for empty inputs + while true: + b + if not a: + break + +proc chunkTest(rng: var RngState, sizeRange: Slice[int]) = + let size = rng.random_unsafe(sizeRange) + let msg = rng.random_byte_seq(size) + + let chunkSize = rng.random_unsafe(2 ..< 20) + + var bufOnePass: array[32, byte] + sha3_256.hash(bufOnePass, msg) + + var bufChunked: array[32, byte] + let maxChunk = max(2, sizeRange.b div 10) # Consume up to 10% at once + + var ctx: sha3_256 + ctx.init() + var cur = 0 + doWhile size - cur > 0: + let chunkSize = rng.random_unsafe(0 ..< maxChunk) + let stop = min(cur+chunkSize-1, size-1) + let consumed = stop-cur+1 + ctx.update(msg.toOpenArray(cur, stop)) + cur += consumed + + ctx.finish(bufChunked) + + doAssert bufOnePass == bufChunked + +proc main() = + echo "SHA3-256 - Starting differential testing vs OpenSSL" + + var rng: RngState + rng.seed(0xFACADE) + + echo "SHA3-256 - 0 <= size < 64 - exhaustive" + for i in 0 ..< 64: + rng.innerTest(i .. i) + + echo "SHA3-256 - 0 <= size < 64 - exhaustive chunked" + for i in 0 ..< 64: + rng.chunkTest(i .. i) + + echo "SHA3-256 - 135 <= size < 138 - exhaustive (sponge rate = 136)" + for i in 135 ..< 138: + rng.innerTest(i .. i) + + echo "SHA3-256 - 135 <= size < 138 - exhaustive chunked (sponge rate = 136)" + for i in 135 ..< 138: + rng.chunkTest(i .. i) + + echo "SHA3-256 - 64 <= size < 1024B" + for _ in 0 ..< SmallSizeIters: + rng.innerTest(0 ..< 1024) + + echo "SHA3-256 - 64 <= size < 1024B - chunked" + for _ in 0 ..< SmallSizeIters: + rng.chunkTest(0 ..< 1024) + + echo "SHA3-256 - 1MB <= size < 50MB" + for _ in 0 ..< LargeSizeIters: + rng.innerTest(1_000_000 ..< 50_000_000) + + echo "SHA3-256 - Differential testing vs OpenSSL - SUCCESS" + +main()