Kyber C extraction (#153)

* Various rewrites to get C extraction working

* Bash script to automate extracting the Kyber code to C

* Python script to move the Kyber C extraction into NSS.

---------

Co-authored-by: Jonathan Protzenko <[email protected]>
Co-authored-by: xvzcf <[email protected]>

.gitignore

@@ -10,3 +10,4 @@ fuzz/corpus
 fuzz/artifacts
 proofs/fstar/extraction/.cache
 __pycache__
+kyber-crate/

add-c-kyber-to-nss.py

@@ -0,0 +1,133 @@
+#! /usr/bin/env python3
+
+import os
+import subprocess
+import re
+import shutil
+import argparse
+
+
+def shell(command, expect=0, cwd=None, env={}):
+    subprocess_stdout = subprocess.DEVNULL
+
+    os_env = os.environ
+    os_env.update(env)
+
+    result = subprocess.run(
+        command, cwd=cwd, env=os_env, capture_output=True, text=True
+    )
+
+    if result.returncode != expect:
+        raise Exception("Error {}. Expected {}.".format(result, expect))
+
+    return result.stdout
+
+
+def add_libcrux_kyber_h(c_extraction_root, freebl_verified_root):
+    path_to_header = os.path.join(c_extraction_root, "libcrux_kyber.h")
+    destination = os.path.join(freebl_verified_root, "internal", "Libcrux_Kyber_768.h")
+
+    shell(["clang-format", "-i", "-style=Google", destination])
+
+    with open(destination, "r") as f:
+        original = f.read()
+        replaced = re.sub("extern void libcrux_digest_sha3_512.*\n", "", original)
+        replaced = re.sub("extern void libcrux_digest_sha3_256.*\n", "", replaced)
+    with open(destination, "w") as f:
+        f.write(replaced)
+
+    shell(["clang-format", "-i", "-style=Mozilla", destination])
+
+
+def add_libcrux_kyber_c(c_extraction_root, freebl_verified_root):
+    path_to_c_file = os.path.join(c_extraction_root, "libcrux_kyber.c")
+    destination = os.path.join(freebl_verified_root, "Libcrux_Kyber_768.c")
+    shutil.copyfile(path_to_c_file, destination)
+
+    shell(["clang-format", "-i", "-style=Google", destination])
+
+    sed_cmd = shutil.which("gsed")
+    if sed_cmd is None:
+        sed_cmd = shutil.which("sed")
+
+    ctags = shell(["ctags", "--fields=+ne", "-o", "-", destination])
+    sed_input = ""
+    for line in ctags.splitlines():
+        if (
+            "libcrux_kyber_serialize_compress_then_serialize_11___320size_t" in line
+            or "libcrux_kyber_serialize_compress_then_serialize_5___128size_t" in line
+        ):
+            line_start = re.findall(r"line:(\d+)", line)[0]
+            line_end = re.findall(r"end:(\d+)", line)[0]
+            sed_input = "{},{}d;{}".format(line_start, line_end, sed_input)
+
+    shell([sed_cmd, "-i", sed_input, destination])
+
+    with open(destination, "r") as f:
+        original = f.read()
+        replaced = re.sub(
+            '#include "libcrux_kyber.h"',
+            '#include "internal/Libcrux_Kyber_768.h"',
+            original,
+        )
+        replaced = re.sub(
+            '#include "libcrux_hacl_glue.h"',
+            '#include "Libcrux_Kyber_Hash_Functions.h"',
+            replaced,
+        )
+        replaced = re.sub("uu____0 = !false", "uu____0 = false", replaced)
+    with open(destination, "w") as f:
+        f.write(replaced)
+
+    shell(["clang-format", "-i", "-style=Mozilla", destination])
+
+
+def add_internal_core_h(c_extraction_root, freebl_verified_root):
+    src_file = os.path.join(c_extraction_root, "internal", "core.h")
+    destination = os.path.join(freebl_verified_root, "internal", "core.h")
+
+    shutil.copyfile(src_file, destination)
+    shell(["clang-format", "-i", "-style=Mozilla", destination])
+
+
+def add_Eurydice_h(c_extraction_root, freebl_verified_root):
+    src_file = os.path.join(c_extraction_root, "Eurydice.h")
+    destination = os.path.join(freebl_verified_root, "eurydice", "Eurydice.h")
+
+    shutil.copyfile(src_file, destination)
+    shell(["clang-format", "-i", "-style=Mozilla", destination])
+
+
+def add_eurydice_glue_h(c_extraction_root, freebl_verified_root):
+    src_file = os.path.join(c_extraction_root, "eurydice_glue.h")
+    destination = os.path.join(freebl_verified_root, "eurydice", "eurydice_glue.h")
+
+    shutil.copyfile(src_file, destination)
+    shell(["clang-format", "-i", "-style=Mozilla", destination])
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--nss-root",
+    required=True,
+    help="Absolute or relative path to the root directory containing the NSS source code.",
+    type=os.path.abspath,
+)
+parser.add_argument(
+    "--kyber-c-root",
+    required=True,
+    help="Absolute or relative path to the root directory containing the extracted Kyber C code.",
+    type=os.path.abspath,
+)
+args = parser.parse_args()
+
+nss_root = args.nss_root
+freebl_verified_root = os.path.join(nss_root, "lib", "freebl", "verified")
+
+c_extraction_root = args.kyber_c_root
+
+add_libcrux_kyber_h(c_extraction_root, freebl_verified_root)
+add_libcrux_kyber_c(c_extraction_root, freebl_verified_root)
+add_internal_core_h(c_extraction_root, freebl_verified_root)
+add_Eurydice_h(c_extraction_root, freebl_verified_root)
+add_eurydice_glue_h(c_extraction_root, freebl_verified_root)

kyber-crate-tests/kyber.rs

@@ -0,0 +1,200 @@
+use libcrux::{
+    digest::shake256,
+    kem::{self, Algorithm, Ct, PrivateKey},
+};
+
+use rand::{CryptoRng, Rng};
+
+use rand::rngs::OsRng;
+
+const SHARED_SECRET_SIZE: usize = 32;
+
+macro_rules! impl_consistency {
+    ($name:ident, $alg:expr) => {
+        #[test]
+        fn $name() {
+            let mut rng = OsRng;
+
+            if let Ok((secret_key, public_key)) = kem::key_gen($alg, &mut rng) {
+                if let Ok((shared_secret, ciphertext)) = kem::encapsulate(&public_key, &mut rng) {
+                    let shared_secret_decapsulated =
+                        kem::decapsulate(&ciphertext, &secret_key).unwrap();
+                    assert_eq!(shared_secret.encode(), shared_secret_decapsulated.encode());
+                }
+            }
+
+            // If the randomness was not enough for the rejection sampling step
+            // in key-generation and encapsulation, simply return without
+            // failing.
+        }
+    };
+}
+
+fn modify_ciphertext(alg: Algorithm, rng: &mut (impl CryptoRng + Rng), ciphertext: Ct) -> Ct {
+    let mut raw_ciphertext = ciphertext.encode();
+
+    let mut random_u32: usize = rng.next_u32().try_into().unwrap();
+
+    let mut random_byte: u8 = (random_u32 & 0xFF) as u8;
+    if random_byte == 0 {
+        random_byte += 1;
+    }
+    random_u32 >>= 8;
+
+    let position = random_u32 % raw_ciphertext.len();
+    raw_ciphertext[position] ^= random_byte;
+
+    Ct::decode(alg, &raw_ciphertext).unwrap()
+}
+
+macro_rules! impl_modified_ciphertext {
+    ($name:ident, $alg:expr) => {
+        #[test]
+        fn $name() {
+            let mut rng = OsRng;
+
+            if let Ok((secret_key, public_key)) = kem::key_gen($alg, &mut rng) {
+                if let Ok((shared_secret, ciphertext)) = kem::encapsulate(&public_key, &mut rng) {
+                    let ciphertext = modify_ciphertext($alg, &mut rng, ciphertext);
+                    let shared_secret_decapsulated =
+                        kem::decapsulate(&ciphertext, &secret_key).unwrap();
+
+                    assert_ne!(shared_secret.encode(), shared_secret_decapsulated.encode());
+                }
+            }
+            // if the randomness was not enough for the rejection sampling step
+            // in key-generation and encapsulation, simply return without
+            // failing.
+        }
+    };
+}
+
+fn modify_secret_key(
+    alg: Algorithm,
+    rng: &mut (impl CryptoRng + Rng),
+    secret_key: PrivateKey,
+    modify_implicit_rejection_value: bool,
+) -> PrivateKey {
+    let mut raw_secret_key = secret_key.encode();
+
+    let mut random_u32: usize = rng.next_u32().try_into().unwrap();
+
+    let mut random_byte: u8 = (random_u32 & 0xFF) as u8;
+    if random_byte == 0 {
+        random_byte += 1;
+    }
+    random_u32 >>= 8;
+
+    let position = if modify_implicit_rejection_value {
+        (raw_secret_key.len() - SHARED_SECRET_SIZE) + (random_u32 % SHARED_SECRET_SIZE)
+    } else {
+        random_u32 % (raw_secret_key.len() - SHARED_SECRET_SIZE)
+    };
+
+    raw_secret_key[position] ^= random_byte;
+
+    PrivateKey::decode(alg, &raw_secret_key).unwrap()
+}
+
+fn compute_implicit_rejection_shared_secret(
+    ciphertext: Ct,
+    secret_key: PrivateKey,
+) -> [u8; SHARED_SECRET_SIZE] {
+    let raw_secret_key = secret_key.encode();
+
+    let mut to_hash = raw_secret_key[raw_secret_key.len() - SHARED_SECRET_SIZE..].to_vec();
+    to_hash.extend_from_slice(&ciphertext.encode());
+
+    shake256(&to_hash)
+}
+
+macro_rules! impl_modified_secret_key {
+    ($name:ident, $alg:expr) => {
+        #[test]
+        fn $name() {
+            let mut rng = OsRng;
+
+            if let Ok((secret_key, public_key)) = kem::key_gen($alg, &mut rng) {
+                if let Ok((shared_secret, ciphertext)) = kem::encapsulate(&public_key, &mut rng) {
+                    let secret_key = modify_secret_key($alg, &mut rng, secret_key, false);
+                    let shared_secret_decapsulated =
+                        kem::decapsulate(&ciphertext, &secret_key).unwrap();
+                    assert_ne!(shared_secret.encode(), shared_secret_decapsulated.encode());
+
+                    let secret_key = modify_secret_key($alg, &mut rng, secret_key, true);
+                    let shared_secret_decapsulated =
+                        kem::decapsulate(&ciphertext, &secret_key).unwrap();
+
+                    assert_eq!(
+                        shared_secret_decapsulated.encode(),
+                        compute_implicit_rejection_shared_secret(ciphertext, secret_key)
+                    );
+                }
+            }
+
+            // if the randomness was not enough for the rejection sampling step
+            // in key-generation and encapsulation, simply return without
+            // failing.
+        }
+    };
+}
+
+macro_rules! impl_modified_ciphertext_and_implicit_rejection_value {
+    ($name:ident, $alg:expr) => {
+        #[test]
+        fn $name() {
+            let mut rng = OsRng;
+
+            if let Ok((secret_key, public_key)) = kem::key_gen($alg, &mut rng) {
+                if let Ok((_, ciphertext)) = kem::encapsulate(&public_key, &mut rng) {
+                    let ciphertext = modify_ciphertext($alg, &mut rng, ciphertext);
+                    let shared_secret_decapsulated =
+                        kem::decapsulate(&ciphertext, &secret_key).unwrap();
+
+                    let secret_key = modify_secret_key($alg, &mut rng, secret_key, true);
+                    let shared_secret_decapsulated_1 =
+                        kem::decapsulate(&ciphertext, &secret_key).unwrap();
+
+                    assert_ne!(
+                        shared_secret_decapsulated.encode(),
+                        shared_secret_decapsulated_1.encode()
+                    );
+
+                    assert_eq!(
+                        shared_secret_decapsulated_1.encode(),
+                        compute_implicit_rejection_shared_secret(ciphertext, secret_key)
+                    );
+                }
+            }
+
+            // if the randomness was not enough for the rejection sampling step
+            // in key-generation and encapsulation, simply return without
+            // failing.
+        }
+    };
+}
+
+impl_consistency!(consistency_512, Algorithm::Kyber512);
+impl_consistency!(consistency_768, Algorithm::Kyber768);
+impl_consistency!(consistency_1024, Algorithm::Kyber1024);
+
+impl_modified_ciphertext!(modified_ciphertext_512, Algorithm::Kyber512);
+impl_modified_ciphertext!(modified_ciphertext_768, Algorithm::Kyber768);
+impl_modified_ciphertext!(modified_ciphertext_1024, Algorithm::Kyber1024);
+
+impl_modified_secret_key!(modified_secret_key_512, Algorithm::Kyber512);
+impl_modified_secret_key!(modified_secret_key_768, Algorithm::Kyber768);
+impl_modified_secret_key!(modified_secret_key_1024, Algorithm::Kyber1024);
+
+impl_modified_ciphertext_and_implicit_rejection_value!(
+    modified_ciphertext_and_implicit_rejection_value_512,
+    Algorithm::Kyber512
+);
+impl_modified_ciphertext_and_implicit_rejection_value!(
+    modified_ciphertext_and_implicit_rejection_value_768,
+    Algorithm::Kyber768
+);
+impl_modified_ciphertext_and_implicit_rejection_value!(
+    modified_ciphertext_and_implicit_rejection_value_1024,
+    Algorithm::Kyber1024
+);

kyber-crate-tests/kyber_kats/README.md

@@ -0,0 +1 @@
+In order to regenerate the JSON KAT files for all parameter sets, simply run `./generate_kats.py`.