avx2 mlkem

code/jasmin/mlkem_avx2/Makefile

@@ -0,0 +1,141 @@
+# -*- Makefile -*-
+
+
+-include ../../Makefile.conf
+
+CC     ?= /usr/bin/gcc
+GFLAGS ?=
+CFLAGS := -Wall -Wextra -g -Ofast -fomit-frame-pointer
+JFLAGS := -lea ${JADDFLAGS}
+OS     := $(shell uname -s)
+
+.SECONDARY: jpoly.s jpolyvec.s jfips202.s jindcpa.s jindcpa.o jkem.s
+
+default: test speed
+
+test: test/test_poly_compress \
+      test/test_poly_decompress \
+      test/test_poly_tobytes \
+      test/test_poly_frombytes \
+      test/test_poly_tomsg \
+      test/test_poly_frommsg \
+      test/test_poly_add2 \
+      test/test_poly_sub \
+      test/test_poly_ntt \
+      test/test_poly_invntt \
+      test/test_poly_basemul \
+      test/test_poly_frommont \
+      test/test_poly_reduce \
+      test/test_poly_csubq \
+      test/test_poly_getnoise \
+      test/test_polyvec_compress\
+      test/test_polyvec_decompress\
+      test/test_polyvec_tobytes \
+      test/test_polyvec_frombytes \
+      test/test_polyvec_add2 \
+      test/test_polyvec_ntt \
+      test/test_polyvec_invntt \
+      test/test_polyvec_pointwise_acc \
+      test/test_polyvec_reduce\
+      test/test_polyvec_csubq \
+      test/test_fips202 \
+	    test/test_indcpa \
+	    test/test_kem
+
+speed: test/speed_indcpa \
+       test/speed_kyber
+
+HEADERS = params.h poly.h fips202.h ntt.h indcpa.h kem.h \
+
+JHEADERS = params.jinc \
+           reduce.jinc \
+           fips202_common.jinc \
+           fips202.jinc \
+           fips202_4x.jinc \
+           keccakf1600.jinc \
+           consts.jinc \
+           shuffle.jinc \
+           indcpa.jinc \
+           verify.jinc
+
+POLYHEADERS = poly.jinc \
+							consts.jinc \
+
+POLYVECHEADERS = polyvec.jinc \
+                 gen_matrix.jinc \
+
+INCS    = fq.inc shuffle.inc
+SOURCES = poly.c polyvec.c cbd.c fips202.c ntt.c reduce.c symmetric-fips202.c indcpa.c kem.c consts.c shuffle.S fq.S\
+
+test/test_indcpa: test/test_indcpa.c $(HEADERS) $(SOURCES) $(INCS) jindcpa.o
+	$(CC) $(CFLAGS) -o $@ $(SOURCES) jindcpa.o $<
+
+test/test_kem: test/test_kem.c $(HEADERS) $(SOURCES) $(INCS) jkem.o
+	$(CC) $(CFLAGS) -o $@ $(SOURCES) jkem.o ~/Desktop/Repos/jasmin/compiler/syscall/jasmin_syscall.o $<
+
+test/speed_indcpa: test/speed_indcpa.c $(HEADERS) $(SOURCES) $(INCS) jindcpa.o
+	$(CC) $(CFLAGS) -o $@ $(SOURCES) jindcpa.o $<
+
+test/speed_kyber: test/speed_kyber.c $(HEADERS) $(SOURCES) $(INCS) jspeed.s
+	$(CC) $(CFLAGS) -o $@ $(SOURCES) jspeed.s $<
+
+test/test_fips202: test/test_fips202.c $(HEADERS) fips202.c jfips202.s
+	$(CC) $(CFLAGS) -o $@ fips202.c jfips202.s $<
+
+test/test_gen_matrix: test/test_gen_matrix.c $(HEADERS) gen_matrix.s
+	$(CC) $(CFLAGS) -o $@ gen_matrix.s $<
+
+test/test_poly_%: test/test_poly_%.c $(HEADERS) $(SOURCES) $(INCS) jpoly.s 
+	$(CC) $(CFLAGS) -o $@ $(SOURCES) jpoly.s $<
+
+test/test_polyvec_%: test/test_polyvec_%.c $(HEADERS) $(SOURCES) $(INCS) jpolyvec.s
+	$(CC) $(CFLAGS) -o $@ $(SOURCES) jpolyvec.s $<
+
+%.s: %.jazz
+	$(JASMINC) -o $@ $(JFLAGS) $^
+
+.PHONY: clean
+
+clean:
+	-rm -f *.o
+	-rm -f gen_matrix.s
+	-rm -f jindcpa.s
+	-rm -f jkem.s
+	-rm -f jfips202.s
+	-rm -f jpoly.s
+	-rm -f jpolyvec.s
+	-rm -f jspeed.s
+	-rm -f test/test_poly_compress
+	-rm -f test/test_poly_decompress
+	-rm -f test/test_poly_tobytes
+	-rm -f test/test_poly_frombytes
+	-rm -f test/test_poly_tomsg
+	-rm -f test/test_poly_frommsg
+	-rm -f test/test_poly_add2
+	-rm -f test/test_poly_sub
+	-rm -f test/test_poly_ntt
+	-rm -f test/test_poly_invntt
+	-rm -f test/test_poly_basemul
+	-rm -f test/test_poly_frommont
+	-rm -f test/test_poly_reduce
+	-rm -f test/test_poly_csubq
+	-rm -f test/test_poly_getnoise
+	-rm -f test/test_polyvec_compress
+	-rm -f test/test_polyvec_decompress
+	-rm -f test/test_polyvec_tobytes
+	-rm -f test/test_polyvec_frombytes
+	-rm -f test/test_polyvec_add2
+	-rm -f test/test_polyvec_ntt
+	-rm -f test/test_polyvec_invntt
+	-rm -f test/test_polyvec_pointwise_acc
+	-rm -f test/test_polyvec_reduce
+	-rm -f test/test_polyvec_csubq
+	-rm -f test/test_fips202
+	-rm -f test/test_gen_matrix
+	-rm -f test/test_indcpa
+	-rm -f test/test_kem
+	-rm -f test/speed_indcpa
+	-rm -f test/speed_kyber
+ifeq ($(OS),Darwin)
+	-rm -r -f test/*.dSYM
+endif

code/jasmin/mlkem_avx2/cbd.c

@@ -0,0 +1,128 @@
+#include <stdint.h>
+#include "params.h"
+#include "cbd.h"
+
+/*************************************************
+* Name:        load32_littleendian
+*
+* Description: load bytes into a 32-bit integer
+*              in little-endian order
+*
+* Arguments:   - const unsigned char *x: pointer to input byte array
+*
+* Returns 32-bit unsigned integer loaded from x
+**************************************************/
+static uint32_t load32_littleendian(const unsigned char *x)
+{
+  uint32_t r;
+  r  = (uint32_t)x[0];
+  r |= (uint32_t)x[1] << 8;
+  r |= (uint32_t)x[2] << 16;
+  r |= (uint32_t)x[3] << 24;
+  return r;
+}
+
+/*************************************************
+* Name:        load24_littleendian
+*
+* Description: load 3 bytes into a 32-bit integer
+*              in little-endian order.
+*              This function is only needed for Kyber-512
+*
+* Arguments:   - const uint8_t *x: pointer to input byte array
+*
+* Returns 32-bit unsigned integer loaded from x (most significant byte is zero)
+**************************************************/
+#if KYBER_ETA1 == 3
+static uint32_t load24_littleendian(const uint8_t x[3])
+{
+  uint32_t r;
+  r  = (uint32_t)x[0];
+  r |= (uint32_t)x[1] << 8;
+  r |= (uint32_t)x[2] << 16;
+  return r;
+}
+#endif
+
+
+/*************************************************
+* Name:        cbd2
+*
+* Description: Given an array of uniformly random bytes, compute
+*              polynomial with coefficients distributed according to
+*              a centered binomial distribution with parameter eta=2
+*
+* Arguments:   - poly *r: pointer to output polynomial
+*              - const uint8_t *buf: pointer to input byte array
+**************************************************/
+static void cbd2(poly *r, const uint8_t buf[2*KYBER_N/4])
+{
+  unsigned int i,j;
+  uint32_t t,d;
+  int16_t a,b;
+
+  for(i=0;i<KYBER_N/8;i++) {
+    t  = load32_littleendian(buf+4*i);
+    d  = t & 0x55555555;
+    d += (t>>1) & 0x55555555;
+
+    for(j=0;j<8;j++) {
+      a = (d >> (4*j+0)) & 0x3;
+      b = (d >> (4*j+2)) & 0x3;
+      r->coeffs[8*i+j] = a - b;
+    }
+  }
+}
+
+/*************************************************
+* Name:        cbd3
+*
+* Description: Given an array of uniformly random bytes, compute
+*              polynomial with coefficients distributed according to
+*              a centered binomial distribution with parameter eta=3.
+*              This function is only needed for Kyber-512
+*
+* Arguments:   - poly *r: pointer to output polynomial
+*              - const uint8_t *buf: pointer to input byte array
+**************************************************/
+#if KYBER_ETA1 == 3
+static void cbd3(poly *r, const uint8_t buf[3*KYBER_N/4])
+{
+  unsigned int i,j;
+  uint32_t t,d;
+  int16_t a,b;
+
+  for(i=0;i<KYBER_N/4;i++) {
+    t  = load24_littleendian(buf+3*i);
+    d  = t & 0x00249249;
+    d += (t>>1) & 0x00249249;
+    d += (t>>2) & 0x00249249;
+
+    for(j=0;j<4;j++) {
+      a = (d >> (6*j+0)) & 0x7;
+      b = (d >> (6*j+3)) & 0x7;
+      r->coeffs[4*i+j] = a - b;
+    }
+  }
+}
+#endif
+
+void poly_cbd_eta1(poly *r, const uint8_t buf[KYBER_ETA1*KYBER_N/4])
+{
+#if KYBER_ETA1 == 2
+  cbd2(r, buf);
+#elif KYBER_ETA1 == 3
+  cbd3(r, buf);
+#else
+#error "This implementation requires eta1 in {2,3}"
+#endif
+}
+
+void poly_cbd_eta2(poly *r, const uint8_t buf[KYBER_ETA2*KYBER_N/4])
+{
+#if KYBER_ETA2 == 2
+  cbd2(r, buf);
+#else
+#error "This implementation requires eta2 = 2"
+#endif
+}

code/jasmin/mlkem_avx2/cbd.h

@@ -0,0 +1,9 @@
+#ifndef CBD_H
+#define CBD_H
+
+#include "poly.h"
+
+void poly_cbd_eta1(poly *r, const uint8_t buf[KYBER_ETA1*KYBER_N/4]);
+void poly_cbd_eta2(poly *r, const uint8_t buf[KYBER_ETA2*KYBER_N/4]);
+
+#endif