diff --git a/backend/groth16/bn254/icicle/icicle.go b/backend/groth16/bn254/icicle/icicle.go new file mode 100644 index 000000000..5e58db320 --- /dev/null +++ b/backend/groth16/bn254/icicle/icicle.go @@ -0,0 +1,585 @@ +//go:build icicle + +package icicle_bn254 + +import ( + "fmt" + "math/big" + "math/bits" + "os" + "time" + + curve "github.com/consensys/gnark-crypto/ecc/bn254" + "github.com/consensys/gnark-crypto/ecc/bn254/fp" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" + "github.com/consensys/gnark-crypto/ecc/bn254/fr/fft" + "github.com/consensys/gnark-crypto/ecc/bn254/fr/hash_to_field" + "github.com/consensys/gnark-crypto/ecc/bn254/fr/pedersen" + "github.com/consensys/gnark/backend" + groth16_bn254 "github.com/consensys/gnark/backend/groth16/bn254" + "github.com/consensys/gnark/backend/groth16/internal" + "github.com/consensys/gnark/backend/witness" + "github.com/consensys/gnark/constraint" + cs "github.com/consensys/gnark/constraint/bn254" + "github.com/consensys/gnark/constraint/solver" + "github.com/consensys/gnark/internal/utils" + "github.com/consensys/gnark/logger" + icicle_core "github.com/ingonyama-zk/icicle/v2/wrappers/golang/core" + icicle_cr "github.com/ingonyama-zk/icicle/v2/wrappers/golang/cuda_runtime" + icicle_bn254 "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254" + icicle_g2 "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/g2" + icicle_msm "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/msm" + icicle_ntt "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/ntt" + icicle_vecops "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/vecOps" + "github.com/rs/zerolog" + + fcs "github.com/consensys/gnark/frontend/cs" +) + +const HasIcicle = true + +func (pk *ProvingKey) setupDevicePointers() error { + if pk.deviceInfo != nil { + return nil + } + pk.deviceInfo = &deviceInfo{} + gen, _ := fft.Generator(2 * pk.Domain.Cardinality) + /************************* Den ***************************/ + n := int(pk.Domain.Cardinality) + var denI, oneI fr.Element + oneI.SetOne() + denI.Exp(gen, big.NewInt(int64(pk.Domain.Cardinality))) + denI.Sub(&denI, &oneI).Inverse(&denI) + + log2SizeFloor := bits.Len(uint(n)) - 1 + denIcicleArr := []fr.Element{denI} + for i := 0; i < log2SizeFloor; i++ { + denIcicleArr = append(denIcicleArr, denIcicleArr...) + } + pow2Remainder := n - 1< len(toRemove) +// filterHeap modifies toRemove +func filterHeap(slice []fr.Element, sliceFirstIndex int, toRemove []int) (r []fr.Element) { + + if len(toRemove) == 0 { + return slice + } + + heap := utils.IntHeap(toRemove) + heap.Heapify() + + r = make([]fr.Element, 0, len(slice)) + + // note: we can optimize that for the likely case where len(slice) >>> len(toRemove) + for i := 0; i < len(slice); i++ { + if len(heap) > 0 && i+sliceFirstIndex == heap[0] { + for len(heap) > 0 && i+sliceFirstIndex == heap[0] { + heap.Pop() + } + continue + } + r = append(r, slice[i]) + } + + return +} + +func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_core.DeviceSlice { + // H part of Krs + // Compute H (hz=ab-c, where z=-2 on ker X^n+1 (z(x)=x^n-1)) + // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) + // 2 - ca = fft_coset(_a), ba = fft_coset(_b), cc = fft_coset(_c) + // 3 - h = ifft_coset(ca o cb - cc) + _, isProfile := os.LookupEnv("profile") + startTotal := time.Now() + n := len(a) + + // add padding to ensure input length is domain cardinality + padding := make([]fr.Element, int(pk.Domain.Cardinality)-n) + a = append(a, padding...) + b = append(b, padding...) + c = append(c, padding...) + n = len(a) + + computeADone := make(chan icicle_core.DeviceSlice, 1) + computeBDone := make(chan icicle_core.DeviceSlice, 1) + computeCDone := make(chan icicle_core.DeviceSlice, 1) + + computeInttNttOnDevice := func(scalars []fr.Element, channel chan icicle_core.DeviceSlice) { + cfg := icicle_ntt.GetDefaultNttConfig() + scalarsStream, _ := icicle_cr.CreateStream() + cfg.Ctx.Stream = &scalarsStream + cfg.Ordering = icicle_core.KNM + cfg.IsAsync = true + scalarsHost := icicle_core.HostSliceFromElements(scalars) + var scalarsDevice icicle_core.DeviceSlice + scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) + start := time.Now() + icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) + cfg.Ordering = icicle_core.KMN + cfg.CosetGen = pk.CosetGenerator + icicle_ntt.Ntt(scalarsDevice, icicle_core.KForward, &cfg, scalarsDevice) + icicle_cr.SynchronizeStream(&scalarsStream) + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("computeH: NTT + INTT") + } + channel <- scalarsDevice + } + + go computeInttNttOnDevice(a, computeADone) + go computeInttNttOnDevice(b, computeBDone) + go computeInttNttOnDevice(c, computeCDone) + + aDevice := <-computeADone + bDevice := <-computeBDone + cDevice := <-computeCDone + + vecCfg := icicle_core.DefaultVecOpsConfig() + start := time.Now() + icicle_bn254.FromMontgomery(&aDevice) + icicle_vecops.VecOp(aDevice, bDevice, aDevice, vecCfg, icicle_core.Mul) + icicle_vecops.VecOp(aDevice, cDevice, aDevice, vecCfg, icicle_core.Sub) + icicle_vecops.VecOp(aDevice, pk.DenDevice, aDevice, vecCfg, icicle_core.Mul) + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("computeH: vecOps") + } + defer bDevice.Free() + defer cDevice.Free() + + cfg := icicle_ntt.GetDefaultNttConfig() + cfg.CosetGen = pk.CosetGenerator + cfg.Ordering = icicle_core.KNR + start = time.Now() + icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("computeH: INTT final") + } + icicle_bn254.FromMontgomery(&aDevice) + + if isProfile { + log.Debug().Dur("took", time.Since(startTotal)).Msg("computeH: Total") + } + return aDevice +} diff --git a/backend/groth16/bn254/icicle/noicicle.go b/backend/groth16/bn254/icicle/noicicle.go new file mode 100644 index 000000000..3e1d4ad2c --- /dev/null +++ b/backend/groth16/bn254/icicle/noicicle.go @@ -0,0 +1,23 @@ +//go:build !icicle + +package icicle_bn254 + +import ( + "fmt" + + "github.com/consensys/gnark/backend" + groth16_bn254 "github.com/consensys/gnark/backend/groth16/bn254" + "github.com/consensys/gnark/backend/witness" + cs "github.com/consensys/gnark/constraint/bn254" +) + +const HasIcicle = false + +func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...backend.ProverOption) (*groth16_bn254.Proof, error) { + return nil, fmt.Errorf("icicle backend requested but program compiled without 'icicle' build tag") +} + +func SetupDevicePointers(pk *ProvingKey) error { + fmt.Println("WARN: no icicle to use or load") + return nil +} diff --git a/backend/groth16/bn254/icicle/provingkey.go b/backend/groth16/bn254/icicle/provingkey.go new file mode 100644 index 000000000..c6bfbb2ab --- /dev/null +++ b/backend/groth16/bn254/icicle/provingkey.go @@ -0,0 +1,32 @@ +package icicle_bn254 + +import ( + "github.com/consensys/gnark-crypto/ecc/bn254/fr" + groth16_bn254 "github.com/consensys/gnark/backend/groth16/bn254" + cs "github.com/consensys/gnark/constraint/bn254" + icicle_core "github.com/ingonyama-zk/icicle/v2/wrappers/golang/core" +) + +type deviceInfo struct { + CosetGenerator [fr.Limbs * 2]uint32 + G1Device struct { + A, B, K, Z icicle_core.DeviceSlice + } + G2Device struct { + B icicle_core.DeviceSlice + } + DenDevice icicle_core.DeviceSlice +} + +type ProvingKey struct { + *groth16_bn254.ProvingKey + *deviceInfo +} + +func Setup(r1cs *cs.R1CS, pk *ProvingKey, vk *groth16_bn254.VerifyingKey) error { + return groth16_bn254.Setup(r1cs, pk.ProvingKey, vk) +} + +func DummySetup(r1cs *cs.R1CS, pk *ProvingKey) error { + return groth16_bn254.DummySetup(r1cs, pk.ProvingKey) +} diff --git a/backend/groth16/groth16.go b/backend/groth16/groth16.go index 84b6b3460..620d50bea 100644 --- a/backend/groth16/groth16.go +++ b/backend/groth16/groth16.go @@ -20,7 +20,6 @@ package groth16 import ( - icicle_bw6761 "github.com/consensys/gnark/backend/groth16/bw6-761/icicle" "io" "github.com/consensys/gnark-crypto/ecc" @@ -46,13 +45,16 @@ import ( gnarkio "github.com/consensys/gnark/io" groth16_bls12377 "github.com/consensys/gnark/backend/groth16/bls12-377" - icicle_bls12377 "github.com/consensys/gnark/backend/groth16/bls12-377/icicle" groth16_bls12381 "github.com/consensys/gnark/backend/groth16/bls12-381" groth16_bls24315 "github.com/consensys/gnark/backend/groth16/bls24-315" groth16_bls24317 "github.com/consensys/gnark/backend/groth16/bls24-317" groth16_bn254 "github.com/consensys/gnark/backend/groth16/bn254" groth16_bw6633 "github.com/consensys/gnark/backend/groth16/bw6-633" groth16_bw6761 "github.com/consensys/gnark/backend/groth16/bw6-761" + + icicle_bls12377 "github.com/consensys/gnark/backend/groth16/bls12-377/icicle" + icicle_bn254 "github.com/consensys/gnark/backend/groth16/bn254/icicle" + icicle_bw6761 "github.com/consensys/gnark/backend/groth16/bw6-761/icicle" ) type groth16Object interface { @@ -178,10 +180,11 @@ func Prove(r1cs constraint.ConstraintSystem, pk ProvingKey, fullWitness witness. case *cs_bls12381.R1CS: return groth16_bls12381.Prove(_r1cs, pk.(*groth16_bls12381.ProvingKey), fullWitness, opts...) - case *cs_bn254.R1CS: + if icicle_bn254.HasIcicle { + return icicle_bn254.Prove(_r1cs, pk.(*icicle_bn254.ProvingKey), fullWitness, opts...) + } return groth16_bn254.Prove(_r1cs, pk.(*groth16_bn254.ProvingKey), fullWitness, opts...) - case *cs_bw6761.R1CS: if icicle_bw6761.HasIcicle { return icicle_bw6761.Prove(_r1cs, pk.(*icicle_bw6761.ProvingKey), fullWitness, opts...) @@ -236,6 +239,9 @@ func Setup(r1cs constraint.ConstraintSystem) (ProvingKey, VerifyingKey, error) { if err := groth16_bn254.Setup(_r1cs, &pk, &vk); err != nil { return nil, nil, err } + if icicle_bn254.HasIcicle { + return &icicle_bn254.ProvingKey{ProvingKey: &pk}, &vk, nil + } return &pk, &vk, nil case *cs_bw6761.R1CS: var vk groth16_bw6761.VerifyingKey @@ -297,6 +303,13 @@ func DummySetup(r1cs constraint.ConstraintSystem) (ProvingKey, error) { } return &pk, nil case *cs_bn254.R1CS: + if icicle_bn254.HasIcicle { + var pk icicle_bn254.ProvingKey + if err := icicle_bn254.DummySetup(_r1cs, &pk); err != nil { + return nil, err + } + return &pk, nil + } var pk groth16_bn254.ProvingKey if err := groth16_bn254.DummySetup(_r1cs, &pk); err != nil { return nil, err @@ -345,6 +358,11 @@ func NewProvingKey(curveID ecc.ID) ProvingKey { switch curveID { case ecc.BN254: pk = &groth16_bn254.ProvingKey{} + if icicle_bn254.HasIcicle { + pk = &icicle_bn254.ProvingKey{ + ProvingKey: &groth16_bn254.ProvingKey{}, + } + } case ecc.BLS12_377: pk = &groth16_bls12377.ProvingKey{} if icicle_bls12377.HasIcicle {