From b08bc0177798bf41b059981ccbc999ceb29c1cb0 Mon Sep 17 00:00:00 2001 From: Andrew Gouin Date: Thu, 5 Dec 2024 15:29:33 -0700 Subject: [PATCH] binary encoding --- gturbine/gtencoding/binary_encoder.go | 88 ++++++++++ gturbine/gtencoding/binary_encoder_test.go | 154 ++++++++++++++++++ gturbine/gtencoding/encoder.go | 8 + .../process_shred_test.go | 2 +- .../{shredding => gtshredding}/processor.go | 2 +- .../processor_test.go | 2 +- .../{shredding => gtshredding}/shred_group.go | 3 +- 7 files changed, 255 insertions(+), 4 deletions(-) create mode 100644 gturbine/gtencoding/binary_encoder.go create mode 100644 gturbine/gtencoding/binary_encoder_test.go create mode 100644 gturbine/gtencoding/encoder.go rename gturbine/{shredding => gtshredding}/process_shred_test.go (99%) rename gturbine/{shredding => gtshredding}/processor.go (99%) rename gturbine/{shredding => gtshredding}/processor_test.go (99%) rename gturbine/{shredding => gtshredding}/shred_group.go (98%) diff --git a/gturbine/gtencoding/binary_encoder.go b/gturbine/gtencoding/binary_encoder.go new file mode 100644 index 0000000..6b31f2d --- /dev/null +++ b/gturbine/gtencoding/binary_encoder.go @@ -0,0 +1,88 @@ +package gtencoding + +import ( + "encoding/binary" + "fmt" + + "github.com/google/uuid" + "github.com/gordian-engine/gordian/gturbine" +) + +const ( + intSize = 8 + uuidSize = 16 + blockHashSize = 32 + prefixSize = intSize*5 + uuidSize + blockHashSize +) + +// BinaryShardCodec represents a codec for encoding and decoding shreds +type BinaryShardCodec struct{} + +func (bsc *BinaryShardCodec) Encode(shred gturbine.Shred) ([]byte, error) { + out := make([]byte, prefixSize+len(shred.Data)) + + // Write full data size + binary.LittleEndian.PutUint64(out[:8], uint64(shred.FullDataSize)) + + // Write block hash + copy(out[8:40], shred.BlockHash) + + uid, err := uuid.Parse(shred.GroupID) + if err != nil { + return nil, fmt.Errorf("failed to parse group ID: %w", err) + } + // Write group ID + copy(out[40:56], uid[:]) + + // Write height + binary.LittleEndian.PutUint64(out[56:64], shred.Height) + + // Write index + binary.LittleEndian.PutUint64(out[64:72], uint64(shred.Index)) + + // Write total data shreds + binary.LittleEndian.PutUint64(out[72:80], uint64(shred.TotalDataShreds)) + + // Write total recovery shreds + binary.LittleEndian.PutUint64(out[80:88], uint64(shred.TotalRecoveryShreds)) + + // Write data + copy(out[prefixSize:], shred.Data) + + return out, nil + +} + +func (bsc *BinaryShardCodec) Decode(data []byte) (gturbine.Shred, error) { + shred := gturbine.Shred{} + + // Read full data size + shred.FullDataSize = int(binary.LittleEndian.Uint64(data[:8])) + + // Read block hash + shred.BlockHash = make([]byte, blockHashSize) + copy(shred.BlockHash, data[8:40]) + + // Read group ID + uid := uuid.UUID{} + copy(uid[:], data[40:56]) + shred.GroupID = uid.String() + + // Read height + shred.Height = binary.LittleEndian.Uint64(data[56:64]) + + // Read index + shred.Index = int(binary.LittleEndian.Uint64(data[64:72])) + + // Read total data shreds + shred.TotalDataShreds = int(binary.LittleEndian.Uint64(data[72:80])) + + // Read total recovery shreds + shred.TotalRecoveryShreds = int(binary.LittleEndian.Uint64(data[80:88])) + + // Read data + shred.Data = make([]byte, len(data)-prefixSize) + copy(shred.Data, data[prefixSize:]) + + return shred, nil +} diff --git a/gturbine/gtencoding/binary_encoder_test.go b/gturbine/gtencoding/binary_encoder_test.go new file mode 100644 index 0000000..2dfd415 --- /dev/null +++ b/gturbine/gtencoding/binary_encoder_test.go @@ -0,0 +1,154 @@ +package gtencoding + +import ( + "bytes" + "testing" + + "github.com/google/uuid" + "github.com/gordian-engine/gordian/gturbine" +) + +func TestBinaryShardCodec_EncodeDecode(t *testing.T) { + tests := []struct { + name string + shred gturbine.Shred + wantErr bool + }{ + { + name: "basic encode/decode", + shred: gturbine.Shred{ + FullDataSize: 1000, + BlockHash: bytes.Repeat([]byte{1}, 32), + GroupID: uuid.New().String(), + Height: 12345, + Index: 5, + TotalDataShreds: 10, + TotalRecoveryShreds: 2, + Data: []byte("test data"), + }, + wantErr: false, + }, + { + name: "empty data", + shred: gturbine.Shred{ + FullDataSize: 0, + BlockHash: bytes.Repeat([]byte{2}, 32), + GroupID: uuid.New().String(), + Height: 67890, + Index: 0, + TotalDataShreds: 1, + TotalRecoveryShreds: 0, + Data: []byte{}, + }, + wantErr: false, + }, + { + name: "large data", + shred: gturbine.Shred{ + FullDataSize: 1000000, + BlockHash: bytes.Repeat([]byte{3}, 32), + GroupID: uuid.New().String(), + Height: 999999, + Index: 50, + TotalDataShreds: 100, + TotalRecoveryShreds: 20, + Data: bytes.Repeat([]byte("large data"), 1000), + }, + wantErr: false, + }, + } + + codec := &BinaryShardCodec{} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test encoding + encoded, err := codec.Encode(tt.shred) + if (err != nil) != tt.wantErr { + t.Errorf("BinaryShardCodec.Encode() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.wantErr { + return + } + + // Test decoding + decoded, err := codec.Decode(encoded) + if err != nil { + t.Errorf("BinaryShardCodec.Decode() error = %v", err) + return + } + + // Verify all fields match + if decoded.FullDataSize != tt.shred.FullDataSize { + t.Errorf("FullDataSize mismatch: got %v, want %v", decoded.FullDataSize, tt.shred.FullDataSize) + } + + if !bytes.Equal(decoded.BlockHash, tt.shred.BlockHash) { + t.Errorf("BlockHash mismatch: got %v, want %v", decoded.BlockHash, tt.shred.BlockHash) + } + + if decoded.GroupID != tt.shred.GroupID { + t.Errorf("GroupID mismatch: got %v, want %v", decoded.GroupID, tt.shred.GroupID) + } + + if decoded.Height != tt.shred.Height { + t.Errorf("Height mismatch: got %v, want %v", decoded.Height, tt.shred.Height) + } + + if decoded.Index != tt.shred.Index { + t.Errorf("Index mismatch: got %v, want %v", decoded.Index, tt.shred.Index) + } + + if decoded.TotalDataShreds != tt.shred.TotalDataShreds { + t.Errorf("TotalDataShreds mismatch: got %v, want %v", decoded.TotalDataShreds, tt.shred.TotalDataShreds) + } + + if decoded.TotalRecoveryShreds != tt.shred.TotalRecoveryShreds { + t.Errorf("TotalRecoveryShreds mismatch: got %v, want %v", decoded.TotalRecoveryShreds, tt.shred.TotalRecoveryShreds) + } + + if !bytes.Equal(decoded.Data, tt.shred.Data) { + t.Errorf("Data mismatch: got %v, want %v", decoded.Data, tt.shred.Data) + } + }) + } +} + +func TestBinaryShardCodec_InvalidGroupID(t *testing.T) { + codec := &BinaryShardCodec{} + shred := gturbine.Shred{ + GroupID: "invalid-uuid", + // Other fields can be empty for this test + } + + _, err := codec.Encode(shred) + if err == nil { + t.Error("Expected error when encoding invalid GroupID, got nil") + } +} + +func TestBinaryShardCodec_DataSizes(t *testing.T) { + codec := &BinaryShardCodec{} + shred := gturbine.Shred{ + FullDataSize: 1000, + BlockHash: bytes.Repeat([]byte{1}, 32), + GroupID: uuid.New().String(), + Height: 12345, + Index: 5, + TotalDataShreds: 10, + TotalRecoveryShreds: 2, + Data: []byte("test data"), + } + + encoded, err := codec.Encode(shred) + if err != nil { + t.Fatalf("Failed to encode shred: %v", err) + } + + expectedPrefixSize := intSize*5 + uuidSize + blockHashSize + if len(encoded) != expectedPrefixSize+len(shred.Data) { + t.Errorf("Encoded data size mismatch: got %v, want %v", len(encoded), expectedPrefixSize+len(shred.Data)) + } +} diff --git a/gturbine/gtencoding/encoder.go b/gturbine/gtencoding/encoder.go new file mode 100644 index 0000000..9cbe7cb --- /dev/null +++ b/gturbine/gtencoding/encoder.go @@ -0,0 +1,8 @@ +package gtencoding + +import "github.com/gordian-engine/gordian/gturbine" + +type ShardCodec interface { + Encode(shred gturbine.Shred) ([]byte, error) + Decode(data []byte) (gturbine.Shred, error) +} diff --git a/gturbine/shredding/process_shred_test.go b/gturbine/gtshredding/process_shred_test.go similarity index 99% rename from gturbine/shredding/process_shred_test.go rename to gturbine/gtshredding/process_shred_test.go index 5651dd0..cf68911 100644 --- a/gturbine/shredding/process_shred_test.go +++ b/gturbine/gtshredding/process_shred_test.go @@ -1,4 +1,4 @@ -package shredding +package gtshredding import ( "bytes" diff --git a/gturbine/shredding/processor.go b/gturbine/gtshredding/processor.go similarity index 99% rename from gturbine/shredding/processor.go rename to gturbine/gtshredding/processor.go index 7d5c08c..c4786b6 100644 --- a/gturbine/shredding/processor.go +++ b/gturbine/gtshredding/processor.go @@ -1,4 +1,4 @@ -package shredding +package gtshredding import ( "fmt" diff --git a/gturbine/shredding/processor_test.go b/gturbine/gtshredding/processor_test.go similarity index 99% rename from gturbine/shredding/processor_test.go rename to gturbine/gtshredding/processor_test.go index 31d9f35..c487066 100644 --- a/gturbine/shredding/processor_test.go +++ b/gturbine/gtshredding/processor_test.go @@ -1,4 +1,4 @@ -package shredding +package gtshredding // import ( // "bytes" diff --git a/gturbine/shredding/shred_group.go b/gturbine/gtshredding/shred_group.go similarity index 98% rename from gturbine/shredding/shred_group.go rename to gturbine/gtshredding/shred_group.go index 94e52da..bafb43d 100644 --- a/gturbine/shredding/shred_group.go +++ b/gturbine/gtshredding/shred_group.go @@ -1,4 +1,4 @@ -package shredding +package gtshredding import ( "crypto/sha256" @@ -9,6 +9,7 @@ import ( "github.com/gordian-engine/gordian/gturbine/erasure" ) +// ShredGroup represents a group of shreds that can be used to reconstruct a block. type ShredGroup struct { DataShreds []*gturbine.Shred RecoveryShreds []*gturbine.Shred