chore: Make Blob be variable size (#52)

* add blobDegree to computeFiatShamir

* Blobs are now slices so Blob{} or gokzg4844.Blob will give an empty slice

* cannot index a pointer to a slice

* create constants for minimal and mainnet

* modify hexStrToBlob to not use a nil slice

* trusted setup now uses array as we could pass in a minimal trusted setup

* Deserialize now needs to take in the expected size of the Blob

* use MainnetScalarsPerBlob in NewContext4096 function

api.go

@@ -39,7 +39,7 @@ var PointAtInfinity = [48]byte{0xc0}
 // methods. "4096" denotes that we will only be able to commit to polynomials with at most 4096 evaluations. "Insecure"
 // denotes that this method should not be used in production since the secret (1337) is known.
 func NewContext4096Insecure1337() (*Context, error) {
-	if ScalarsPerBlob != 4096 {
+	if MainnetScalarsPerBlob != 4096 {
 		// This is a library bug and so we panic.
 		panic("this method is named `NewContext4096Insecure1337` we expect SCALARS_PER_BLOB to be 4096")
 	}
@@ -51,7 +51,7 @@ func NewContext4096Insecure1337() (*Context, error) {
 		return nil, err
 	}
 
-	if ScalarsPerBlob != len(parsedSetup.SetupG1) {
+	if MainnetScalarsPerBlob != len(parsedSetup.SetupG1) {
 		// This is a library method and so we panic
 		panic("this method is named `NewContext4096Insecure1337` we expect the number of G1 elements in the trusted setup to be 4096")
 	}
@@ -105,7 +105,7 @@ func NewContext4096(trustedSetup *JSONTrustedSetup) (*Context, error) {
 		AlphaG2: alphaGenG2,
 	}
 
-	domain := kzg.NewDomain(ScalarsPerBlob)
+	domain := kzg.NewDomain(MainnetScalarsPerBlob)
 	// Bit-Reverse the roots and the trusted setup according to the specs
 	// The bit reversal is not needed for simple KZG however it was
 	// implemented to make the step for full dank-sharding easier.

api_test.go

@@ -81,7 +81,7 @@ func TestNonCanonicalSmoke(t *testing.T) {
 // Below are helper methods which allow us to change a serialized element into
 // its non-canonical counterpart by adding the modulus
 func modifyBlob(blob *gokzg4844.Blob, newValue gokzg4844.Scalar, index int) {
-	copy(blob[index:index+gokzg4844.SerializedScalarSize], newValue[:])
+	copy((*blob)[index:], newValue[:])
 }
 
 func nonCanonicalScalar(seed int64) gokzg4844.Scalar {

bench_test.go

@@ -11,6 +11,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const scalarsPerBlob = 4096
+
 // / Returns a serialized random field element in big-endian
 func GetRandFieldElement(seed int64) [32]byte {
 	rand.Seed(seed)
@@ -28,8 +30,8 @@ func GetRandFieldElement(seed int64) [32]byte {
 }
 
 func GetRandBlob(seed int64) gokzg4844.Blob {
-	var blob gokzg4844.Blob
-	bytesPerBlob := gokzg4844.ScalarsPerBlob * gokzg4844.SerializedScalarSize
+	bytesPerBlob := scalarsPerBlob * gokzg4844.SerializedScalarSize
+	blob := make(gokzg4844.Blob, bytesPerBlob)
 	for i := 0; i < bytesPerBlob; i += gokzg4844.SerializedScalarSize {
 		fieldElementBytes := GetRandFieldElement(seed + int64(i))
 		copy(blob[i:i+gokzg4844.SerializedScalarSize], fieldElementBytes[:])

consensus_specs_test.go

@@ -356,16 +356,15 @@ func TestVerifyBlobKZGProofBatch(t *testing.T) {
 }
 
 func hexStrToBlob(hexStr string) (gokzg4844.Blob, error) {
-	var blob gokzg4844.Blob
 	byts, err := hexStrToBytes(hexStr)
 	if err != nil {
-		return blob, err
+		return nil, err
 	}
-
-	if len(blob) != len(byts) {
-		return blob, fmt.Errorf("blob does not have the correct length, %d ", len(byts))
+	if len(byts) != gokzg4844.MainnetNumBytesPerBlob {
+		return nil, fmt.Errorf("blob does not have the correct length, %d ", len(byts))
 	}
-	copy(blob[:], byts)
+	blob := make(gokzg4844.Blob, len(byts))
+	copy(blob, byts)
 	return blob, nil
 }
 

fiatshamir.go

@@ -17,8 +17,8 @@ const DomSepProtocol = "FSBLOBVERIFY_V1_"
 // computeChallenge is provided to match the spec at [compute_challenge].
 //
 // [compute_challenge]: https://github.com/ethereum/consensus-specs/blob/017a8495f7671f5fff2075a9bfc9238c1a0982f8/specs/deneb/polynomial-commitments.md#compute_challenge
-func computeChallenge(blob Blob, commitment KZGCommitment) fr.Element {
-	polyDegreeBytes := u64ToByteArray16(ScalarsPerBlob)
+func computeChallenge(blob Blob, commitment KZGCommitment, scalarsPerBlob uint64) fr.Element {
+	polyDegreeBytes := u64ToByteArray16(scalarsPerBlob)
 	data := append([]byte(DomSepProtocol), polyDegreeBytes...)
 	data = append(data, blob[:]...)
 	data = append(data, commitment[:]...)

fiatshamir_test.go

@@ -11,9 +11,9 @@ import (
 // If the way computeChallenge is computed is updated
 // then this test will fail
 func TestComputeChallengeInterop(t *testing.T) {
-	blob := Blob{}
+	blob := make(Blob, 4096*32)
 	commitment := SerializeG1Point(bls12381.G1Affine{})
-	challenge := computeChallenge(blob, KZGCommitment(commitment))
+	challenge := computeChallenge(blob, KZGCommitment(commitment), 4096)
 	expected := []byte{
 		0x04, 0xb7, 0xb2, 0x2a, 0xf6, 0x3d, 0x2b, 0x2f,
 		0x1c, 0xed, 0x8d, 0x55, 0x05, 0x60, 0xe5, 0xd1,

prove.go

@@ -14,7 +14,7 @@ func (c *Context) BlobToKZGCommitment(blob Blob, numGoRoutines int) (KZGCommitme
 	// 1. Deserialization
 	//
 	// Deserialize blob into polynomial
-	polynomial, err := DeserializeBlob(blob)
+	polynomial, err := DeserializeBlob(blob, c.domain.Cardinality)
 	if err != nil {
 		return KZGCommitment{}, err
 	}
@@ -46,7 +46,7 @@ func (c *Context) BlobToKZGCommitment(blob Blob, numGoRoutines int) (KZGCommitme
 func (c *Context) ComputeBlobKZGProof(blob Blob, blobCommitment KZGCommitment, numGoRoutines int) (KZGProof, error) {
 	// 1. Deserialization
 	//
-	polynomial, err := DeserializeBlob(blob)
+	polynomial, err := DeserializeBlob(blob, c.domain.Cardinality)
 	if err != nil {
 		return KZGProof{}, err
 	}
@@ -60,7 +60,8 @@ func (c *Context) ComputeBlobKZGProof(blob Blob, blobCommitment KZGCommitment, n
 	}
 
 	// 2. Compute Fiat-Shamir challenge
-	evaluationChallenge := computeChallenge(blob, blobCommitment)
+	blobDegree := uint64(len(c.domain.Roots))
+	evaluationChallenge := computeChallenge(blob, blobCommitment, blobDegree)
 
 	// 3. Create opening proof
 	openingProof, err := kzg.Open(c.domain, polynomial, evaluationChallenge, c.commitKey, numGoRoutines)
@@ -85,7 +86,7 @@ func (c *Context) ComputeBlobKZGProof(blob Blob, blobCommitment KZGCommitment, n
 func (c *Context) ComputeKZGProof(blob Blob, inputPointBytes Scalar, numGoRoutines int) (KZGProof, Scalar, error) {
 	// 1. Deserialization
 	//
-	polynomial, err := DeserializeBlob(blob)
+	polynomial, err := DeserializeBlob(blob, c.domain.Cardinality)
 	if err != nil {
 		return KZGProof{}, [32]byte{}, err
 	}

serialization.go

@@ -21,7 +21,7 @@ const CompressedG2Size = 96
 // [BYTES_PER_FIELD_ELEMENT]: https://github.com/ethereum/consensus-specs/blob/017a8495f7671f5fff2075a9bfc9238c1a0982f8/specs/deneb/polynomial-commitments.md#constants
 const SerializedScalarSize = 32
 
-// ScalarsPerBlob is the number of serialized scalars in a blob.
+// MainnetScalarsPerBlob is the number of serialized scalars in a blob for mainnet.
 //
 // It matches [FIELD_ELEMENTS_PER_BLOB] in the spec.
 //
@@ -30,7 +30,18 @@ const SerializedScalarSize = 32
 //
 // [BLS_MODULUS]: https://github.com/ethereum/consensus-specs/blob/017a8495f7671f5fff2075a9bfc9238c1a0982f8/specs/deneb/polynomial-commitments.md#constants
 // [FIELD_ELEMENTS_PER_BLOB]: https://github.com/ethereum/consensus-specs/blob/017a8495f7671f5fff2075a9bfc9238c1a0982f8/specs/deneb/polynomial-commitments.md#blob
-const ScalarsPerBlob = 4096
+const MainnetScalarsPerBlob = 4096
+
+// The number of bytes needed to represent a blob on mainnet.
+const MainnetNumBytesPerBlob = MainnetScalarsPerBlob * SerializedScalarSize
+
+// MinimalScalarsPerBlob is the number of serialized scalars in a blob for what is known as
+// minimal configuration. This configuration is used for testing, prototyping and in some
+// consumers, end to end tests.
+const MinimalScalarsPerBlob = 4
+
+// The number of bytes needed to represent a blob using minimal configurations.
+const MinimalNumBytesPerBlob = MainnetScalarsPerBlob * SerializedScalarSize
 
 type (
 	// G1Point matches [G1Point] in the spec.
@@ -50,10 +61,12 @@ type (
 
 	// Blob is a flattened representation of a serialized polynomial.
 	//
-	// It matches [Blob] in the spec.
+	// Note: A blob in the specifications is a fixed size array.
+	// This datatype does not have a fixed size as we want to support
+	// two different sizes of Blobs.
 	//
 	// [Blob]: https://github.com/ethereum/consensus-specs/blob/017a8495f7671f5fff2075a9bfc9238c1a0982f8/specs/deneb/polynomial-commitments.md#custom-types
-	Blob [ScalarsPerBlob * SerializedScalarSize]byte
+	Blob []byte
 
 	// KZGProof is a serialized commitment to the quotient polynomial.
 	//
@@ -107,9 +120,9 @@ func DeserializeKZGProof(proof KZGProof) (bls12381.G1Affine, error) {
 // DeserializeBlob implements [blob_to_polynomial].
 //
 // [blob_to_polynomial]: https://github.com/ethereum/consensus-specs/blob/017a8495f7671f5fff2075a9bfc9238c1a0982f8/specs/deneb/polynomial-commitments.md#blob_to_polynomial
-func DeserializeBlob(blob Blob) (kzg.Polynomial, error) {
-	poly := make(kzg.Polynomial, ScalarsPerBlob)
-	for i := 0; i < ScalarsPerBlob; i++ {
+func DeserializeBlob(blob Blob, scalarsPerBlob uint64) (kzg.Polynomial, error) {
+	poly := make(kzg.Polynomial, int(scalarsPerBlob))
+	for i := 0; i < int(scalarsPerBlob); i++ {
 		chunk := blob[i*SerializedScalarSize : (i+1)*SerializedScalarSize]
 		serScalar := (*Scalar)(chunk)
 		scalar, err := DeserializeScalar(*serScalar)
@@ -144,8 +157,9 @@ func SerializeScalar(element fr.Element) Scalar {
 // Note: This method is never used in the API because we always expect a byte array and will never receive deserialized
 // field elements. We include it so that upstream fuzzers do not need to reimplement it.
 func SerializePoly(poly kzg.Polynomial) Blob {
-	var blob Blob
-	for i := 0; i < ScalarsPerBlob; i++ {
+	scalarsPerBlob := len(poly)
+	blob := make(Blob, scalarsPerBlob*SerializedScalarSize)
+	for i := 0; i < scalarsPerBlob; i++ {
 		chunk := blob[i*SerializedScalarSize : (i+1)*SerializedScalarSize]
 		serScalar := SerializeScalar(poly[i])
 		copy(chunk, serScalar[:])

serialization_test.go

@@ -30,8 +30,7 @@ func TestSerializePolyNotZero(t *testing.T) {
 
 	poly := randPoly4096()
 	blob := gokzg4844.SerializePoly(poly)
-
-	var zeroBlob gokzg4844.Blob
+	zeroBlob := make(gokzg4844.Blob, len(blob))
 	if bytes.Equal(blob[:], zeroBlob[:]) {
 		t.Error("blobs are all zeroes, which can only happen with negligible probability")
 	}
@@ -44,11 +43,11 @@ func TestSerializePolyRoundTrip(t *testing.T) {
 	blobA := gokzg4844.SerializePoly(expectedPolyA)
 	blobB := gokzg4844.SerializePoly(expectedPolyB)
 
-	gotPolyA, err := gokzg4844.DeserializeBlob(blobA)
+	gotPolyA, err := gokzg4844.DeserializeBlob(blobA, 4096)
 	if err != nil {
 		t.Error(err)
 	}
-	gotPolyB, err := gokzg4844.DeserializeBlob(blobB)
+	gotPolyB, err := gokzg4844.DeserializeBlob(blobB, 4096)
 	if err != nil {
 		t.Error(err)
 	}

trusted_setup.go

@@ -23,9 +23,9 @@ import (
 // The intended use-case is that library users store the trusted setup in a JSON file and we provide such a file
 // as part of the package.
 type JSONTrustedSetup struct {
-	SetupG1         [ScalarsPerBlob]G1CompressedHexStr `json:"setup_G1"`
-	SetupG2         []G2CompressedHexStr               `json:"setup_G2"`
-	SetupG1Lagrange [ScalarsPerBlob]G1CompressedHexStr `json:"setup_G1_lagrange"`
+	SetupG1         []G1CompressedHexStr `json:"setup_G1"`
+	SetupG2         []G2CompressedHexStr `json:"setup_G2"`
+	SetupG1Lagrange []G1CompressedHexStr `json:"setup_G1_lagrange"`
 }
 
 // G1CompressedHexStr is a hex-string (with the 0x prefix) of a compressed G1 point.
@@ -65,7 +65,7 @@ func CheckTrustedSetupIsWellFormed(trustedSetup *JSONTrustedSetup) error {
 		setupG1Points = append(setupG1Points, point)
 	}
 
-	domain := kzg.NewDomain(ScalarsPerBlob)
+	domain := kzg.NewDomain(uint64(len(setupG1Points)))
 	// The G1 points will be in monomial form
 	// Convert them to lagrange form
 	// See 3.1 onwards in https://eprint.iacr.org/2017/602.pdf for further details

verify.go

@@ -48,7 +48,7 @@ func (c *Context) VerifyKZGProof(blobCommitment KZGCommitment, inputPointBytes,
 func (c *Context) VerifyBlobKZGProof(blob Blob, blobCommitment KZGCommitment, kzgProof KZGProof) error {
 	// 1. Deserialize
 	//
-	polynomial, err := DeserializeBlob(blob)
+	polynomial, err := DeserializeBlob(blob, c.domain.Cardinality)
 	if err != nil {
 		return err
 	}
@@ -64,7 +64,8 @@ func (c *Context) VerifyBlobKZGProof(blob Blob, blobCommitment KZGCommitment, kz
 	}
 
 	// 2. Compute the evaluation challenge
-	evaluationChallenge := computeChallenge(blob, blobCommitment)
+	blobDegree := uint64(len(c.domain.Roots))
+	evaluationChallenge := computeChallenge(blob, blobCommitment, blobDegree)
 
 	// 3. Compute output point/ claimed value
 	outputPoint, err := c.domain.EvaluateLagrangePolynomial(polynomial, evaluationChallenge)
@@ -115,13 +116,13 @@ func (c *Context) VerifyBlobKZGProofBatch(blobs []Blob, polynomialCommitments []
 		}
 
 		blob := blobs[i]
-		polynomial, err := DeserializeBlob(blob)
+		polynomial, err := DeserializeBlob(blob, c.domain.Cardinality)
 		if err != nil {
 			return err
 		}
 
 		// 2b. Compute the evaluation challenge
-		evaluationChallenge := computeChallenge(blob, serComm)
+		evaluationChallenge := computeChallenge(blob, serComm, 4096)
 
 		// 2c. Compute output point/ claimed value
 		outputPoint, err := c.domain.EvaluateLagrangePolynomial(polynomial, evaluationChallenge)