consensus,store: enforce block size limits, compact serializations, misc badger fixes

app/node/build.go

@@ -367,7 +367,7 @@ func schemaExists(ctx context.Context, db sql.Executor, schema string) (bool, er
 
 func buildBlockStore(d *coreDependencies, closers *closeFuncs) *store.BlockStore {
 	blkStrDir := config.BlockstoreDir(d.rootDir)
-	bs, err := store.NewBlockStore(blkStrDir)
+	bs, err := store.NewBlockStore(blkStrDir, store.WithCompression(d.cfg.Store.Compression))
 	if err != nil {
 		failBuild(err, "failed to open blockstore")
 	}

app/shared/display/message_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/hex"
 	"errors"
-	"fmt"
 	"io"
 	"math/big"
 	"os"
@@ -172,7 +171,7 @@ func Example_respTxQuery_json() {
 	//       "log": "This is log",
 	//       "events": null
 	//     },
-	//     "warning": "HASH MISMATCH: requested 0102030400000000000000000000000000000000000000000000000000000000; received a349a2279adc9a6067733523298932e8a78b8ae51e41fbfcb9f6c4149bcdfeaf"
+	//     "warning": "HASH MISMATCH: requested 0102030400000000000000000000000000000000000000000000000000000000; received 53096abc68a1f0a09823a4d8dea302b0ea930715627fc80be7607a9fa714fe60"
 	//   },
 	//   "error": ""
 	// }
@@ -207,8 +206,8 @@ func Example_respTxQuery_WithRaw_json() {
 	//       "log": "This is log",
 	//       "events": null
 	//     },
-	//     "raw": "00005500000041000000cb3fed7f6ff36e59054c04a831b215e514052753ee353e6fe31d4b4ef736acd6155127db555d3006ba14fcb4c79bbad56c8e63b81a9896319bb053a9e2534758000c000000736563703235366b315f6570c1000000220000005468697320697320612074657374207472616e73616374696f6e20666f7220636c69740000000000390000007866363137616631636137373465626264366432336538666531326335366434316432356132326438316538386636376336633665653064340b0000006372656174655f75736572010001001e00000000000f000000000000000004746578740000000000010003000000666f6f070000006578656375746501030000003130300a00000000000000040000006173646606000000636f6e63617400000000",
-	//     "warning": "HASH MISMATCH: requested 0102030400000000000000000000000000000000000000000000000000000000; received a349a2279adc9a6067733523298932e8a78b8ae51e41fbfcb9f6c4149bcdfeaf"
+	//     "raw": "00009e0141cb3fed7f6ff36e59054c04a831b215e514052753ee353e6fe31d4b4ef736acd6155127db555d3006ba14fcb4c79bbad56c8e63b81a9896319bb053a9e2534758000c736563703235366b315f6570e602225468697320697320612074657374207472616e73616374696f6e20666f7220636c69e8010000390000007866363137616631636137373465626264366432336538666531326335366434316432356132326438316538386636376336633665653064340b0000006372656174655f75736572010001001e00000000000f000000000000000004746578740000000000010003000000666f6f076578656375746501033130300a00000000000000046173646606636f6e63617400",
+	//     "warning": "HASH MISMATCH: requested 0102030400000000000000000000000000000000000000000000000000000000; received 53096abc68a1f0a09823a4d8dea302b0ea930715627fc80be7607a9fa714fe60"
 	//   },
 	//   "error": ""
 	// }
@@ -229,7 +228,7 @@ func Test_TxHashAndExecResponse(t *testing.T) {
 	assert.Equal(t, expectText, string(outText), "MarshalText should return expected text")
 
 	outJSON, err := resp.MarshalJSON()
-	fmt.Println(string(outJSON))
+	// fmt.Println(string(outJSON))
 	assert.NoError(t, err, "MarshalJSON should not return error")
 	assert.Equal(t, expectJSON, string(outJSON), "MarshalJSON should return expected json")
 }
@@ -286,7 +285,7 @@ func TestRespTxQuery_MarshalText(t *testing.T) {
 			name: "pending status",
 			input: &RespTxQuery{
 				Msg: &types.TxQueryResponse{
-					Hash:   mustUnmarshalHash("8d741508a6849f9d11c8f478584b5067a6bcfc1114300feff53454b0e064c0a0"),
+					Hash:   mustUnmarshalHash("1f456bec9c3819f077a7aafce25cf43ad9ab0a264cbae6efeaa8b92ec0bf4b47"),
 					Height: -1, // -1 height indicates pending
 					Tx: &types.Transaction{
 						Body: &types.TransactionBody{
@@ -304,7 +303,7 @@ func TestRespTxQuery_MarshalText(t *testing.T) {
 					},
 				},
 			},
-			expected: "Transaction ID: 8d741508a6849f9d11c8f478584b5067a6bcfc1114300feff53454b0e064c0a0\nStatus: pending\nHeight: -1\nLog: transaction pending",
+			expected: "Transaction ID: 1f456bec9c3819f077a7aafce25cf43ad9ab0a264cbae6efeaa8b92ec0bf4b47\nStatus: pending\nHeight: -1\nLog: transaction pending",
 		},
 	}
 

config/config.go

@@ -267,6 +267,9 @@ func DefaultConfig() *Config {
 			BlockProposalInterval: types.Duration(1 * time.Second),
 			BlockAnnInterval:      types.Duration(3 * time.Second),
 		},
+		Store: StoreConfig{
+			Compression: true,
+		},
 		DB: DBConfig{
 			Host:          "127.0.0.1",
 			Port:          "5432",
@@ -323,6 +326,7 @@ type Config struct {
 	P2P          PeerConfig                   `toml:"p2p" comment:"P2P related configuration"`
 	Consensus    ConsensusConfig              `toml:"consensus" comment:"Consensus related configuration"`
 	DB           DBConfig                     `toml:"db" comment:"DB (PostgreSQL) related configuration"`
+	Store        StoreConfig                  `toml:"store" comment:"Block store configuration"`
 	RPC          RPCConfig                    `toml:"rpc" comment:"User RPC service configuration"`
 	Admin        AdminConfig                  `toml:"admin" comment:"Admin RPC service configuration"`
 	Snapshots    SnapshotConfig               `toml:"snapshots" comment:"Snapshot creation and provider configuration"`
@@ -344,6 +348,18 @@ type PeerConfig struct {
 	ExternalAddress   string   `toml:"external_address" comment:"external address in host:port format to advertise to the network"`
 }
 
+// StoreConfig contains options related to the block store. This is the embedded
+// database used to store the raw block data, unlike the DBConfig which is
+// effectively the state store.
+type StoreConfig struct {
+	Compression bool `toml:"compression" comment:"compress data when writing new data"`
+
+	// Internal block size and block cache size may be of use soon.
+	//   https://github.com/kwilteam/kwil-db/issues/1347
+	// CacheSize int `toml:"cache_size" comment:"size of the block store cache in bytes"`
+	// ChunkSize int `toml:"chunk_size" comment:"size of the block store's internal blocks"`
+}
+
 type DBConfig struct {
 	// PostgreSQL DB settings. DBName is the name if the PostgreSQL database to
 	// connect to. The different data stores (e.g. engine, acct store, event

core/crypto/auth/signature_test.go

@@ -113,3 +113,105 @@ func TestSignature_UnmarshalBinary(t *testing.T) {
 		})
 	}
 }
+
+func TestSignature_SerializeSize(t *testing.T) {
+	getSerializedSigLen := func(sig *auth.Signature) int64 {
+		return int64(len(sig.Bytes()))
+	}
+	tests := []struct {
+		name     string
+		sig      auth.Signature
+		expected int64
+	}{
+		{
+			name: "Empty signature",
+			sig: auth.Signature{
+				Data: []byte{},
+				Type: "",
+			},
+			expected: 2, // 1 byte for each uvarint(0)
+		},
+		{
+			name: "Standard signature",
+			sig: auth.Signature{
+				Data: []byte{1, 2, 3, 4, 5},
+				Type: "secp256k1",
+			},
+			expected: 16, // 1 + 5 + 1 + 9
+		},
+		{
+			name: "Large signature",
+			sig: auth.Signature{
+				Data: make([]byte, 65),
+				Type: "eth_personal_sign",
+			},
+			expected: 84, // 1 + 65 + 1 + 17
+		},
+		{
+			name: "Only type",
+			sig: auth.Signature{
+				Data: nil,
+				Type: "ed25519",
+			},
+			expected: 9, // 1 + 0 + 1 + 7
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actualSigLen := getSerializedSigLen(&tt.sig)
+			size := tt.sig.SerializeSize()
+			assert.Equal(t, size, actualSigLen)
+			assert.Equal(t, tt.expected, size)
+		})
+	}
+}
+
+func TestSignature_SerializeSizeMatchesWriteTo(t *testing.T) {
+	tests := []struct {
+		name string
+		sig  auth.Signature
+	}{
+		{
+			name: "Empty signature",
+			sig: auth.Signature{
+				Data: []byte{},
+				Type: "",
+			},
+		},
+		{
+			name: "Standard signature",
+			sig: auth.Signature{
+				Data: []byte{1, 2, 3, 4, 5},
+				Type: "secp256k1",
+			},
+		},
+		{
+			name: "Large signature",
+			sig: auth.Signature{
+				Data: make([]byte, 65),
+				Type: "eth_personal_sign",
+			},
+		},
+		{
+			name: "Only type",
+			sig: auth.Signature{
+				Data: nil,
+				Type: "ed25519",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			expectedSize := tt.sig.SerializeSize()
+
+			buf := new(bytes.Buffer)
+			n, err := tt.sig.WriteTo(buf)
+
+			require.NoError(t, err)
+			assert.Equal(t, expectedSize, n, "SerializeSize should match WriteTo output size")
+			assert.Equal(t, expectedSize, int64(buf.Len()), "SerializeSize should match actual bytes written")
+		})
+	}
+}

core/crypto/auth/signer.go

@@ -14,6 +14,14 @@ import (
 	"github.com/kwilteam/kwil-db/core/utils"
 )
 
+func uvarintLen(x uint64) int {
+	return len(binary.AppendUvarint(nil, x))
+}
+
+func serializedLen(x int) int {
+	return x + uvarintLen(uint64(x))
+}
+
 // Signature is a signature with a designated AuthType, which should
 // be used to determine how to verify the signature.
 type Signature struct {
@@ -24,18 +32,25 @@ type Signature struct {
 	Type string `json:"type"`
 }
 
+func (s Signature) SerializeSize() int64 {
+	return int64(serializedLen(len(s.Data)) + serializedLen(len(s.Type)))
+	// return int64(4 + len(s.Data) + 4 + len(s.Type))
+}
+
 var _ io.WriterTo = Signature{}
 
 func (s Signature) WriteTo(w io.Writer) (int64, error) {
 	cw := utils.NewCountingWriter(w)
-	if err := binary.Write(cw, binary.LittleEndian, uint32(len(s.Data))); err != nil {
+	_, err := cw.Write(binary.AppendUvarint(nil, uint64(len(s.Data))))
+	if err != nil {
 		return cw.Written(), fmt.Errorf("failed to write signature length: %w", err)
 	}
 	if err := binary.Write(cw, binary.LittleEndian, s.Data); err != nil {
 		return cw.Written(), fmt.Errorf("failed to write signature data: %w", err)
 	}
 
-	if err := binary.Write(cw, binary.LittleEndian, uint32(len(s.Type))); err != nil {
+	_, err = cw.Write(binary.AppendUvarint(nil, uint64(len(s.Type))))
+	if err != nil {
 		return cw.Written(), fmt.Errorf("failed to write signature type length: %w", err)
 	}
 	if err := binary.Write(cw, binary.LittleEndian, []byte(s.Type)); err != nil {
@@ -73,53 +88,49 @@ func (s *Signature) UnmarshalBinary(data []byte) error {
 
 func (s *Signature) ReadFrom(r io.Reader) (int64, error) {
 	rl, _ := r.(interface{ Len() int })
-	var n int64
-	var sigLen uint32
-	if err := binary.Read(r, binary.LittleEndian, &sigLen); err != nil {
-		return 0, fmt.Errorf("failed to read signature length: %w", err)
+	cr := utils.NewCountingReader(r)
+	sigLen, err := binary.ReadUvarint(cr) // signature length
+	if err != nil {
+		return cr.ReadCount(), fmt.Errorf("failed to read signature length: %w", err)
 	}
-	n += 4
 
 	if sigLen > 0 {
-		if rl != nil {
+		if rl, ok := r.(interface{ Len() int }); ok {
 			if int(sigLen) > rl.Len() {
-				return 0, fmt.Errorf("impossibly long signature length: %d", sigLen)
+				return cr.ReadCount(), fmt.Errorf("impossibly long signature length: %d", sigLen)
 			}
 			s.Data = make([]byte, sigLen)
-			if _, err := io.ReadFull(r, s.Data); err != nil {
-				return 0, fmt.Errorf("failed to read signature data: %w", err)
+			if _, err := io.ReadFull(cr, s.Data); err != nil {
+				return cr.ReadCount(), fmt.Errorf("failed to read signature data: %w", err)
 			}
 		} else {
 			sigBuf := &bytes.Buffer{}
-			_, err := io.CopyN(sigBuf, r, int64(sigLen))
+			_, err := io.CopyN(sigBuf, cr, int64(sigLen))
 			if err != nil {
-				return 0, fmt.Errorf("failed to read signature data: %w", err)
+				return cr.ReadCount(), fmt.Errorf("failed to read signature data: %w", err)
 			}
 			s.Data = sigBuf.Bytes()
 		}
 
 	}
-	n += int64(sigLen)
 
-	var typeLen uint32
-	if err := binary.Read(r, binary.LittleEndian, &typeLen); err != nil {
-		return 0, fmt.Errorf("failed to read signature type length: %w", err)
+	typeLen, err := binary.ReadUvarint(cr)
+	if err != nil {
+		return cr.ReadCount(), fmt.Errorf("failed to read signature type length: %w", err)
 	}
-	n += 4
 
 	if typeLen > 0 {
 		if rl != nil && int(typeLen) > rl.Len() {
-			return 0, fmt.Errorf("impossibly long sig type length: %d", typeLen)
+			return cr.ReadCount(), fmt.Errorf("impossibly long sig type length: %d", typeLen)
 		}
 		typeBytes := make([]byte, typeLen)
-		if _, err := io.ReadFull(r, typeBytes); err != nil {
-			return 0, fmt.Errorf("failed to read signature type: %w", err)
+		if _, err := io.ReadFull(cr, typeBytes); err != nil {
+			return cr.ReadCount(), fmt.Errorf("failed to read signature type: %w", err)
 		}
 		s.Type = string(typeBytes)
 	}
-	n += int64(typeLen)
 
-	return n, nil
+	return cr.ReadCount(), nil
 }
 
 // Signer is an interface for something that can sign messages.