diff --git a/sei-tendermint/config/config.go b/sei-tendermint/config/config.go index 19eda4370a..edc9270bf9 100644 --- a/sei-tendermint/config/config.go +++ b/sei-tendermint/config/config.go @@ -884,8 +884,8 @@ type MempoolConfig struct { // valid again in the future. KeepInvalidTxsInCache bool `mapstructure:"keep-invalid-txs-in-cache"` - // Maximum size of a single transaction - // NOTE: the max size of a tx transmitted over the network is {max-tx-bytes}. + // Maximum size of a single transaction. + // XXX: Unused. Admission uses the protocol gossip limit. A different value logs a warning. MaxTxBytes int `mapstructure:"max-tx-bytes"` // Maximum size of a batch of transactions to send to a peer diff --git a/sei-tendermint/config/toml.go b/sei-tendermint/config/toml.go index cd0b605fab..93e5eaa5f0 100644 --- a/sei-tendermint/config/toml.go +++ b/sei-tendermint/config/toml.go @@ -437,7 +437,7 @@ duplicate-txs-cache-size = "{{ .Mempool.DuplicateTxsCacheSize }}" keep-invalid-txs-in-cache = {{ .Mempool.KeepInvalidTxsInCache }} # Maximum size of a single transaction. -# NOTE: the max size of a tx transmitted over the network is {max-tx-bytes}. +# XXX: Unused. Admission uses the protocol gossip limit. A different value logs a warning. max-tx-bytes = {{ .Mempool.MaxTxBytes }} # Maximum size of a batch of transactions to send to a peer diff --git a/sei-tendermint/internal/mempool/mempool.go b/sei-tendermint/internal/mempool/mempool.go index 5242f3060c..97ee35657e 100644 --- a/sei-tendermint/internal/mempool/mempool.go +++ b/sei-tendermint/internal/mempool/mempool.go @@ -53,8 +53,8 @@ type Config struct { // valid again in the future. KeepInvalidTxsInCache bool - // Maximum size of a single transaction - // NOTE: the max size of a tx transmitted over the network is {max-tx-bytes}. + // Maximum size of a single transaction. + // XXX: Unused. Admission uses the protocol gossip limit. A different value logs a warning. MaxTxBytes int // time after which transaction is removed from mempool. @@ -129,7 +129,7 @@ func DefaultConfig() *Config { MaxTxsBytes: 1024 * 1024 * 1024, // 1GB CacheSize: 10000, DuplicateTxsCacheSize: 100000, - MaxTxBytes: 1024 * 1024, // 1MB + MaxTxBytes: types.MaxGossipTxBytes, TTLDuration: utils.Some(5 * time.Second), // prevent stale txs from filling mempool TTLNumBlocks: utils.Some(int64(10)), // remove txs after 10 blocks TxNotifyThreshold: 0, @@ -209,6 +209,11 @@ func NewTxMempool( app *proxy.Proxy, txConstraintsFetcher TxConstraintsFetcher, ) *TxMempool { + if cfg.MaxTxBytes != types.MaxGossipTxBytes { + logger.Warn("mempool max-tx-bytes differs from the protocol gossip limit; admission uses the protocol limit", + "max-tx-bytes", cfg.MaxTxBytes, + "protocol", types.MaxGossipTxBytes) + } txmp := &TxMempool{ config: cfg, app: app, @@ -291,8 +296,8 @@ func (txmp *TxMempool) CheckTx(ctx context.Context, tx types.Tx) (*abci.Response defer txmp.mtx.RUnlock() // Early exit if tx is too large. - if txSize := len(tx); txSize > txmp.config.MaxTxBytes { - return nil, fmt.Errorf("%w: max size is %d, but got %d", ErrTxTooLarge, txmp.config.MaxTxBytes, txSize) + if txSize := len(tx); txSize > types.MaxGossipTxBytes { + return nil, fmt.Errorf("%w: max size is %d, but got %d", ErrTxTooLarge, types.MaxGossipTxBytes, txSize) } hTx := newHashedTx(tx) diff --git a/sei-tendermint/internal/mempool/mempool_test.go b/sei-tendermint/internal/mempool/mempool_test.go index d5af52969f..b54a04d127 100644 --- a/sei-tendermint/internal/mempool/mempool_test.go +++ b/sei-tendermint/internal/mempool/mempool_test.go @@ -553,14 +553,14 @@ func TestTxMempool_CheckTxExceedsMaxSize(t *testing.T) { txmp := setup(cfg, proxy.New(client), NopTxConstraintsFetcher) rng := rand.New(rand.NewSource(time.Now().UnixNano())) - tx := make([]byte, txmp.config.MaxTxBytes+1) + tx := make([]byte, types.MaxGossipTxBytes+1) _, err := rng.Read(tx) require.NoError(t, err) _, err = txmp.CheckTx(ctx, tx) require.Error(t, err) - tx = make([]byte, txmp.config.MaxTxBytes-1) + tx = make([]byte, types.MaxGossipTxBytes-1) _, err = rng.Read(tx) require.NoError(t, err) diff --git a/sei-tendermint/internal/mempool/reactor/reactor.go b/sei-tendermint/internal/mempool/reactor/reactor.go index d5aae51173..14ab912c9a 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor.go +++ b/sei-tendermint/internal/mempool/reactor/reactor.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "runtime/debug" + "sync" "github.com/sei-protocol/sei-chain/sei-tendermint/config" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/libs/clist" @@ -22,6 +23,15 @@ var ( logger = seilog.NewLogger("tendermint", "internal", "mempool") _ service.Service = (*Reactor)(nil) + + // mempoolRecvMessageCapacity is the encoded size of one MaxGossipTxBytes transaction. + mempoolRecvMessageCapacity = sync.OnceValue(func() int { + return (&pb.Message{ + Sum: &pb.Message_Txs{ + Txs: &pb.Txs{Txs: [][]byte{make([]byte, types.MaxGossipTxBytes)}}, + }, + }).Size() + }) ) const MempoolChannel p2p.ChannelID = 0x30 @@ -45,7 +55,7 @@ type Reactor struct { // NewReactor returns a reference to a new reactor. func NewReactor(cfg *config.MempoolConfig, txmp *mempool.TxMempool, router *p2p.Router) (*Reactor, error) { - channel, err := p2p.OpenChannel(router, GetChannelDescriptor(cfg)) + channel, err := p2p.OpenChannel(router, GetChannelDescriptor()) if err != nil { return nil, fmt.Errorf("router.OpenChannel(): %w", err) } @@ -65,19 +75,13 @@ func (r *Reactor) MarkReadyToStart() { r.readyToStart <- struct{}{} } // GetChannelDescriptor produces an instance of a descriptor for this package's // required channels. -func GetChannelDescriptor(cfg *config.MempoolConfig) p2p.ChannelDescriptor[*pb.Message] { - largestTx := make([]byte, cfg.MaxTxBytes) - batchMsg := &pb.Message{ - Sum: &pb.Message_Txs{ - Txs: &pb.Txs{Txs: [][]byte{largestTx}}, - }, - } - +func GetChannelDescriptor() p2p.ChannelDescriptor[*pb.Message] { return p2p.ChannelDescriptor[*pb.Message]{ ID: MempoolChannel, MessageType: new(pb.Message), Priority: 5, - RecvMessageCapacity: batchMsg.Size(), + RecvMessageCapacity: mempoolRecvMessageCapacity(), + DiscardOversized: true, RecvBufferCapacity: 128, Name: "mempool", } @@ -239,11 +243,18 @@ func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID) { } for { tx := next.Value() - r.channel.Send(&pb.Message{ - Sum: &pb.Message_Txs{ - Txs: &pb.Txs{Txs: [][]byte{tx}}, - }, - }, peerID) + if len(tx) > types.MaxGossipTxBytes { + logger.Debug("skipping gossip of tx above protocol size", + "tx", tx.Hash(), + "size", len(tx), + "peer", peerID) + } else { + r.channel.Send(&pb.Message{ + Sum: &pb.Message_Txs{ + Txs: &pb.Txs{Txs: [][]byte{tx}}, + }, + }, peerID) + } next, err = next.NextWait(ctx) if err != nil { diff --git a/sei-tendermint/internal/mempool/reactor/reactor_test.go b/sei-tendermint/internal/mempool/reactor/reactor_test.go index f587cf919e..ec12374f05 100644 --- a/sei-tendermint/internal/mempool/reactor/reactor_test.go +++ b/sei-tendermint/internal/mempool/reactor/reactor_test.go @@ -40,12 +40,25 @@ type reactorTestSuite struct { } func setupMempool(t testing.TB, app *proxy.Proxy, cacheSize int, txConstraintsFetcher mempool.TxConstraintsFetcher) *mempool.TxMempool { + return setupMempoolTweaked(t, app, cacheSize, txConstraintsFetcher, nil) +} + +func setupMempoolTweaked( + t testing.TB, + app *proxy.Proxy, + cacheSize int, + txConstraintsFetcher mempool.TxConstraintsFetcher, + tweak func(*config.MempoolConfig), +) *mempool.TxMempool { t.Helper() cfg, err := config.ResetTestRoot(t.TempDir(), strings.ReplaceAll(t.Name(), "/", "|")) require.NoError(t, err) cfg.Mempool.CacheSize = cacheSize cfg.Mempool.DropUtilisationThreshold = 0.0 + if tweak != nil { + tweak(cfg.Mempool) + } t.Cleanup(func() { os.RemoveAll(cfg.RootDir) }) @@ -87,6 +100,17 @@ func setupReactorsWithConfig( numNodes int, cfg *config.MempoolConfig, txConstraintsFetcher mempool.TxConstraintsFetcher, +) *reactorTestSuite { + return setupReactorsWithNodeMempool(ctx, t, numNodes, cfg, txConstraintsFetcher, nil) +} + +func setupReactorsWithNodeMempool( + ctx context.Context, + t *testing.T, + numNodes int, + cfg *config.MempoolConfig, + txConstraintsFetcher mempool.TxConstraintsFetcher, + tweakNodeMempool func(int, *config.MempoolConfig), ) *reactorTestSuite { t.Helper() @@ -97,16 +121,21 @@ func setupReactorsWithConfig( kvstores: make(map[types.NodeID]*kvstore.Application, numNodes), } - for _, node := range rts.network.Nodes() { + for i, node := range rts.network.Nodes() { nodeID := node.NodeID rts.kvstores[nodeID] = kvstore.NewApplication() app := rts.kvstores[nodeID] proxyApp := proxy.New(app) - txmp := setupMempool(t, proxyApp, 0, txConstraintsFetcher) + var tweak func(*config.MempoolConfig) + if tweakNodeMempool != nil { + tweak = func(mc *config.MempoolConfig) { tweakNodeMempool(i, mc) } + } + txmp := setupMempoolTweaked(t, proxyApp, 0, txConstraintsFetcher, tweak) rts.mempools[nodeID] = txmp - reactor, err := NewReactor(cfg, txmp, node.Router) + nodeCfg := *cfg + reactor, err := NewReactor(&nodeCfg, txmp, node.Router) if err != nil { t.Fatalf("NewReactor(): %v", err) } @@ -190,6 +219,12 @@ func peerFailedCheckTxCount(reactor *Reactor, nodeID types.NodeID) utils.Option[ panic("unreachable") } +func txConstraintsWithMaxDataBytes(maxDataBytes int64) mempool.TxConstraintsFetcher { + return func() (mempool.TxConstraints, error) { + return mempool.TxConstraints{MaxDataBytes: maxDataBytes, MaxGas: -1}, nil + } +} + func TestReactorBroadcastTxs(t *testing.T) { numTxs := 512 numNodes := 4 @@ -220,12 +255,16 @@ func TestReactorFailedCheckTxCountEvictsPeer(t *testing.T) { cfg.CheckTxErrorBlacklistEnabled = true cfg.CheckTxErrorThreshold = 2 - rts := setupReactorsWithConfig(ctx, t, 2, cfg, func() (mempool.TxConstraints, error) { - return mempool.TxConstraints{ - MaxDataBytes: 10, - MaxGas: -1, - }, nil - }) + good1 := []byte("good-1") + good2 := []byte("good-2") + maxDataBytes := types.ComputeProtoSizeForTxs([]types.Tx{good1}) + if n := types.ComputeProtoSizeForTxs([]types.Tx{good2}); n > maxDataBytes { + maxDataBytes = n + } + badTx := []byte("bad=" + strings.Repeat("x", 64)) + require.Greater(t, types.ComputeProtoSizeForTxs([]types.Tx{badTx}), maxDataBytes) + + rts := setupReactorsWithConfig(ctx, t, 2, cfg, txConstraintsWithMaxDataBytes(maxDataBytes)) t.Cleanup(leaktest.Check(t)) sender := rts.nodes[0] @@ -249,14 +288,13 @@ func TestReactorFailedCheckTxCountEvictsPeer(t *testing.T) { return peerFailedCheckTxCount(receiverReactor, sender) == utils.Some(0) }, time.Second, 50*time.Millisecond) - require.NoError(t, receiverReactor.handleMempoolMessage(ctx, msgForTx([]byte("good-1")))) + require.NoError(t, receiverReactor.handleMempoolMessage(ctx, msgForTx(good1))) require.Equal(t, utils.Some(0), peerFailedCheckTxCount(receiverReactor, sender)) - badTx := []byte("bad-transaction") require.NoError(t, receiverReactor.handleMempoolMessage(ctx, msgForTx(badTx))) require.Equal(t, utils.Some(1), peerFailedCheckTxCount(receiverReactor, sender)) - require.NoError(t, receiverReactor.handleMempoolMessage(ctx, msgForTx([]byte("good-2")))) + require.NoError(t, receiverReactor.handleMempoolMessage(ctx, msgForTx(good2))) require.Equal(t, utils.Some(1), peerFailedCheckTxCount(receiverReactor, sender)) require.NoError(t, receiverReactor.handleMempoolMessage(ctx, msgForTx(badTx))) @@ -269,15 +307,7 @@ func TestReactorFailedCheckTxCountEvictsPeer(t *testing.T) { } func TestReactorPeerDownClearsFailedCheckTxCount(t *testing.T) { - reactor, _ := setupReactorForTest( - t, - func() (mempool.TxConstraints, error) { - return mempool.TxConstraints{ - MaxDataBytes: 10, - MaxGas: -1, - }, nil - }, - ) + reactor, _ := setupReactorForTest(t, txConstraintsWithMaxDataBytes(1)) for counts := range reactor.failedCheckTxCounts.Lock() { counts["other"] = 1 } @@ -285,7 +315,7 @@ func TestReactorPeerDownClearsFailedCheckTxCount(t *testing.T) { From: "sender", Message: &pb.Message{ Sum: &pb.Message_Txs{ - Txs: &pb.Txs{Txs: [][]byte{[]byte("precheck-bad-transaction")}}, + Txs: &pb.Txs{Txs: [][]byte{[]byte("x")}}, }, }, } @@ -308,20 +338,12 @@ func TestReactorPeerDownClearsFailedCheckTxCount(t *testing.T) { } func TestReactorMissingFailedCheckTxCountIsNotRecreated(t *testing.T) { - reactor, _ := setupReactorForTest( - t, - func() (mempool.TxConstraints, error) { - return mempool.TxConstraints{ - MaxDataBytes: 10, - MaxGas: -1, - }, nil - }, - ) + reactor, _ := setupReactorForTest(t, txConstraintsWithMaxDataBytes(1)) msg := p2p.RecvMsg[*pb.Message]{ From: "sender", Message: &pb.Message{ Sum: &pb.Message_Txs{ - Txs: &pb.Txs{Txs: [][]byte{[]byte("precheck-bad-transaction")}}, + Txs: &pb.Txs{Txs: [][]byte{[]byte("x")}}, }, }, } @@ -393,7 +415,6 @@ func TestReactorConcurrency(t *testing.T) { func TestReactor_MaxTxBytes(t *testing.T) { numNodes := 2 - cfg := config.TestConfig() ctx := t.Context() rts := setupReactors(ctx, t, numNodes) @@ -402,7 +423,7 @@ func TestReactor_MaxTxBytes(t *testing.T) { primary := rts.nodes[0] secondary := rts.nodes[1] - tx1 := tmrand.Bytes(cfg.Mempool.MaxTxBytes) + tx1 := tmrand.Bytes(types.MaxGossipTxBytes) _, err := rts.reactors[primary].mempool.CheckTx( ctx, tx1, @@ -414,11 +435,115 @@ func TestReactor_MaxTxBytes(t *testing.T) { rts.reactors[primary].mempool.Flush() rts.reactors[secondary].mempool.Flush() - tx2 := tmrand.Bytes(cfg.Mempool.MaxTxBytes + 1) + tx2 := tmrand.Bytes(types.MaxGossipTxBytes + 1) _, err = rts.mempools[primary].CheckTx(ctx, tx2) require.Error(t, err) } +func TestGetChannelDescriptorProtocolRecvCapacity(t *testing.T) { + // Setup: mempool channel descriptor used by every node. + desc := GetChannelDescriptor() + + // Test: receive capacity is the protocol gossip envelope, not local max-tx-bytes. + // Verify: capacity covers MaxGossipTxBytes and oversized messages are discarded. + require.Equal(t, mempoolRecvMessageCapacity(), desc.RecvMessageCapacity) + require.GreaterOrEqual(t, desc.RecvMessageCapacity, types.MaxGossipTxBytes) + require.True(t, desc.DiscardOversized) +} + +func TestReactorMismatchedMaxTxBytesKeepsConnection(t *testing.T) { + ctx := t.Context() + + // Setup: nodes with different max-tx-bytes toml; admission is the protocol cap. + senderMaxTxBytes := 512 + receiverMaxTxBytes := 64 + limits := []int{senderMaxTxBytes, receiverMaxTxBytes} + rts := setupReactorsWithNodeMempool(ctx, t, 2, config.TestMempoolConfig(), mempool.NopTxConstraintsFetcher, + func(i int, mempoolCfg *config.MempoolConfig) { + mempoolCfg.MaxTxBytes = limits[i] + }) + t.Cleanup(leaktest.Check(t)) + + sender := rts.nodes[0] + receiver := rts.nodes[1] + rts.start(t) + rts.network.Node(receiver).WaitForConnAndGet(ctx, sender) + require.Eventually(t, func() bool { + return peerFailedCheckTxCount(rts.reactors[receiver], sender) == utils.Some(0) + }, time.Second, 50*time.Millisecond) + + tx := []byte("large=" + strings.Repeat("x", 200)) + require.Greater(t, len(tx), receiverMaxTxBytes) + require.LessOrEqual(t, len(tx), senderMaxTxBytes) + require.LessOrEqual(t, len(tx), types.MaxGossipTxBytes) + + // Test: gossip a tx above the receiver's toml max-tx-bytes and below the protocol cap. + _, err := rts.mempools[sender].CheckTx(ctx, tx) + require.NoError(t, err) + + // Verify: the receiver admits it, the connection survived, and the sender is not blacklisted. + rts.waitForTxns(t, []types.Tx{tx}, receiver) + require.Equal(t, 1, rts.mempools[receiver].Size()) + require.Equal(t, utils.Some(0), peerFailedCheckTxCount(rts.reactors[receiver], sender)) +} + +func TestBroadcastSkipsProtocolOversizedTx(t *testing.T) { + ctx := t.Context() + + // Setup: two connected reactors. CheckTx will not admit a protocol-oversized + // tx, so the oversized bytes are placed on the gossip list directly. + rts := setupReactors(ctx, t, 2) + t.Cleanup(leaktest.Check(t)) + + sender := rts.nodes[0] + receiver := rts.nodes[1] + rts.reactors[receiver].cfg.Broadcast = false + rts.start(t) + rts.network.Node(receiver).WaitForConnAndGet(ctx, sender) + sentBefore := p2p.ChannelOutMsgs(MempoolChannel) + + oversized := types.Tx(make([]byte, types.MaxGossipTxBytes+1)) + require.NoError(t, rts.mempools[sender].InsertReadyTxForTest(oversized)) + + okTx := types.Tx("gossip-ok=1") + _, err := rts.mempools[sender].CheckTx(ctx, okTx) + require.NoError(t, err) + + // Test: broadcast walks an oversized tx then a gossip-legal tx. + // Verify: only the legal tx is sent. + require.Eventually(t, func() bool { + found, missing := rts.mempools[receiver].SafeGetTxsForHashes([]types.TxHash{okTx.Hash()}) + return len(missing) == 0 && len(found) == 1 + }, time.Minute, 50*time.Millisecond) + require.Equal(t, sentBefore+1, p2p.ChannelOutMsgs(MempoolChannel)) + require.Equal(t, 1, rts.mempools[receiver].Size()) + _, missing := rts.mempools[receiver].SafeGetTxsForHashes([]types.TxHash{oversized.Hash()}) + require.Equal(t, []types.TxHash{oversized.Hash()}, missing) +} + +func TestReactorProtocolOversizedTxIsCounted(t *testing.T) { + // Setup: peer is already tracked so protocol oversize can increment the blacklist. + reactor, _ := setupReactorForTest(t, mempool.NopTxConstraintsFetcher) + reactor.cfg.CheckTxErrorBlacklistEnabled = true + for counts := range reactor.failedCheckTxCounts.Lock() { + counts["sender"] = 0 + } + + tx := make([]byte, types.MaxGossipTxBytes+1) + msg := p2p.RecvMsg[*pb.Message]{ + From: "sender", + Message: &pb.Message{ + Sum: &pb.Message_Txs{Txs: &pb.Txs{Txs: [][]byte{tx}}}, + }, + } + + // Test: a tx above the protocol gossip size is skipped and counted. + require.NoError(t, reactor.handleMempoolMessage(t.Context(), msg)) + + // Verify: the sender is charged one protocol violation and the tx is not in the mempool. + require.Equal(t, utils.Some(1), peerFailedCheckTxCount(reactor, "sender")) +} + func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") diff --git a/sei-tendermint/internal/mempool/testonly.go b/sei-tendermint/internal/mempool/testonly.go index 30132013c8..5a4c1f7287 100644 --- a/sei-tendermint/internal/mempool/testonly.go +++ b/sei-tendermint/internal/mempool/testonly.go @@ -4,6 +4,7 @@ import ( "time" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) func TestConfig() *Config { @@ -15,3 +16,17 @@ func TestConfig() *Config { cfg.TTLDuration = utils.None[time.Duration]() return cfg } + +// InsertReadyTxForTest adds tx to the gossip list without CheckTx. +func (txmp *TxMempool) InsertReadyTxForTest(tx types.Tx) error { + wtx := &WrappedTx{ + hashedTx: newHashedTx(tx), + timestamp: time.Now().UTC(), + height: txmp.height, + } + if err := txmp.txStore.Insert(wtx); err != nil { + return err + } + txmp.notifyTxsAvailable() + return nil +} diff --git a/sei-tendermint/internal/p2p/conn/oldmux.go b/sei-tendermint/internal/p2p/conn/oldmux.go index 9759547279..591b34035c 100644 --- a/sei-tendermint/internal/p2p/conn/oldmux.go +++ b/sei-tendermint/internal/p2p/conn/oldmux.go @@ -36,6 +36,10 @@ type ChannelDescriptorT[T gogoproto.Message] struct { SendQueueCapacity int RecvMessageCapacity int + // DiscardOversized drops a message that exceeds RecvMessageCapacity instead of + // closing the multiplexed connection. + DiscardOversized bool + // RecvBufferCapacity defines the max buffer size of inbound messages for a // given p2p Channel queue. RecvBufferCapacity int @@ -52,6 +56,7 @@ func (chDesc ChannelDescriptorT[T]) ToGeneric() ChannelDescriptor { MessageType: chDesc.MessageType, SendQueueCapacity: chDesc.SendQueueCapacity, RecvMessageCapacity: chDesc.RecvMessageCapacity, + DiscardOversized: chDesc.DiscardOversized, RecvBufferCapacity: chDesc.RecvBufferCapacity, Name: chDesc.Name, } @@ -379,7 +384,7 @@ func (c *MConnection) recvRoutine(ctx context.Context) (err error) { channels := map[ChannelID]*recvChannel{} for q := range c.sendQueue.Lock() { for _, ch := range q.channels { - channels[ch.desc.ID] = newRecvChannel(ch.desc) + channels[ch.desc.ID] = newRecvChannel(ch.desc, c.String()) } } @@ -471,14 +476,17 @@ func (ch *sendChannel) popMsg(maxPayload int) *pb.PacketMsg { } type recvChannel struct { - desc ChannelDescriptor - buf []byte + desc ChannelDescriptor + buf []byte + discarding bool + peer string } -func newRecvChannel(desc ChannelDescriptor) *recvChannel { +func newRecvChannel(desc ChannelDescriptor, peer string) *recvChannel { return &recvChannel{ desc: desc.withDefaults(), buf: make([]byte, 0, desc.RecvBufferCapacity), + peer: peer, } } @@ -486,8 +494,25 @@ func newRecvChannel(desc ChannelDescriptor) *recvChannel { // complete, which is owned by the caller and will not be modified. // Not goroutine-safe func (ch *recvChannel) pushMsg(packet *pb.PacketMsg) ([]byte, error) { + if ch.discarding { + if packet.Eof { + ch.discarding = false + } + return nil, nil + } if got, wantMax := len(ch.buf)+len(packet.Data), ch.desc.RecvMessageCapacity; got > wantMax { - return nil, fmt.Errorf("received message exceeds available capacity: %v < %v", wantMax, got) + if !ch.desc.DiscardOversized { + return nil, fmt.Errorf("received message exceeds available capacity: %v < %v", wantMax, got) + } + logger.Warn("discarding oversized p2p message", + "channel", ch.desc.Name, + "peer", ch.peer, + "capacity", wantMax, + "size", got, + ) + ch.buf = make([]byte, 0, ch.desc.RecvBufferCapacity) + ch.discarding = !packet.Eof + return nil, nil } ch.buf = append(ch.buf, packet.Data...) if packet.Eof { diff --git a/sei-tendermint/internal/p2p/conn/oldmux_test.go b/sei-tendermint/internal/p2p/conn/oldmux_test.go index 732928378f..1a2cbde865 100644 --- a/sei-tendermint/internal/p2p/conn/oldmux_test.go +++ b/sei-tendermint/internal/p2p/conn/oldmux_test.go @@ -270,6 +270,78 @@ func TestConnVectors(t *testing.T) { } } +func TestMConnectionRecvCapacityOverflow(t *testing.T) { + t.Cleanup(leaktest.CheckTimeout(t, 10*time.Second)) + err := scope.Run(t.Context(), func(ctx context.Context, s scope.Scope) error { + // Setup: receive capacity smaller than the message we will send. + chDescs := []*ChannelDescriptor{{ + ID: 0x01, Priority: 1, SendQueueCapacity: 1, + RecvMessageCapacity: 32, + }} + client, server := NewTestConn() + m1 := newMConnectionWithCh(client, chDescs) + m2 := newMConnectionWithCh(server, chDescs) + s.Spawn(func() error { + if err := m1.Run(ctx); err == nil { + return fmt.Errorf("expected recv capacity error, got nil") + } + return nil + }) + s.SpawnBg(func() error { return utils.IgnoreCancel(m2.Run(ctx)) }) + + // Test: deliver a message over RecvMessageCapacity. + if err := m2.Send(ctx, 0x01, make([]byte, 64)); err != nil { + return fmt.Errorf("m2.Send(): %w", err) + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +func TestMConnectionDiscardOversized(t *testing.T) { + t.Cleanup(leaktest.CheckTimeout(t, 10*time.Second)) + err := scope.Run(t.Context(), func(ctx context.Context, s scope.Scope) error { + // Setup: mempool-style channel that discards over-capacity messages. + chDescs := []*ChannelDescriptor{{ + ID: 0x01, Priority: 1, SendQueueCapacity: 1, + RecvMessageCapacity: 32, + DiscardOversized: true, + Name: "mempool", + }} + cfg := makeCfg() + cfg.MaxPacketMsgPayloadSize = 16 + client, server := NewTestConn() + m1 := newMConnectionWithCfg(client, chDescs, cfg) + m2 := newMConnectionWithCfg(server, chDescs, cfg) + s.SpawnBgNamed("m1", func() error { return utils.IgnoreCancel(m1.Run(ctx)) }) + s.SpawnBgNamed("m2", func() error { return utils.IgnoreCancel(m2.Run(ctx)) }) + + // Test: oversized fragmented message, then a valid follow-up. + if err := m2.Send(ctx, 0x01, make([]byte, 64)); err != nil { + return fmt.Errorf("m2.Send() oversized: %w", err) + } + want := []byte("ok") + if err := m2.Send(ctx, 0x01, want); err != nil { + return fmt.Errorf("m2.Send() follow-up: %w", err) + } + + // Verify: connection stays up and the follow-up is delivered. + _, got, err := m1.Recv(ctx) + if err != nil { + return fmt.Errorf("m1.Recv(): %w", err) + } + if err := utils.TestDiff(want, got); err != nil { + return fmt.Errorf("m1.Recv(): %v", err) + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + func TestMConnectionChannelOverflow(t *testing.T) { err := scope.Run(t.Context(), func(ctx context.Context, s scope.Scope) error { c1, c2 := NewTestConn() diff --git a/sei-tendermint/internal/p2p/testonly.go b/sei-tendermint/internal/p2p/testonly.go index 926bff7f8e..e8d3f1970d 100644 --- a/sei-tendermint/internal/p2p/testonly.go +++ b/sei-tendermint/internal/p2p/testonly.go @@ -12,6 +12,7 @@ import ( "github.com/gogo/protobuf/proto" gogotypes "github.com/gogo/protobuf/types" + dto "github.com/prometheus/client_model/go" "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/ed25519" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p/conn" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -24,6 +25,15 @@ import ( // Message is a simple message containing a string-typed Value field. type TestMessage = gogotypes.StringValue +// ChannelOutMsgs returns the process-wide count of messages sent on chID. +func ChannelOutMsgs(chID ChannelID) int64 { + var m dto.Metric + if err := Global.channelMsgsAt(fmt.Sprint(chID), "out").Write(&m); err != nil { + panic(err) + } + return int64(m.GetCounter().GetValue()) +} + func NodeInSlice(id types.NodeID, ids []types.NodeID) bool { for _, n := range ids { if id == n { diff --git a/sei-tendermint/types/params.go b/sei-tendermint/types/params.go index adee700bbc..bf0807a872 100644 --- a/sei-tendermint/types/params.go +++ b/sei-tendermint/types/params.go @@ -17,6 +17,9 @@ const ( // MaxBlockSizeBytes is the maximum permitted size of the blocks. MaxBlockSizeBytes = 104857600 // 100MB + // MaxGossipTxBytes is the largest transaction the mempool P2P channel will gossip. + MaxGossipTxBytes = 1024 * 1024 // 1MiB + // BlockPartSizeBytes is the size of one block part. BlockPartSizeBytes uint32 = 1048576 // 1MB