forked from evstack/ev-node
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathfull_node_integration_test.go
More file actions
481 lines (392 loc) · 17.8 KB
/
Copy pathfull_node_integration_test.go
File metadata and controls
481 lines (392 loc) · 17.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
//go:build integration
package node
import (
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
evconfig "github.com/evstack/ev-node/pkg/config"
)
// TestTxGossipingMultipleNodesNoDA tests that transactions are gossiped and blocks are sequenced and synced across multiple nodes without the DA layer over P2P.
// It creates 3 nodes (1 sequencer, 2 full nodes), injects a transaction, waits for all nodes to sync, and asserts block equality.
func TestTxGossipingMultipleNodesNoDA(t *testing.T) {
require := require.New(t)
config := getTestConfig(t, 1)
// Set the DA block time to a very large value to ensure that the DA layer is not used
config.DA.BlockTime = evconfig.DurationWrapper{Duration: 100 * time.Second}
numNodes := 3
nodes, cleanups := createNodesWithCleanup(t, numNodes, config)
for _, cleanup := range cleanups {
defer cleanup()
}
ctxs, cancels := createNodeContexts(numNodes)
var runningWg sync.WaitGroup
errChan := make(chan error, numNodes)
// Start only the sequencer first
startNodeInBackground(t, nodes, ctxs, &runningWg, 0, errChan)
// Wait for the first block to be produced by the sequencer
err := waitForFirstBlock(nodes[0], Header)
require.NoError(err)
// Add a small delay to ensure P2P services are fully ready
time.Sleep(500 * time.Millisecond)
// Start the other nodes
for i := 1; i < numNodes; i++ {
startNodeInBackground(t, nodes, ctxs, &runningWg, i, errChan)
// Add a small delay between starting nodes to avoid connection race
if i < numNodes-1 {
time.Sleep(100 * time.Millisecond)
}
}
// Inject a transaction into the sequencer's executor
if state := castState(t, nodes[0]); state.bc.Executor != nil {
// Access the core executor from the block executor
coreExec := state.bc.Executor.GetCoreExecutor()
if dummyExec, ok := coreExec.(interface{ InjectTx([]byte) }); ok {
dummyExec.InjectTx([]byte("test tx"))
} else {
t.Fatal("Warning: Could not cast core executor to DummyExecutor")
}
} else {
t.Fatal("executor empty")
}
blocksToWaitFor := uint64(3)
// Wait for all nodes to reach at least blocksToWaitFor blocks
for i, nodeItem := range nodes {
requireEmptyChan(t, errChan)
require.NoError(waitForAtLeastNBlocks(nodeItem, blocksToWaitFor, Store), "node %d", i)
}
// Assert that all nodes have the same block up to height blocksToWaitFor
assertAllNodesSynced(t, nodes, blocksToWaitFor)
// Shutdown all nodes and wait
shutdownAndWait(t, cancels, &runningWg, 10*time.Second)
}
// TestTxGossipingMultipleNodesDAIncluded tests that transactions are gossiped and blocks are sequenced and synced across multiple nodes only using DA. P2P gossiping is disabled.
// It creates 4 nodes (1 sequencer, 3 full nodes), injects a transaction, waits for all nodes to sync with DA inclusion, and asserts block equality.
func TestTxGossipingMultipleNodesDAIncluded(t *testing.T) {
require := require.New(t)
config := getTestConfig(t, 1)
// Disable P2P gossiping
config.P2P.Peers = "none"
numNodes := 4
nodes, cleanups := createNodesWithCleanup(t, numNodes, config)
ctxs, cancels := createNodeContexts(numNodes)
var runningWg sync.WaitGroup
errChan := make(chan error, numNodes)
// Start only the sequencer first
startNodeInBackground(t, nodes, ctxs, &runningWg, 0, errChan)
t.Cleanup(func() { shutdownAndWait(t, cleanups, &runningWg, 10*time.Second) })
// Wait for the first block to be produced by the sequencer
err := waitForFirstBlock(nodes[0], Header)
require.NoError(err)
// Verify block components are properly initialized
require.True(nodes[0].IsRunning(), "Block components should be initialized")
// Add a small delay to ensure P2P services are fully ready
time.Sleep(500 * time.Millisecond)
// Start the other nodes
for i := 1; i < numNodes; i++ {
startNodeInBackground(t, nodes, ctxs, &runningWg, i, errChan)
// Add a small delay between starting nodes to avoid connection race
if i < numNodes-1 {
time.Sleep(100 * time.Millisecond)
}
}
// Inject transactions into the sequencer's executor
if state := castState(t, nodes[0]); state != nil && state.bc.Executor != nil {
// Access the core executor from the block executor
coreExec := state.bc.Executor.GetCoreExecutor()
if dummyExec, ok := coreExec.(interface{ InjectTx([]byte) }); ok {
dummyExec.InjectTx([]byte("test tx 1"))
dummyExec.InjectTx([]byte("test tx 2"))
dummyExec.InjectTx([]byte("test tx 3"))
} else {
t.Fatalf("Could not cast core executor to DummyExecutor")
}
}
blocksToWaitFor := uint64(5)
// Wait for all nodes to reach at least blocksToWaitFor blocks with DA inclusion
for _, nodeItem := range nodes {
requireEmptyChan(t, errChan)
require.NoError(waitForAtLeastNDAIncludedHeight(nodeItem, blocksToWaitFor))
}
// Assert that all nodes have the same block up to height blocksToWaitFor
assertAllNodesSynced(t, nodes, blocksToWaitFor)
// Shutdown all nodes and wait
shutdownAndWait(t, cancels, &runningWg, 5*time.Second)
}
func castState(t *testing.T, node *FullNode) *failoverState {
v, ok := node.leaderElection.(testSupportElection)
require.True(t, ok)
state := v.state()
require.NotNil(t, state)
return state
}
// TestFastDASync verifies that a new node can quickly synchronize with the DA layer using fast sync.
//
// This test sets up two nodes with different block and DA block times. It starts the sequencer node, waits for it to produce and DA-include several blocks,
// then starts the syncing full node and measures how quickly it can catch up to the sequencer node's height. The test asserts that the syncing full node syncs within
// a small delta of the DA block time, and verifies that both nodes have identical block hashes and that all blocks are DA-included.
func TestFastDASync(t *testing.T) {
require := require.New(t)
// Set up two nodes where DA is faster than block production
config := getTestConfig(t, 1)
// Slow block production (sequencer takes 1s per block)
config.Node.BlockTime = evconfig.DurationWrapper{Duration: 1 * time.Second}
// Fast DA availability (DA makes blocks available every 200ms)
config.DA.BlockTime = evconfig.DurationWrapper{Duration: 200 * time.Millisecond}
nodes, cleanups := createNodesWithCleanup(t, 2, config)
ctxs, cancels := createNodeContexts(len(nodes))
var runningWg sync.WaitGroup
errChan := make(chan error, len(nodes))
// Start only the first node
startNodeInBackground(t, nodes, ctxs, &runningWg, 0, errChan)
t.Cleanup(func() { shutdownAndWait(t, cleanups, &runningWg, 10*time.Second) })
// Wait for the first node to produce a few blocks
blocksToWaitFor := uint64(2)
require.NoError(waitForAtLeastNDAIncludedHeight(nodes[0], blocksToWaitFor))
// Add a small delay to ensure P2P services are fully ready
time.Sleep(500 * time.Millisecond)
// Now start the second node and time its sync
startNodeInBackground(t, nodes, ctxs, &runningWg, 1, errChan)
start := time.Now()
// Wait for the second node to catch up to the first node
require.NoError(waitForAtLeastNBlocks(nodes[1], blocksToWaitFor, Store))
syncDuration := time.Since(start)
// The key test: sync should be much faster than sequential block production time
// Since DA provides blocks faster than sequencer block time, sync should complete
// in significantly less time than it took the sequencer to produce them sequentially
expectedSequentialTime := time.Duration(blocksToWaitFor) * config.Node.BlockTime.Duration
maxReasonableSyncTime := expectedSequentialTime / 3 // Should be at least 3x faster than sequential
require.Less(syncDuration, maxReasonableSyncTime,
"DA fast sync took %v, should be much faster than sequential block time %v (max reasonable: %v). ",
syncDuration, expectedSequentialTime, maxReasonableSyncTime)
requireEmptyChan(t, errChan)
// Verify both nodes are synced and that the synced block is DA-included
assertAllNodesSynced(t, nodes, blocksToWaitFor)
// Cancel all node contexts to signal shutdown and wait
shutdownAndWait(t, cancels, &runningWg, 5*time.Second)
}
// TestSingleSequencerTwoFullNodesBlockSyncSpeed tests that block synchronization is not bottlenecked by DA block time.
//
// This test sets up three nodes (one sequencer and two full nodes) with a fast block time and a slow DA block time. It starts the sequencer first, waits for it to produce a block, then starts the full nodes. The test waits for all nodes to sync a set number of blocks, measures the total sync duration, and asserts that block sync completes within a reasonable multiple of the block time (not the DA block time). It also verifies that all nodes have identical block hashes up to the target height.
func TestSingleSequencerTwoFullNodesBlockSyncSpeed(t *testing.T) {
require := require.New(t)
// Set up three nodes: 1 sequencer, 2 full nodes
config := getTestConfig(t, 1)
config.Node.BlockTime = evconfig.DurationWrapper{Duration: 100 * time.Millisecond} // fast block time
config.DA.BlockTime = evconfig.DurationWrapper{Duration: 10 * time.Second} // slow DA block time
numNodes := 3
nodes, cleanups := createNodesWithCleanup(t, numNodes, config)
for _, cleanup := range cleanups {
defer cleanup()
}
ctxs, cancels := createNodeContexts(numNodes)
var runningWg sync.WaitGroup
errChan := make(chan error, numNodes)
// Start only the sequencer first
startNodeInBackground(t, nodes, ctxs, &runningWg, 0, errChan)
// Wait for the sequencer to produce at first block
require.NoError(waitForFirstBlock(nodes[0], Store))
// Add a small delay to ensure P2P services are fully ready
time.Sleep(500 * time.Millisecond)
// Now start the other nodes
for i := 1; i < numNodes; i++ {
startNodeInBackground(t, nodes, ctxs, &runningWg, i, errChan)
// Add a small delay between starting nodes to avoid connection race
if i < numNodes-1 {
time.Sleep(100 * time.Millisecond)
}
}
blocksToWaitFor := uint64(10)
start := time.Now()
// Wait for all nodes to reach the target block height
for _, nodeItem := range nodes {
requireEmptyChan(t, errChan)
require.NoError(waitForAtLeastNBlocks(nodeItem, blocksToWaitFor, Store))
}
totalDuration := time.Since(start)
// The test should complete within a reasonable multiple of block time, not DA block time
maxExpected := config.Node.BlockTime.Duration*time.Duration(blocksToWaitFor) + 200*time.Millisecond
require.Less(totalDuration, maxExpected, "Block sync should not be bottlenecked by DA block time")
for i := 1; i < numNodes; i++ {
require.NoError(verifyNodesSynced(nodes[0], nodes[i], Store))
}
// Cancel all node contexts to signal shutdown and wait
shutdownAndWait(t, cancels, &runningWg, 5*time.Second)
}
// TestDataExchange verifies data exchange and synchronization between nodes in various network topologies.
//
// This test runs two sub-tests:
// 1. Single sequencer and single full
// 2. Single sequencer and two full nodes.
//
// Each sub-test checks data exchange and synchronization to ensure correct data propagation and consistency across nodes.
func TestDataExchange(t *testing.T) {
t.Run("SingleSequencerSingleFullNode", func(t *testing.T) {
testSingleSequencerSingleFullNode(t, Data)
})
t.Run("SingleSequencerTwoFullNodes", func(t *testing.T) {
testSingleSequencerTwoFullNodes(t, Data)
})
}
// TestHeaderExchange verifies header exchange and synchronization between nodes in various network topologies.
//
// This test runs two sub-tests:
// 1. Single sequencer and single full
// 2. Single sequencer and two full nodes.
//
// Each sub-test checks header exchange and synchronization to ensure correct header propagation and consistency across nodes.
func TestHeaderExchange(t *testing.T) {
t.Run("SingleSequencerSingleFullNode", func(t *testing.T) {
testSingleSequencerSingleFullNode(t, Header)
})
t.Run("SingleSequencerTwoFullNodes", func(t *testing.T) {
testSingleSequencerTwoFullNodes(t, Header)
})
}
// testSingleSequencerSingleFullNode sets up a single sequencer and a single full node, starts the sequencer, waits for it to produce a block, then starts the full
// It waits for both nodes to reach a target block height (using the provided 'source' to determine block inclusion), verifies that both nodes are fully synced, and then shuts them down.
func testSingleSequencerSingleFullNode(t *testing.T, source Source) {
require := require.New(t)
// Set up one sequencer and one full node
config := getTestConfig(t, 1)
numNodes := 2
nodes, cleanups := createNodesWithCleanup(t, numNodes, config)
for _, cleanup := range cleanups {
defer cleanup()
}
ctxs, cancels := createNodeContexts(numNodes)
var runningWg sync.WaitGroup
errChan := make(chan error, numNodes)
// Start the sequencer first
startNodeInBackground(t, nodes, ctxs, &runningWg, 0, errChan)
t.Cleanup(func() { shutdownAndWait(t, cancels, &runningWg, 10*time.Second) })
// Wait for the sequencer to produce at first block
require.NoError(waitForFirstBlock(nodes[0], source))
// Add a small delay to ensure P2P services are fully ready
time.Sleep(500 * time.Millisecond)
// Start the full node
startNodeInBackground(t, nodes, ctxs, &runningWg, 1, errChan)
blocksToWaitFor := uint64(3)
// Wait for both nodes to reach at least blocksToWaitFor blocks
for _, nodeItem := range nodes {
requireEmptyChan(t, errChan)
require.NoError(waitForAtLeastNBlocks(nodeItem, blocksToWaitFor, source))
}
// Verify both nodes are synced using the helper
require.NoError(verifyNodesSynced(nodes[0], nodes[1], source))
}
// testSingleSequencerTwoFullNodes sets up a single sequencer and two full nodes, starts the sequencer, waits for it to produce a block, then starts the full nodes.
// It waits for all nodes to reach a target block height (using the provided 'source' to determine block inclusion), verifies that all nodes are fully synced, and then shuts them down.
func testSingleSequencerTwoFullNodes(t *testing.T, source Source) {
require := require.New(t)
// Set up one sequencer and two full nodes
config := getTestConfig(t, 1)
numNodes := 3
nodes, cleanups := createNodesWithCleanup(t, numNodes, config)
for _, cleanup := range cleanups {
defer cleanup()
}
ctxs, cancels := createNodeContexts(numNodes)
var runningWg sync.WaitGroup
errChan := make(chan error, numNodes)
// Start the sequencer first
startNodeInBackground(t, nodes, ctxs, &runningWg, 0, errChan)
t.Cleanup(func() { shutdownAndWait(t, cancels, &runningWg, 10*time.Second) })
// Wait for the sequencer to produce at first block
require.NoError(waitForFirstBlock(nodes[0], source))
// Add a small delay to ensure P2P services are fully ready
time.Sleep(500 * time.Millisecond)
// Start the full nodes
for i := 1; i < numNodes; i++ {
startNodeInBackground(t, nodes, ctxs, &runningWg, i, errChan)
// Add a small delay between starting nodes to avoid connection race
if i < numNodes-1 {
time.Sleep(100 * time.Millisecond)
}
}
blocksToWaitFor := uint64(3)
// Wait for all nodes to reach at least blocksToWaitFor blocks
for _, nodeItem := range nodes {
requireEmptyChan(t, errChan)
require.NoError(waitForAtLeastNBlocks(nodeItem, blocksToWaitFor, source))
}
// Verify all nodes are synced using the helper
for i := 1; i < numNodes; i++ {
require.NoError(verifyNodesSynced(nodes[0], nodes[i], source))
}
// Cancel all node contexts to signal shutdown and wait
shutdownAndWait(t, cancels, &runningWg, 5*time.Second)
}
// TestTwoChainsInOneNamespace verifies that two chains in the same namespace can coexist without any issues.
func TestTwoChainsInOneNamespace(t *testing.T) {
cases := []struct {
name string
chainID1 string
chainID2 string
}{
{
name: "same chain ID",
chainID1: "test-1",
chainID2: "test-1",
},
{
name: "different chain IDs",
chainID1: "foo-1",
chainID2: "bar-2",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
testTwoChainsInOneNamespace(t, c.chainID1, c.chainID2)
})
}
}
// testTwoChainsInOneNamespace sets up two chains in the same namespace, starts the sequencers, and waits for blocks.
// It waits for all nodes to reach a target block height, and verifies that all nodes are fully synced, and then shuts them down.
func testTwoChainsInOneNamespace(t *testing.T, chainID1 string, chainID2 string) {
require := require.New(t)
// Set up nodes for the first chain
configChain1 := getTestConfig(t, 1)
nodes1, cleanups := createNodesWithCleanup(t, 1, configChain1)
for _, cleanup := range cleanups {
defer cleanup()
}
// Set up nodes for the second chain
configChain2 := getTestConfig(t, 1000)
nodes2, cleanups := createNodesWithCleanup(t, 1, configChain2)
for _, cleanup := range cleanups {
defer cleanup()
}
// Set up context and wait group for the sequencer of chain 1
ctxs1, cancels1 := createNodeContexts(1)
var runningWg1 sync.WaitGroup
errChan := make(chan error, 2)
// Start the sequencer of chain 1
startNodeInBackground(t, nodes1, ctxs1, &runningWg1, 0, errChan)
// Wait for the sequencer to produce at first block
require.NoError(waitForFirstBlock(nodes1[0], Store))
// Set up context and wait group for the sequencer of chain 2
ctxs2, cancels2 := createNodeContexts(1)
var runningWg2 sync.WaitGroup
// Start the sequencer of chain 2
startNodeInBackground(t, nodes2, ctxs2, &runningWg2, 0, errChan)
// Wait for the sequencer to produce at first block
require.NoError(waitForFirstBlock(nodes2[0], Store))
blocksToWaitFor := uint64(3)
requireEmptyChan(t, errChan)
// Wait for the full node of chain 1 to reach at least blocksToWaitFor blocks
require.NoError(waitForAtLeastNBlocks(nodes1[0], blocksToWaitFor, Store))
// Wait for the full node of chain 2 to reach at least blocksToWaitFor blocks
require.NoError(waitForAtLeastNBlocks(nodes2[0], blocksToWaitFor, Store))
// Cancel all node contexts to signal shutdown and wait for both chains
shutdownAndWait(t, cancels1, &runningWg1, 5*time.Second)
shutdownAndWait(t, cancels2, &runningWg2, 5*time.Second)
}
func requireEmptyChan(t *testing.T, errChan chan error) {
t.Helper()
select {
case err := <-errChan:
t.Fatalf("Error received: %v", err)
default:
}
}