-
Notifications
You must be signed in to change notification settings - Fork 108
Expand file tree
/
Copy pathintegration_tests.py
More file actions
102 lines (85 loc) · 3.04 KB
/
integration_tests.py
File metadata and controls
102 lines (85 loc) · 3.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
import unittest
from typing import Optional
from feldera import PipelineBuilder
from feldera.runtime_config import RuntimeConfig, Storage
from tests import TEST_CLIENT
DEFAULT_ENDPOINT = "http://minio.extra.svc.cluster.local:9000"
DEFAULT_BUCKET = "default"
ACCESS_KEY = "minio"
SECRET_KEY = "miniopasswd"
def storage_cfg(
endpoint: Optional[str] = None,
start_from_checkpoint: bool = False,
auth_err: bool = False,
) -> dict:
return {
"backend": {
"name": "file",
"config": {
"sync": {
"bucket": DEFAULT_BUCKET,
"access_key": ACCESS_KEY,
"secret_key": SECRET_KEY if not auth_err else SECRET_KEY + "extra",
"provider": "Minio",
"endpoint": endpoint or DEFAULT_ENDPOINT,
"start_from_checkpoint": start_from_checkpoint,
}
},
}
}
class TestIntegrationTests(unittest.TestCase):
def test_checkpoint_sync(
self, auth_err: bool = False, name: Optional[str] = None, suspend: bool = False
):
c = TEST_CLIENT
storage_config = storage_cfg()
pipeline_name = name or "test_checkpoint_sync0"
sql = """
CREATE TABLE t0 (c0 int, c1 varchar);
CREATE MATERIALIZED VIEW v0 AS SELECT c0 FROM t0;
"""
# Create and start pipeline
p = PipelineBuilder(
c,
pipeline_name,
sql,
runtime_config=RuntimeConfig(storage=Storage(config=storage_config)),
).create_or_replace()
p.start()
# Insert data and trigger checkpoint
data = [{"c0": i, "c1": str(i)} for i in range(1, 4)]
p.input_json("t0", data)
p.execute("INSERT INTO t0 VALUES (4, 'exists')")
got_before = list(p.query("SELECT * FROM v0"))
print("got before:", got_before)
p.checkpoint(wait=True)
p.sync_checkpoint(wait=True)
if suspend:
p.suspend()
else:
p.shutdown()
# Restart from checkpoint
storage_config = storage_cfg(start_from_checkpoint=True, auth_err=auth_err)
p = PipelineBuilder(
c,
pipeline_name,
sql,
runtime_config=RuntimeConfig(storage=Storage(config=storage_config)),
).create_or_replace()
p.start()
got_after = list(p.query("SELECT * FROM v0"))
self.assertCountEqual(got_before, got_after)
if suspend:
p.suspend()
else:
p.shutdown()
def test_checkpoint_sync_suspend(self):
name = "test_checkpoint_sync0"
self.test_checkpoint_sync(name=name, suspend=True)
TEST_CLIENT.shutdown_pipeline(name)
def test_checkpoint_sync_err(self):
name = "test_checkpoint_sync0"
with self.assertRaisesRegex(RuntimeError, "SignatureDoesNotMatch"):
self.test_checkpoint_sync(name=name, auth_err=True)
TEST_CLIENT.shutdown_pipeline(name)
TEST_CLIENT.delete_pipeline(name)