-
Notifications
You must be signed in to change notification settings - Fork 108
Expand file tree
/
Copy pathtest_pipeline_builder.py
More file actions
67 lines (58 loc) · 1.89 KB
/
test_pipeline_builder.py
File metadata and controls
67 lines (58 loc) · 1.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import unittest
from feldera.testutils import (
unique_pipeline_name,
FELDERA_TEST_NUM_WORKERS,
FELDERA_TEST_NUM_HOSTS,
)
from tests import TEST_CLIENT
from feldera import PipelineBuilder
from feldera.runtime_config import RuntimeConfig
class TestPipelineBuilder(unittest.TestCase):
def test_connector_orchestration(self):
sql = """
CREATE TABLE numbers (
num INT
) WITH (
'connectors' = '[
{
"name": "c1",
"paused": true,
"transport": {
"name": "datagen",
"config": {"plan": [{ "rate": 1, "fields": { "num": { "range": [0, 10], "strategy": "uniform" } } }]}
}
}
]'
);
"""
pipeline_name = unique_pipeline_name("test_connector_orchestration")
pipeline = PipelineBuilder(
TEST_CLIENT,
pipeline_name,
sql=sql,
runtime_config=RuntimeConfig(
workers=FELDERA_TEST_NUM_WORKERS,
hosts=FELDERA_TEST_NUM_HOSTS,
),
).create_or_replace()
pipeline.start()
pipeline.resume_connector("numbers", "c1")
stats = TEST_CLIENT.get_pipeline_stats(pipeline_name)
c1_status = next(
item["paused"]
for item in stats["inputs"]
if item["endpoint_name"] == "numbers.c1"
)
assert not c1_status
pipeline.pause_connector("numbers", "c1")
stats = TEST_CLIENT.get_pipeline_stats(pipeline_name)
c2_status = next(
item["paused"]
for item in stats["inputs"]
if item["endpoint_name"] == "numbers.c1"
)
assert c2_status
pipeline.stop(force=True)
pipeline.clear_storage()
if __name__ == "__main__":
unittest.main()