-
Notifications
You must be signed in to change notification settings - Fork 107
Expand file tree
/
Copy pathpipeline.py
More file actions
94 lines (82 loc) · 3.66 KB
/
pipeline.py
File metadata and controls
94 lines (82 loc) · 3.66 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
from typing import Mapping, Any, Optional
from feldera.rest.sql_table import SQLTable
from feldera.rest.sql_view import SQLView
class Pipeline:
"""
Represents a Feldera pipeline
"""
def __init__(
self,
name: str,
sql: str,
udf_rust: str,
udf_toml: str,
program_config: Mapping[str, Any],
runtime_config: Mapping[str, Any],
description: Optional[str] = None,
):
"""
Initializes a new pipeline
:param name: The name of the pipeline
:param sql: The SQL code of the pipeline
:param udf_rust: Rust code for UDFs
:param udf_toml: Rust dependencies required by UDFs (in the TOML format)
:param program_config: The program config of the pipeline
:param runtime_config: The configuration of the pipeline
:param description: Optional. The description of the pipeline
"""
self.name: str = name
self.program_code: str = sql.strip()
self.udf_rust: str = udf_rust
self.udf_toml: str = udf_toml
self.description: Optional[str] = description
self.program_config: Mapping[str, Any] = program_config
self.runtime_config: Mapping[str, Any] = runtime_config
self.id: Optional[str] = None
self.tables: list[SQLTable] = []
self.views: list[SQLView] = []
self.deployment_status: Optional[str] = None
self.deployment_status_since: Optional[str] = None
self.created_at: Optional[str] = None
self.version: Optional[int] = None
self.program_version: Optional[int] = None
self.deployment_config: Optional[dict] = None
self.deployment_desired_status: Optional[str] = None
self.deployment_desired_status_since: Optional[str] = None
self.deployment_id: Optional[str] = None
self.deployment_initial: Optional[str] = None
self.bootstrap_policy: Optional[str] = None
self.deployment_error: Optional[dict] = None
self.deployment_location: Optional[str] = None
self.program_info: Optional[dict] = (
None # info about input & output connectors and the schema
)
self.program_status: Optional[str] = None
self.program_status_since: Optional[str] = None
self.platform_version: Optional[str] = None
self.program_error: Optional[dict] = None
self.storage_status: Optional[str] = None
self.deployment_resources_desired_status: Optional[str] = None
self.deployment_resources_desired_status_since: Optional[str] = None
self.deployment_resources_status: Optional[str] = None
self.deployment_resources_status_since: Optional[str] = None
self.deployment_runtime_desired_status: Optional[str] = None
self.deployment_runtime_desired_status_since: Optional[str] = None
self.deployment_runtime_status: Optional[str] = None
self.deployment_runtime_status_details: Optional[dict] = None
self.deployment_runtime_status_since: Optional[str] = None
@classmethod
def from_dict(cls, d: Mapping[str, Any]):
pipeline = cls("", "", "", "", {}, {})
pipeline.__dict__ = d
pipeline.tables = []
pipeline.views = []
info = d.get("program_info")
if info is not None:
for i in info["schema"]["inputs"]:
tbl = SQLTable.from_dict(i)
pipeline.tables.append(tbl)
for output in info["schema"]["outputs"]:
v = SQLView.from_dict(output)
pipeline.views.append(v)
return pipeline