forked from feldera/feldera
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathruntime_config.py
More file actions
94 lines (77 loc) · 3.17 KB
/
runtime_config.py
File metadata and controls
94 lines (77 loc) · 3.17 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
from typing import Optional, Any, Mapping
class Resources:
"""
Class used to specify the resource configuration for a pipeline.
:param config: A dictionary containing all the configuration values.
:param cpu_cores_max: The maximum number of CPU cores to reserve for an instance of the pipeline.
:param cpu_cores_min: The minimum number of CPU cores to reserve for an instance of the pipeline.
:param memory_mb_max: The maximum memory in Megabytes to reserve for an instance of the pipeline.
:param memory_mb_min: The minimum memory in Megabytes to reserve for an instance of the pipeline.
:param storage_class: The storage class to use for the pipeline. The class determines storage performance such
as IOPS and throughput.
:param storage_mb_max: The storage in Megabytes to reserve for an instance of the pipeline.
"""
def __init__(
self,
config: Optional[Mapping[str, Any]] = None,
cpu_cores_max: Optional[int] = None,
cpu_cores_min: Optional[int] = None,
memory_mb_max: Optional[int] = None,
memory_mb_min: Optional[int] = None,
storage_class: Optional[str] = None,
storage_mb_max: Optional[int] = None,
):
config = config or {}
self.cpu_cores_max = cpu_cores_max
self.cpu_cores_min = cpu_cores_min
self.memory_mb_max = memory_mb_max
self.memory_mb_min = memory_mb_min
self.storage_class = storage_class
self.storage_mb_max = storage_mb_max
self.__dict__.update(config)
class Storage:
"""Storage configuration for a pipeline.
:param min_storage_bytes: The minimum estimated number of bytes in a batch of data to write it to storage.
"""
def __init__(
self,
config: Optional[Mapping[str, Any]] = None,
min_storage_bytes: Optional[int] = None,
):
config = config or {}
self.min_storage_bytes = min_storage_bytes
self.__dict__.update(config)
class RuntimeConfig:
"""
Runtime configuration class to define the configuration for a pipeline.
"""
def __init__(
self,
workers: Optional[int] = None,
storage: Optional[Storage] = None,
tracing: Optional[bool] = False,
tracing_endpoint_jaeger: Optional[str] = "",
cpu_profiler: bool = True,
max_buffering_delay_usecs: int = 0,
min_batch_size_records: int = 0,
clock_resolution_usecs: Optional[int] = None,
resources: Optional[Resources] = None,
):
self.workers = workers
self.storage = storage
self.tracing = tracing
self.tracing_endpoint_jaeger = tracing_endpoint_jaeger
self.cpu_profiler = cpu_profiler
self.max_buffering_delay_usecs = max_buffering_delay_usecs
self.min_batch_size_records = min_batch_size_records
self.clock_resolution_usecs = clock_resolution_usecs
if resources is not None:
self.resources = resources.__dict__
@classmethod
def from_dict(cls, d: Mapping[str, Any]):
"""
Create a `.RuntimeConfig` object from a dictionary.
"""
conf = cls()
conf.__dict__ = d
return conf