Skip to content

Commit 837e34f

Browse files
franciscojavierarceofranciscojavierarceo
committed
feat: Add unified transformation
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
1 parent b59fa4d commit 837e34f

6 files changed

Lines changed: 747 additions & 7 deletions

File tree

‎sdk/python/feast/feature_store.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -964,6 +964,29 @@ def apply(
964964
services_to_update,
965965
)
966966

967+
# Handle dual registration for online_enabled FeatureViews
968+
online_enabled_views = [
969+
view for view in views_to_update
970+
if hasattr(view, 'online_enabled') and view.online_enabled
971+
]
972+
973+
for fv in online_enabled_views:
974+
# Create OnDemandFeatureView for online serving with same transformation
975+
if hasattr(fv, 'feature_transformation') and fv.feature_transformation:
976+
# Create ODFV with same transformation logic
977+
online_fv = OnDemandFeatureView(
978+
name=f"{fv.name}_online",
979+
sources=fv.source_views or [], # Use source views for ODFV
980+
schema=fv.schema or [],
981+
feature_transformation=fv.feature_transformation, # Same transformation!
982+
description=f"Online serving for {fv.name}",
983+
tags=dict(fv.tags or {}, **{"generated_from": fv.name, "dual_registration": "true"}),
984+
owner=fv.owner,
985+
)
986+
987+
# Add to ODFVs to be registered
988+
odfvs_to_update.append(online_fv)
989+
967990
# Add all objects to the registry and update the provider's infrastructure.
968991
for project in projects_to_update:
969992
self._registry.apply_project(project, commit=False)

‎sdk/python/feast/feature_view.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@
3939
from feast.protos.feast.core.Transformation_pb2 import (
4040
FeatureTransformationV2 as FeatureTransformationProto,
4141
)
42-
from feast.transformation.mode import TransformationMode
42+
from feast.transformation.base import Transformation
43+
from feast.transformation.mode import TransformationMode, TransformationTiming
4344
from feast.types import from_value_type
4445
from feast.value_type import ValueType
4546

@@ -107,6 +108,9 @@ class FeatureView(BaseFeatureView):
107108
owner: str
108109
materialization_intervals: List[Tuple[datetime, datetime]]
109110
mode: Optional[Union["TransformationMode", str]]
111+
feature_transformation: Optional[Transformation]
112+
when: Optional[Union[TransformationTiming, str]]
113+
online_enabled: bool
110114

111115
def __init__(
112116
self,
@@ -123,6 +127,9 @@ def __init__(
123127
tags: Optional[Dict[str, str]] = None,
124128
owner: str = "",
125129
mode: Optional[Union["TransformationMode", str]] = None,
130+
feature_transformation: Optional[Transformation] = None,
131+
when: Optional[Union[TransformationTiming, str]] = None,
132+
online_enabled: bool = False,
126133
):
127134
"""
128135
Creates a FeatureView object.
@@ -148,6 +155,12 @@ def __init__(
148155
primary maintainer.
149156
mode (optional): The transformation mode for feature transformations. Only meaningful
150157
when transformations are applied. Choose from TransformationMode enum values.
158+
feature_transformation (optional): The transformation object containing the UDF and
159+
mode for this feature view. Used for derived feature views.
160+
when (optional): The timing for when transformation should execute. Choose from
161+
TransformationTiming enum values (on_read, on_write, batch, streaming).
162+
online_enabled (optional): Whether to enable dual registration for both batch
163+
materialization and online serving with Feature Server.
151164
152165
Raises:
153166
ValueError: A field mapping conflicts with an Entity or a Feature.
@@ -157,6 +170,11 @@ def __init__(
157170
self.ttl = ttl
158171
schema = schema or []
159172
self.mode = mode
173+
# Don't override feature_transformation if it's already set by subclass (e.g., BatchFeatureView)
174+
if not hasattr(self, 'feature_transformation') or self.feature_transformation is None:
175+
self.feature_transformation = feature_transformation
176+
self.when = when
177+
self.online_enabled = online_enabled
160178

161179
# Normalize source
162180
self.stream_source = None

‎sdk/python/feast/transformation/base.py‎

Lines changed: 107 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import functools
22
from abc import ABC
3-
from typing import Any, Callable, Dict, Optional, Union
3+
from typing import Any, Callable, Dict, List, Optional, Union
44

55
import dill
66

@@ -14,7 +14,26 @@
1414
TRANSFORMATION_CLASS_FOR_TYPE,
1515
get_transformation_class_from_type,
1616
)
17-
from feast.transformation.mode import TransformationMode
17+
from feast.transformation.mode import TransformationMode, TransformationTiming
18+
from feast.entity import Entity
19+
from feast.field import Field
20+
21+
# Online compatibility constants
22+
ONLINE_COMPATIBLE_MODES = {"python", "pandas"}
23+
BATCH_ONLY_MODES = {"sql", "spark_sql", "spark", "ray", "substrait"}
24+
25+
26+
def is_online_compatible(mode: str) -> bool:
27+
"""
28+
Check if a transformation mode can run online in Feature Server.
29+
30+
Args:
31+
mode: The transformation mode string
32+
33+
Returns:
34+
True if the mode can run in Feature Server, False if batch-only
35+
"""
36+
return mode.lower() in ONLINE_COMPATIBLE_MODES
1837

1938

2039
class Transformation(ABC):
@@ -117,7 +136,12 @@ def infer_features(self, *args, **kwargs) -> Any:
117136

118137

119138
def transformation(
120-
mode: Union[TransformationMode, str],
139+
mode: Union[TransformationMode, str], # Support both enum and string
140+
when: Optional[str] = None,
141+
online: Optional[bool] = None,
142+
sources: Optional[List[Union["FeatureView", "FeatureViewProjection", "RequestSource"]]] = None,
143+
schema: Optional[List[Field]] = None,
144+
entities: Optional[List[Entity]] = None,
121145
name: Optional[str] = None,
122146
tags: Optional[Dict[str, str]] = None,
123147
description: Optional[str] = "",
@@ -130,18 +154,95 @@ def mainify(obj):
130154
obj.__module__ = "__main__"
131155

132156
def decorator(user_function):
157+
# Validate mode (handle both enum and string)
158+
if isinstance(mode, TransformationMode):
159+
mode_str = mode.value
160+
else:
161+
mode_str = mode.lower() # Normalize to lowercase
162+
try:
163+
mode_enum = TransformationMode(mode_str)
164+
except ValueError:
165+
valid_modes = [m.value for m in TransformationMode]
166+
raise ValueError(f"Invalid mode '{mode}'. Valid options: {valid_modes}")
167+
168+
# Validate timing if provided
169+
timing_enum = None
170+
if when is not None:
171+
try:
172+
timing_enum = TransformationTiming(when.lower())
173+
except ValueError:
174+
valid_timings = [t.value for t in TransformationTiming]
175+
raise ValueError(f"Invalid timing '{when}'. Valid options: {valid_timings}")
176+
177+
# Validate online compatibility
178+
if online and not is_online_compatible(mode_str):
179+
compatible_modes = list(ONLINE_COMPATIBLE_MODES)
180+
raise ValueError(
181+
f"Mode '{mode_str}' cannot run online in Feature Server. "
182+
f"Use {compatible_modes} for online transformations."
183+
)
184+
185+
# Create transformation object
133186
udf_string = dill.source.getsource(user_function)
134187
mainify(user_function)
135188
transformation_obj = Transformation(
136-
mode=mode,
189+
mode=mode_str,
137190
name=name or user_function.__name__,
138191
tags=tags,
139192
description=description,
140193
owner=owner,
141194
udf=user_function,
142195
udf_string=udf_string,
143196
)
144-
functools.update_wrapper(wrapper=transformation_obj, wrapped=user_function)
145-
return transformation_obj
197+
198+
# If FeatureView parameters are provided, create and return FeatureView
199+
if any(param is not None for param in [when, online, sources, schema, entities]):
200+
# Import FeatureView here to avoid circular imports
201+
from feast.feature_view import FeatureView
202+
203+
# Validate required parameters when creating FeatureView
204+
if when is None:
205+
raise ValueError("'when' parameter is required when creating FeatureView")
206+
if online is None:
207+
raise ValueError("'online' parameter is required when creating FeatureView")
208+
if sources is None:
209+
raise ValueError("'sources' parameter is required when creating FeatureView")
210+
if schema is None:
211+
raise ValueError("'schema' parameter is required when creating FeatureView")
212+
213+
# Handle source parameter correctly for FeatureView constructor
214+
if not sources:
215+
raise ValueError("At least one source must be provided for FeatureView")
216+
elif len(sources) == 1:
217+
# Single source - pass directly (works for DataSource or FeatureView)
218+
source_param = sources[0]
219+
else:
220+
# Multiple sources - pass as list (must be List[FeatureView])
221+
from feast.feature_view import FeatureView as FV
222+
for src in sources:
223+
if not isinstance(src, (FV, type(src).__name__ == 'FeatureView')):
224+
raise ValueError("Multiple sources must be FeatureViews, not DataSources")
225+
source_param = sources
226+
227+
# Create FeatureView with transformation
228+
fv = FeatureView(
229+
name=name or user_function.__name__,
230+
source=source_param,
231+
entities=entities or [],
232+
schema=schema,
233+
feature_transformation=transformation_obj,
234+
when=when,
235+
online_enabled=online,
236+
description=description,
237+
tags=tags,
238+
owner=owner,
239+
mode=mode_str,
240+
)
241+
functools.update_wrapper(wrapper=fv, wrapped=user_function)
242+
return fv
243+
else:
244+
# Backward compatibility: return Transformation object
245+
functools.update_wrapper(wrapper=transformation_obj, wrapped=user_function)
246+
return transformation_obj
146247

147248
return decorator

‎sdk/python/feast/transformation/mode.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,10 @@ class TransformationMode(Enum):
99
RAY = "ray"
1010
SQL = "sql"
1111
SUBSTRAIT = "substrait"
12+
13+
14+
class TransformationTiming(Enum):
15+
ON_READ = "on_read" # Execute during get_online_features()
16+
ON_WRITE = "on_write" # Execute during materialization, cache results
17+
BATCH = "batch" # Scheduled batch processing
18+
STREAMING = "streaming" # Real-time stream processing

0 commit comments

Comments
 (0)