Skip to content

Commit 9b98eaf

Browse files
franciscojavierarceofranciscojavierarceo
authored
feat: Rename OnDemandTransformations to Transformations (#4038)
* feat: updating protos to separate transformation Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * fixed stuff...i think Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated tests and registry diff function Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated base registry Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated react component Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * formatted Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated stream feature view proto Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * making the proto changes backwards compatable Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * trying to make this backwards compatible Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * caught a bug and fixed the linter Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * actually linted Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated ui component Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * accidentally commented out fixtures Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * Updated Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * incrementing protos Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated tests Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * fixed linting issue and made backwards compatible Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * feat: Renaming OnDemandTransformations to Transformations Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated proto name Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * renamed substrait proto Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * renamed substrait proto Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated * updated Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated integration test * missed one Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> * updated to include Substrait type * linter Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com> --------- Signed-off-by: Francisco Javier Arceo <franciscojavierarceo@users.noreply.github.com>
1 parent c58ef74 commit 9b98eaf

File tree

11 files changed

+102
-97
lines changed

11 files changed

+102
-97
lines changed

‎protos/feast/core/OnDemandFeatureView.proto‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,8 @@ message OnDemandFeatureViewSpec {
4949
// Map of sources for this feature view.
5050
map<string, OnDemandSource> sources = 4;
5151

52-
oneof transformation {
53-
UserDefinedFunction user_defined_function = 5 [deprecated = true];
54-
OnDemandSubstraitTransformation on_demand_substrait_transformation = 9 [deprecated = true];
55-
}
52+
UserDefinedFunction user_defined_function = 5 [deprecated = true];
53+
5654
// Oneof with {user_defined_function, on_demand_substrait_transformation}
5755
FeatureTransformationV2 feature_transformation = 10;
5856

@@ -96,9 +94,3 @@ message UserDefinedFunction {
9694
// The string representation of the udf
9795
string body_text = 3;
9896
}
99-
100-
message OnDemandSubstraitTransformation {
101-
option deprecated = true;
102-
103-
bytes substrait_plan = 1;
104-
}

‎protos/feast/core/Transformation.proto‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,12 @@ message UserDefinedFunctionV2 {
2121

2222
// A feature transformation executed as a user-defined function
2323
message FeatureTransformationV2 {
24-
// Note this Transformation starts at 5 for backwards compatibility
2524
oneof transformation {
2625
UserDefinedFunctionV2 user_defined_function = 1;
27-
OnDemandSubstraitTransformationV2 on_demand_substrait_transformation = 2;
26+
SubstraitTransformationV2 substrait_transformation = 2;
2827
}
2928
}
3029

31-
message OnDemandSubstraitTransformationV2 {
30+
message SubstraitTransformationV2 {
3231
bytes substrait_plan = 1;
3332
}

‎sdk/python/feast/diff/registry_diff.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import warnings
21
from dataclasses import dataclass
32
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, cast
43

@@ -145,22 +144,23 @@ def diff_registry_objects(
145144
if _field.name in FIELDS_TO_IGNORE:
146145
continue
147146
elif getattr(current_spec, _field.name) != getattr(new_spec, _field.name):
148-
# TODO: Delete "transformation" after we've safely deprecated it from the proto
149-
if _field.name in ["transformation", "feature_transformation"]:
150-
warnings.warn(
151-
"transformation will be deprecated in the future please use feature_transformation instead.",
152-
DeprecationWarning,
153-
)
147+
if _field.name == "feature_transformation":
154148
current_spec = cast(OnDemandFeatureViewSpec, current_spec)
155149
new_spec = cast(OnDemandFeatureViewSpec, new_spec)
156150
# Check if the old proto is populated and use that if it is
157-
deprecated_udf = current_spec.user_defined_function
158151
feature_transformation_udf = (
159152
current_spec.feature_transformation.user_defined_function
160153
)
154+
if (
155+
current_spec.HasField("user_defined_function")
156+
and not feature_transformation_udf
157+
):
158+
deprecated_udf = current_spec.user_defined_function
159+
else:
160+
deprecated_udf = None
161161
current_udf = (
162162
deprecated_udf
163-
if deprecated_udf.body_text != ""
163+
if deprecated_udf is not None
164164
else feature_transformation_udf
165165
)
166166
new_udf = new_spec.feature_transformation.user_defined_function

‎sdk/python/feast/infra/registry/base_registry.py‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
from feast.request_feature_view import RequestFeatureView
3434
from feast.saved_dataset import SavedDataset, ValidationReference
3535
from feast.stream_feature_view import StreamFeatureView
36+
from feast.transformation.pandas_transformation import PandasTransformation
37+
from feast.transformation.substrait_transformation import SubstraitTransformation
3638

3739

3840
class BaseRegistry(ABC):
@@ -670,10 +672,33 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]:
670672
"We will be deprecating the usage of spec.userDefinedFunction in a future release please upgrade cautiously.",
671673
DeprecationWarning,
672674
)
673-
odfv_dict["spec"]["featureTransformation"]["userDefinedFunction"][
674-
"body"
675-
] = on_demand_feature_view.feature_transformation.udf_string
676-
registry_dict["onDemandFeatureViews"].append(odfv_dict)
675+
if on_demand_feature_view.feature_transformation:
676+
if isinstance(
677+
on_demand_feature_view.feature_transformation, PandasTransformation
678+
):
679+
if "userDefinedFunction" not in odfv_dict["spec"]:
680+
odfv_dict["spec"]["userDefinedFunction"] = {}
681+
odfv_dict["spec"]["userDefinedFunction"][
682+
"body"
683+
] = on_demand_feature_view.feature_transformation.udf_string
684+
odfv_dict["spec"]["featureTransformation"]["userDefinedFunction"][
685+
"body"
686+
] = on_demand_feature_view.feature_transformation.udf_string
687+
elif isinstance(
688+
on_demand_feature_view.feature_transformation,
689+
SubstraitTransformation,
690+
):
691+
odfv_dict["spec"]["featureTransformation"]["substraitPlan"][
692+
"body"
693+
] = on_demand_feature_view.feature_transformation.substrait_plan
694+
else:
695+
odfv_dict["spec"]["featureTransformation"]["userDefinedFunction"][
696+
"body"
697+
] = None
698+
odfv_dict["spec"]["featureTransformation"]["substraitPlan"][
699+
"body"
700+
] = None
701+
registry_dict["onDemandFeatureViews"].append(odfv_dict)
677702
for request_feature_view in sorted(
678703
self.list_request_feature_views(project=project),
679704
key=lambda request_feature_view: request_feature_view.name,

‎sdk/python/feast/on_demand_feature_view.py‎

Lines changed: 30 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717
from feast.feature_view import FeatureView
1818
from feast.feature_view_projection import FeatureViewProjection
1919
from feast.field import Field, from_value_type
20-
from feast.on_demand_pandas_transformation import OnDemandPandasTransformation
21-
from feast.on_demand_substrait_transformation import OnDemandSubstraitTransformation
2220
from feast.protos.feast.core.OnDemandFeatureView_pb2 import (
2321
OnDemandFeatureView as OnDemandFeatureViewProto,
2422
)
@@ -33,6 +31,8 @@
3331
from feast.protos.feast.core.Transformation_pb2 import (
3432
UserDefinedFunctionV2 as UserDefinedFunctionProto,
3533
)
34+
from feast.transformation.pandas_transformation import PandasTransformation
35+
from feast.transformation.substrait_transformation import SubstraitTransformation
3636
from feast.type_map import (
3737
feast_value_type_to_pandas_type,
3838
python_type_to_feast_value_type,
@@ -57,7 +57,7 @@ class OnDemandFeatureView(BaseFeatureView):
5757
sources with type FeatureViewProjection.
5858
source_request_sources: A map from input source names to the actual input
5959
sources with type RequestSource.
60-
transformation: The user defined transformation.
60+
feature_transformation: The user defined transformation.
6161
description: A human-readable description.
6262
tags: A dictionary of key-value pairs to store arbitrary metadata.
6363
owner: The owner of the on demand feature view, typically the email of the primary
@@ -68,8 +68,7 @@ class OnDemandFeatureView(BaseFeatureView):
6868
features: List[Field]
6969
source_feature_view_projections: Dict[str, FeatureViewProjection]
7070
source_request_sources: Dict[str, RequestSource]
71-
transformation: Union[OnDemandPandasTransformation]
72-
feature_transformation: Union[OnDemandPandasTransformation]
71+
feature_transformation: Union[PandasTransformation, SubstraitTransformation]
7372
description: str
7473
tags: Dict[str, str]
7574
owner: str
@@ -89,8 +88,9 @@ def __init__( # noqa: C901
8988
],
9089
udf: Optional[FunctionType] = None,
9190
udf_string: str = "",
92-
transformation: Optional[Union[OnDemandPandasTransformation]] = None,
93-
feature_transformation: Optional[Union[OnDemandPandasTransformation]] = None,
91+
feature_transformation: Optional[
92+
Union[PandasTransformation, SubstraitTransformation]
93+
] = None,
9494
description: str = "",
9595
tags: Optional[Dict[str, str]] = None,
9696
owner: str = "",
@@ -108,7 +108,6 @@ def __init__( # noqa: C901
108108
udf (deprecated): The user defined transformation function, which must take pandas
109109
dataframes as inputs.
110110
udf_string (deprecated): The source code version of the udf (for diffing and displaying in Web UI)
111-
transformation: The user defined transformation.
112111
feature_transformation: The user defined transformation.
113112
description (optional): A human-readable description.
114113
tags (optional): A dictionary of key-value pairs to store arbitrary metadata.
@@ -123,13 +122,13 @@ def __init__( # noqa: C901
123122
owner=owner,
124123
)
125124

126-
if not transformation:
125+
if not feature_transformation:
127126
if udf:
128127
warnings.warn(
129128
"udf and udf_string parameters are deprecated. Please use transformation=OnDemandPandasTransformation(udf, udf_string) instead.",
130129
DeprecationWarning,
131130
)
132-
transformation = OnDemandPandasTransformation(udf, udf_string)
131+
feature_transformation = PandasTransformation(udf, udf_string)
133132
else:
134133
raise Exception(
135134
"OnDemandFeatureView needs to be initialized with either transformation or udf arguments"
@@ -147,8 +146,7 @@ def __init__( # noqa: C901
147146
odfv_source.name
148147
] = odfv_source.projection
149148

150-
self.transformation = transformation
151-
self.feature_transformation = self.transformation
149+
self.feature_transformation = feature_transformation
152150

153151
@property
154152
def proto_class(self) -> Type[OnDemandFeatureViewProto]:
@@ -160,8 +158,7 @@ def __copy__(self):
160158
schema=self.features,
161159
sources=list(self.source_feature_view_projections.values())
162160
+ list(self.source_request_sources.values()),
163-
transformation=self.transformation,
164-
feature_transformation=self.transformation,
161+
feature_transformation=self.feature_transformation,
165162
description=self.description,
166163
tags=self.tags,
167164
owner=self.owner,
@@ -182,7 +179,6 @@ def __eq__(self, other):
182179
self.source_feature_view_projections
183180
!= other.source_feature_view_projections
184181
or self.source_request_sources != other.source_request_sources
185-
or self.transformation != other.transformation
186182
or self.feature_transformation != other.feature_transformation
187183
):
188184
return False
@@ -218,12 +214,12 @@ def to_proto(self) -> OnDemandFeatureViewProto:
218214
)
219215

220216
feature_transformation = FeatureTransformationProto(
221-
user_defined_function=self.transformation.to_proto()
222-
if type(self.transformation) == OnDemandPandasTransformation
217+
user_defined_function=self.feature_transformation.to_proto()
218+
if isinstance(self.feature_transformation, PandasTransformation)
219+
else None,
220+
substrait_transformation=self.feature_transformation.to_proto()
221+
if isinstance(self.feature_transformation, SubstraitTransformation)
223222
else None,
224-
on_demand_substrait_transformation=self.transformation.to_proto()
225-
if type(self.transformation) == OnDemandSubstraitTransformation
226-
else None, # type: ignore
227223
)
228224
spec = OnDemandFeatureViewSpec(
229225
name=self.name,
@@ -276,17 +272,17 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto):
276272
and on_demand_feature_view_proto.spec.feature_transformation.user_defined_function.body_text
277273
!= ""
278274
):
279-
transformation = OnDemandPandasTransformation.from_proto(
275+
transformation = PandasTransformation.from_proto(
280276
on_demand_feature_view_proto.spec.feature_transformation.user_defined_function
281277
)
282278
elif (
283279
on_demand_feature_view_proto.spec.feature_transformation.WhichOneof(
284280
"transformation"
285281
)
286-
== "on_demand_substrait_transformation"
282+
== "substrait_transformation"
287283
):
288-
transformation = OnDemandSubstraitTransformation.from_proto(
289-
on_demand_feature_view_proto.spec.feature_transformation.on_demand_substrait_transformation
284+
transformation = SubstraitTransformation.from_proto(
285+
on_demand_feature_view_proto.spec.feature_transformation.substrait_transformation
290286
)
291287
elif (
292288
hasattr(on_demand_feature_view_proto.spec, "user_defined_function")
@@ -298,7 +294,7 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto):
298294
body=on_demand_feature_view_proto.spec.user_defined_function.body,
299295
body_text=on_demand_feature_view_proto.spec.user_defined_function.body_text,
300296
)
301-
transformation = OnDemandPandasTransformation.from_proto(
297+
transformation = PandasTransformation.from_proto(
302298
user_defined_function_proto=backwards_compatible_udf,
303299
)
304300
else:
@@ -314,7 +310,7 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto):
314310
for feature in on_demand_feature_view_proto.spec.features
315311
],
316312
sources=sources,
317-
transformation=transformation,
313+
feature_transformation=transformation,
318314
description=on_demand_feature_view_proto.spec.description,
319315
tags=dict(on_demand_feature_view_proto.spec.tags),
320316
owner=on_demand_feature_view_proto.spec.owner,
@@ -374,7 +370,9 @@ def get_transformed_features_df(
374370

375371
# Compute transformed values and apply to each result row
376372

377-
df_with_transformed_features = self.transformation.transform(df_with_features)
373+
df_with_transformed_features = self.feature_transformation.transform(
374+
df_with_features
375+
)
378376

379377
# Work out whether the correct columns names are used.
380378
rename_columns: Dict[str, str] = {}
@@ -424,7 +422,7 @@ def infer_features(self) -> None:
424422
dtype = feast_value_type_to_pandas_type(field.dtype.to_value_type())
425423
sample_val = rand_df_value[dtype] if dtype in rand_df_value else None
426424
df[f"{field.name}"] = pd.Series(sample_val, dtype=dtype)
427-
output_df: pd.DataFrame = self.transformation.transform(df)
425+
output_df: pd.DataFrame = self.feature_transformation.transform(df)
428426
inferred_features = []
429427
for f, dt in zip(output_df.columns, output_df.dtypes):
430428
inferred_features.append(
@@ -521,7 +519,7 @@ def decorator(user_function):
521519
input_fields: Field = []
522520

523521
for s in sources:
524-
if type(s) == FeatureView:
522+
if isinstance(s, FeatureView):
525523
fields = s.projection.features
526524
else:
527525
fields = s.features
@@ -540,19 +538,19 @@ def decorator(user_function):
540538

541539
expr = user_function(ibis.table(input_fields, "t"))
542540

543-
transformation = OnDemandSubstraitTransformation(
541+
transformation = SubstraitTransformation(
544542
substrait_plan=compiler.compile(expr).SerializeToString()
545543
)
546544
else:
547545
udf_string = dill.source.getsource(user_function)
548546
mainify(user_function)
549-
transformation = OnDemandPandasTransformation(user_function, udf_string)
547+
transformation = PandasTransformation(user_function, udf_string)
550548

551549
on_demand_feature_view_obj = OnDemandFeatureView(
552550
name=user_function.__name__,
553551
sources=sources,
554552
schema=schema,
555-
transformation=transformation,
553+
feature_transformation=transformation,
556554
description=description,
557555
tags=tags,
558556
owner=owner,

‎sdk/python/feast/stream_feature_view.py‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
from feast.entity import Entity
1616
from feast.feature_view import FeatureView
1717
from feast.field import Field
18-
from feast.on_demand_pandas_transformation import OnDemandPandasTransformation
1918
from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
2019
from feast.protos.feast.core.OnDemandFeatureView_pb2 import (
2120
UserDefinedFunction as UserDefinedFunctionProto,
@@ -32,6 +31,7 @@
3231
from feast.protos.feast.core.Transformation_pb2 import (
3332
UserDefinedFunctionV2 as UserDefinedFunctionProtoV2,
3433
)
34+
from feast.transformation.pandas_transformation import PandasTransformation
3535

3636
warnings.simplefilter("once", RuntimeWarning)
3737

@@ -80,7 +80,7 @@ class StreamFeatureView(FeatureView):
8080
materialization_intervals: List[Tuple[datetime, datetime]]
8181
udf: Optional[FunctionType]
8282
udf_string: Optional[str]
83-
feature_transformation: Optional[OnDemandPandasTransformation]
83+
feature_transformation: Optional[PandasTransformation]
8484

8585
def __init__(
8686
self,
@@ -99,7 +99,7 @@ def __init__(
9999
timestamp_field: Optional[str] = "",
100100
udf: Optional[FunctionType] = None,
101101
udf_string: Optional[str] = "",
102-
feature_transformation: Optional[Union[OnDemandPandasTransformation]] = None,
102+
feature_transformation: Optional[Union[PandasTransformation]] = None,
103103
):
104104
if not flags_helper.is_test():
105105
warnings.warn(
@@ -371,9 +371,7 @@ def decorator(user_function):
371371
schema=schema,
372372
udf=user_function,
373373
udf_string=udf_string,
374-
feature_transformation=OnDemandPandasTransformation(
375-
user_function, udf_string
376-
),
374+
feature_transformation=PandasTransformation(user_function, udf_string),
377375
description=description,
378376
tags=tags,
379377
online=online,

‎sdk/python/feast/transformation/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)