feast-dev · aniketpalu · Jul 29, 2026 · Jul 29, 2026 · Jul 30, 2026 · Jul 30, 2026
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -1392,7 +1392,7 @@
         "filename": "sdk/python/tests/unit/infra/offline_stores/contrib/postgres_offline_store/test_postgres.py",
         "hashed_secret": "9fb7fe1217aed442b04c0f5e43b5d5a7d3287097",
         "is_verified": false,
-        "line_number": 301
+        "line_number": 364
       }
     ],
     "sdk/python/tests/unit/infra/offline_stores/test_clickhouse.py": [

@@ -109,7 +109,10 @@ def build_dedup_node(self, view, input_node):
     def build_transformation_node(self, view, input_nodes):
         udf_name = view.feature_transformation.name
         udf = view.feature_transformation.udf
-        node = SparkTransformationNode(udf_name, udf, inputs=input_nodes)
+        udf_string = getattr(view.feature_transformation, "udf_string", "") or ""
+        node = SparkTransformationNode(
+            udf_name, udf, inputs=input_nodes, udf_string=udf_string
+        )
         self.nodes.append(node)
         return node
 

@@ -45,6 +45,7 @@
 from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import (
     SparkSource,
 )
+from feast.transformation.udf_rehydrate import resolve_udf
 
 logger = logging.getLogger(__name__)
 
@@ -602,9 +603,29 @@ def execute(self, context: ExecutionContext) -> DAGValue:
 
 
 class SparkTransformationNode(DAGNode):
-    def __init__(self, name: str, udf: Callable, inputs: List[DAGNode]):
+    def __init__(
+        self,
+        name: str,
+        udf: Callable,
+        inputs: List[DAGNode],
+        udf_string: str = "",
+    ):
         super().__init__(name, inputs)
         self.udf = udf
+        self.udf_string = udf_string or ""
+
+    def _resolve_udf(self) -> Callable:
+        """Prefer source reconstruction over dill callables.
+
+        Dill-deserialized functions that call DataFrame.withColumn / __getitem__
+        can segfault (exit 139) on Spark 4.0.1. Re-executing ``udf_string``
+        yields a healthy callable.
+        """
+        return resolve_udf(
+            udf_string=self.udf_string,
+            fallback_udf=self.udf,
+            preferred_name=self.name,
+        )
 
     def execute(self, context: ExecutionContext) -> DAGValue:
         input_values = self.get_input_values(context)
@@ -613,7 +634,7 @@ def execute(self, context: ExecutionContext) -> DAGValue:
 
         input_dfs: List[DataFrame] = [val.data for val in input_values]
 
-        transformed_df = self.udf(*input_dfs)
+        transformed_df = self._resolve_udf()(*input_dfs)
 
         return DAGValue(
             data=transformed_df, format=DAGFormat.SPARK, metadata={"transformed": True}

@@ -101,14 +101,35 @@ def pull_latest_from_table_or_query(
         if created_timestamp_column:
             timestamps.append(created_timestamp_column)
         timestamp_desc_string = " DESC, ".join(_append_alias(timestamps, "a")) + " DESC"
-        a_field_string = ", ".join(
-            _append_alias(join_key_columns + feature_name_columns + timestamps, "a")
-        )
-        b_field_string = ", ".join(
-            _append_alias(join_key_columns + feature_name_columns + timestamps, "b")
-        )
-
-        query = f"""
+        # Empty feature_name_columns means "all source columns". BatchFeatureView
+        # python/pandas/ray transforms signal this via get_column_info. Selecting
+        # only join keys + timestamps would starve the UDF of input features.
+        if not feature_name_columns:
+            distinct_on = ", ".join(f'a."{c}"' for c in join_key_columns) or (
+                f'a."{timestamp_field}"'
+            )
+            order_by_parts = [f'a."{c}"' for c in join_key_columns] + [
+                f'a."{timestamp_field}" DESC'
+            ]
+            if created_timestamp_column:
+                order_by_parts.append(f'a."{created_timestamp_column}" DESC')
+            order_by = ", ".join(order_by_parts)
+            query = f"""
+            SELECT DISTINCT ON ({distinct_on})
+                a.*
+                {f", {repr(DUMMY_ENTITY_VAL)} AS {DUMMY_ENTITY_ID}" if not join_key_columns else ""}
+            FROM {from_expression} a
+            WHERE a."{timestamp_field}" BETWEEN '{start_date}'::timestamptz AND '{end_date}'::timestamptz
+            ORDER BY {order_by}
+            """
+        else:
+            a_field_string = ", ".join(
+                _append_alias(join_key_columns + feature_name_columns + timestamps, "a")
+            )
+            b_field_string = ", ".join(
+                _append_alias(join_key_columns + feature_name_columns + timestamps, "b")
+            )
+            query = f"""
             SELECT
                 {b_field_string}
                 {f", {repr(DUMMY_ENTITY_VAL)} AS {DUMMY_ENTITY_ID}" if not join_key_columns else ""}
@@ -275,12 +296,17 @@ def pull_all_from_table_or_query(
         timestamp_fields = [timestamp_field]
         if created_timestamp_column:
             timestamp_fields.append(created_timestamp_column)
-        field_string = ", ".join(
-            _append_alias(
-                join_key_columns + feature_name_columns + timestamp_fields,
-                "paftoq_alias",
+        # Empty feature_name_columns => SELECT * (BatchFeatureView python mode).
+        # Default materialization uses pull_all (pull_latest_features=False).
+        if not feature_name_columns:
+            field_string = "paftoq_alias.*"
+        else:
+            field_string = ", ".join(
+                _append_alias(
+                    join_key_columns + feature_name_columns + timestamp_fields,
+                    "paftoq_alias",
+                )
             )
-        )
 
         timestamp_filter = get_timestamp_filter_sql(
             start_date,

@@ -1,7 +1,6 @@
 import inspect
 from typing import Any, Callable, Optional, cast, get_type_hints
 
-import dill
 import pandas as pd
 import pyarrow
 
@@ -134,17 +133,28 @@ def __eq__(self, other):
         if not isinstance(other, PandasTransformation):
             return False
 
-        if (
-            self.udf_string != other.udf_string
-            or self.udf.__code__.co_code != other.udf.__code__.co_code
-        ):
-            return False
+        # udf_string is the canonical diff identity. Source-first from_proto
+        # rebuilds a new callable (strip+exec) whose bytecode differs from the
+        # live repo function even when the source is unchanged — do not require
+        # co_code equality when both sides have source text.
+        left = self.udf_string or ""
+        right = other.udf_string or ""
+        if left and right:
+            return left == right
 
-        return True
+        return self.udf.__code__.co_code == other.udf.__code__.co_code
 
     @classmethod
     def from_proto(cls, user_defined_function_proto: UserDefinedFunctionProto):
+        from feast.transformation.udf_rehydrate import resolve_udf
+
+        udf_string = user_defined_function_proto.body_text or ""
+        udf = resolve_udf(
+            udf_string=udf_string,
+            body=user_defined_function_proto.body or None,
+            preferred_name=user_defined_function_proto.name or None,
+        )
         return PandasTransformation(
-            udf=dill.loads(user_defined_function_proto.body),
-            udf_string=user_defined_function_proto.body_text,
+            udf=udf,
+            udf_string=udf_string,
         )
@@ -1,7 +1,6 @@
 from types import FunctionType
 from typing import Any, Dict, Optional, cast
 
-import dill
 import pyarrow
 
 from feast.field import Field, from_value_type
@@ -145,13 +144,16 @@ def __eq__(self, other):
         if not isinstance(other, PythonTransformation):
             return False
 
-        if (
-            self.udf_string != other.udf_string
-            or self.udf.__code__.co_code != other.udf.__code__.co_code
-        ):
-            return False
+        # udf_string is the canonical diff identity. Source-first from_proto
+        # rebuilds a new callable (strip+exec) whose bytecode differs from the
+        # live repo function even when the source is unchanged — do not require
+        # co_code equality when both sides have source text.
+        left = self.udf_string or ""
+        right = other.udf_string or ""
+        if left and right:
+            return left == right
 
-        return True
+        return self.udf.__code__.co_code == other.udf.__code__.co_code
 
     def __reduce__(self):
         """Support for pickle/dill serialization."""
@@ -162,7 +164,15 @@ def __reduce__(self):
 
     @classmethod
     def from_proto(cls, user_defined_function_proto: UserDefinedFunctionProto):
+        from feast.transformation.udf_rehydrate import resolve_udf
+
+        udf_string = user_defined_function_proto.body_text or ""
+        udf = resolve_udf(
+            udf_string=udf_string,
+            body=user_defined_function_proto.body or None,
+            preferred_name=user_defined_function_proto.name or None,
+        )
         return PythonTransformation(
-            udf=dill.loads(user_defined_function_proto.body),
-            udf_string=user_defined_function_proto.body_text,
+            udf=cast(FunctionType, udf),
+            udf_string=udf_string,
         )