fix: build Spark DataFrame from Arrow with schema and empty handling(#5594)

jfw-ppi · jfw-ppi · commit c0201ad56cee · 2026-01-01T23:08:25.000+01:00
Signed-off-by: Jacob Weinhold &lt;29459386+jfw-ppi@users.noreply.github.com&gt;
diff --git a/sdk/python/feast/infra/compute_engines/spark/nodes.py b/sdk/python/feast/infra/compute_engines/spark/nodes.py
@@ -4,6 +4,7 @@
 import pandas as pd
 from pyspark.sql import DataFrame, SparkSession, Window
 from pyspark.sql import functions as F
+from pyspark.sql.pandas.types import from_arrow_schema
 
 from feast import BatchFeatureView, StreamFeatureView
 from feast.aggregation import Aggregation
@@ -80,7 +81,15 @@ def execute(self, context: ExecutionContext) -> DAGValue:
         if isinstance(retrieval_job, SparkRetrievalJob):
             spark_df = cast(SparkRetrievalJob, retrieval_job).to_spark_df()
         else:
-            spark_df = self.spark_session.createDataFrame(retrieval_job.to_arrow())
+            arrow_table = retrieval_job.to_arrow()
+            if arrow_table.num_rows == 0:
+                spark_schema = from_arrow_schema(arrow_table.schema)
+                spark_df = self.spark_session.createDataFrame(
+                    self.spark_session.sparkContext.emptyRDD(), schema=spark_schema
+                )
+            else:
+                spark_df = self.spark_session.createDataFrame(arrow_table.to_pandas())
+
 
         return DAGValue(
             data=spark_df,
@@ -94,7 +103,6 @@ def execute(self, context: ExecutionContext) -> DAGValue:
             },
         )
 
-
 class SparkAggregationNode(DAGNode):
     def __init__(
         self,
