Skip to content

Commit 8b9bb50

Browse files
jyejarejyejare
authored
feat: Resolve pyarrow >21 installation with ibis-framework (#5847)
* feat: Update Pyarrow dependency along with ibis-framework Signed-off-by: jyejare <jyejare@redhat.com> * Remove limit of ibis-framework for all other feast main dependencies Signed-off-by: jyejare <jyejare@redhat.com> * Ibis substrait CI checks removed with import warning, pymilvus updated, setuptools warnings fixed Signed-off-by: jyejare <jyejare@redhat.com> --------- Signed-off-by: jyejare <jyejare@redhat.com>
1 parent c2ea7e9 commit 8b9bb50

26 files changed

+6813
-7071
lines changed

‎pyproject.toml‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ name = "feast"
33
description = "Python SDK for Feast"
44
readme = "README.md"
55
requires-python = ">=3.10.0"
6-
license = {file = "LICENSE"}
6+
license = "Apache-2.0"
7+
license-files = ["LICENSE"]
78
classifiers = [
8-
"License :: OSI Approved :: Apache Software License",
99
"Programming Language :: Python",
1010
"Programming Language :: Python :: 3",
1111
"Programming Language :: Python :: 3.10"
@@ -23,7 +23,7 @@ dependencies = [
2323
"mmh3",
2424
"numpy>=2.0.0,<3",
2525
"pandas>=1.4.3,<3",
26-
"pyarrow<=21.0.0",
26+
"pyarrow>=21.0.0",
2727
"pydantic>=2.10.6",
2828
"pygments>=2.12.0,<3",
2929
"PyYAML>=5.4.0,<7",
@@ -59,7 +59,7 @@ clickhouse = ["clickhouse-connect>=0.7.19"]
5959
couchbase = ["couchbase==4.3.2", "couchbase-columnar==1.0.0"]
6060
delta = ["deltalake<1.0.0"]
6161
docling = ["docling==2.27.0"]
62-
duckdb = ["ibis-framework[duckdb]>=9.0.0,<=9.5.0"]
62+
duckdb = ["ibis-framework[duckdb]>=10.0.0"]
6363
elasticsearch = ["elasticsearch>=8.13.0"]
6464
faiss = ["faiss-cpu>=1.7.0,<=1.10.0"]
6565
gcp = [
@@ -82,9 +82,7 @@ grpcio = [
8282
hazelcast = ["hazelcast-python-client>=5.1"]
8383
hbase = ["happybase>=1.2.0,<3"]
8484
ibis = [
85-
"ibis-framework>=9.0.0,<=9.5.0",
86-
"ibis-substrait>=4.0.0",
87-
"substrait<0.25.0", # TODO: remove this once we upgrade protobuf
85+
"ibis-framework>=10.0.0",
8886
"poetry-core<2",
8987
"poetry-dynamic-versioning",
9088
]
@@ -99,11 +97,11 @@ image = [
9997
"scikit-learn>=1.0.0",
10098
]
10199
milvus = [
102-
"pymilvus==2.4.15",
100+
"pymilvus>2.5",
103101
"milvus-lite==2.4.12",
104102
"feast[setuptools]"
105103
]
106-
mssql = ["ibis-framework[mssql]>=9.0.0,<=9.5.0"]
104+
mssql = ["ibis-framework[mssql]>=10.0.0"]
107105
mysql = ["pymysql", "types-PyMySQL"]
108106
opentelemetry = ["prometheus_client", "psutil"]
109107
spark = ["pyspark>=4.0.0"]

‎sdk/python/feast/infra/offline_stores/duckdb.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,12 @@ def _read_data_source(data_source: DataSource, repo_path: str) -> Table:
3333
if isinstance(data_source.file_format, ParquetFormat):
3434
return ibis.read_parquet(data_source.path)
3535
elif isinstance(data_source.file_format, DeltaFormat):
36-
storage_options = {
37-
"AWS_ENDPOINT_URL": data_source.s3_endpoint_override,
38-
}
39-
40-
return ibis.read_delta(data_source.path, storage_options=storage_options)
36+
if data_source.s3_endpoint_override:
37+
storage_options = {
38+
"AWS_ENDPOINT_URL": data_source.s3_endpoint_override,
39+
}
40+
return ibis.read_delta(data_source.path, storage_options=storage_options)
41+
return ibis.read_delta(data_source.path)
4142

4243

4344
def _write_data_source(

‎sdk/python/feast/infra/offline_stores/ibis.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
77

88
import ibis
9-
import ibis.selectors as s
109
import numpy as np
1110
import pandas as pd
1211
import pyarrow
@@ -393,6 +392,10 @@ def point_in_time_join(
393392

394393
acc_table = entity_table
395394

395+
# Track the columns we want to keep explicitly
396+
entity_columns = list(entity_table.columns)
397+
all_feature_cols: List[str] = []
398+
396399
for (
397400
feature_table,
398401
timestamp_field,
@@ -432,7 +435,8 @@ def point_in_time_join(
432435
entity_table, predicates, lname="", rname="{name}_y"
433436
)
434437

435-
feature_table = feature_table.drop(s.endswith("_y"))
438+
cols_to_drop_y = [c for c in feature_table.columns if c.endswith("_y")]
439+
feature_table = feature_table.drop(*cols_to_drop_y)
436440

437441
feature_table = deduplicate(
438442
table=feature_table,
@@ -445,16 +449,19 @@ def point_in_time_join(
445449
select_cols.extend(feature_refs)
446450
feature_table = feature_table.select(select_cols)
447451

452+
# Track the feature columns we're adding
453+
all_feature_cols.extend(feature_refs)
454+
448455
acc_table = acc_table.left_join(
449456
feature_table,
450457
predicates=[feature_table.entity_row_id == acc_table.entity_row_id],
451458
lname="",
452459
rname="{name}_yyyy",
453460
)
454461

455-
acc_table = acc_table.drop(s.endswith("_yyyy"))
456-
457-
acc_table = acc_table.drop("entity_row_id")
462+
# Select only the columns we want: entity columns (minus entity_row_id) + all feature columns
463+
final_cols = [c for c in entity_columns if c != "entity_row_id"] + all_feature_cols
464+
acc_table = acc_table.select(final_cols)
458465

459466
return acc_table
460467

‎sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -222,10 +222,11 @@ def _get_or_create_collection(
222222
index_name=f"vector_index_{vector_field.name}",
223223
params={"nlist": config.online_store.nlist},
224224
)
225-
self.client.create_index(
226-
collection_name=collection_name,
227-
index_params=index_params,
228-
)
225+
if len(index_params) > 0:
226+
self.client.create_index(
227+
collection_name=collection_name,
228+
index_params=index_params,
229+
)
229230
else:
230231
self.client.load_collection(collection_name)
231232
self._collections[collection_name] = self.client.describe_collection(

‎sdk/python/feast/transformation/substrait_transformation.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,14 @@ def from_ibis(cls, user_function, sources):
175175

176176
import ibis
177177
import ibis.expr.datatypes as dt
178-
from ibis_substrait.compiler.core import SubstraitCompiler
178+
179+
try:
180+
from ibis_substrait.compiler.core import SubstraitCompiler
181+
except ImportError:
182+
raise ImportError(
183+
"Failed to use substrait transformation: 'ibis-substrait' package is not installed. "
184+
"Install it with: `pip install ibis-substrait and only if https://github.com/ibis-project/ibis-substrait/issues/1309 issue is resolved."
185+
)
179186

180187
compiler = SubstraitCompiler()
181188

‎sdk/python/pytest.ini‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,3 @@ filterwarnings =
1717
ignore::DeprecationWarning:httpx.*:
1818
ignore::DeprecationWarning:happybase.*:
1919
ignore::DeprecationWarning:pkg_resources.*:
20-
ignore::FutureWarning:ibis_substrait.compiler.*:

0 commit comments

Comments
 (0)