Skip to content

Commit 85edbbb

Browse files
sudohainguyensudohainguyen
committed
feat: Support pandas v2
Signed-off-by: Hai Nguyen <quanghai.ng1512@gmail.com>
1 parent dd96150 commit 85edbbb

File tree

9 files changed

+102
-85
lines changed

9 files changed

+102
-85
lines changed

‎sdk/python/feast/infra/offline_stores/file.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from pathlib import Path
55
from typing import Any, Callable, List, Literal, Optional, Tuple, Union
66

7+
import dask
78
import dask.dataframe as dd
89
import pandas as pd
910
import pyarrow
@@ -42,6 +43,11 @@
4243
_run_dask_field_mapping,
4344
)
4445

46+
# FileRetrievalJob will cast string objects to string[pyarrow] from dask version 2023.7.1
47+
# This is not the desired behavior for our use case, so we set the convert-string option to False
48+
# See (https://github.com/dask/dask/issues/10881#issuecomment-1923327936)
49+
dask.config.set({"dataframe.convert-string": False})
50+
4551

4652
class FileOfflineStoreConfig(FeastConfigBaseModel):
4753
"""Offline store config for local (file-based) store"""
@@ -366,8 +372,6 @@ def evaluate_offline_job():
366372
source_df[DUMMY_ENTITY_ID] = DUMMY_ENTITY_VAL
367373
columns_to_extract.add(DUMMY_ENTITY_ID)
368374

369-
source_df = source_df.persist()
370-
371375
return source_df[list(columns_to_extract)].persist()
372376

373377
# When materializing a single feature view, we don't need full feature names. On demand transforms aren't materialized

‎sdk/python/requirements/py3.10-ci-requirements.txt‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,11 @@ black==22.12.0
5959
# via feast (setup.py)
6060
bleach==6.1.0
6161
# via nbconvert
62-
boto3==1.34.42
62+
boto3==1.34.44
6363
# via
6464
# feast (setup.py)
6565
# moto
66-
botocore==1.34.42
66+
botocore==1.34.44
6767
# via
6868
# boto3
6969
# moto
@@ -215,7 +215,7 @@ google-api-core[grpc]==2.17.1
215215
# google-cloud-storage
216216
google-api-python-client==2.118.0
217217
# via firebase-admin
218-
google-auth==2.27.0
218+
google-auth==2.28.0
219219
# via
220220
# google-api-core
221221
# google-api-python-client
@@ -226,10 +226,6 @@ google-auth==2.27.0
226226
google-auth-httplib2==0.2.0
227227
# via google-api-python-client
228228
google-cloud-bigquery[pandas]==3.12.0
229-
# via
230-
# feast (setup.py)
231-
# google-cloud-bigquery
232-
google-cloud-bigquery-storage==2.24.0
233229
# via feast (setup.py)
234230
google-cloud-bigquery-storage==2.24.0
235231
# via feast (setup.py)
@@ -264,7 +260,7 @@ googleapis-common-protos[grpc]==1.62.0
264260
# google-api-core
265261
# grpc-google-iam-v1
266262
# grpcio-status
267-
great-expectations==0.18.8
263+
great-expectations==0.18.9
268264
# via feast (setup.py)
269265
greenlet==3.0.3
270266
# via sqlalchemy
@@ -491,7 +487,7 @@ nodeenv==1.8.0
491487
# via pre-commit
492488
notebook==7.1.0
493489
# via great-expectations
494-
notebook-shim==0.2.3
490+
notebook-shim==0.2.4
495491
# via
496492
# jupyterlab
497493
# notebook
@@ -529,7 +525,7 @@ packaging==23.2
529525
# pytest
530526
# snowflake-connector-python
531527
# sphinx
532-
pandas==1.5.3
528+
pandas==2.2.0
533529
# via
534530
# altair
535531
# db-dtypes
@@ -538,7 +534,7 @@ pandas==1.5.3
538534
# great-expectations
539535
# pandavro
540536
# snowflake-connector-python
541-
pandavro==1.5.2
537+
pandavro==1.8.0
542538
# via feast (setup.py)
543539
pandocfilters==1.5.1
544540
# via nbconvert
@@ -552,7 +548,7 @@ pbr==6.0.0
552548
# via mock
553549
pexpect==4.9.0
554550
# via ipython
555-
pip-tools==7.3.0
551+
pip-tools==7.4.0
556552
# via feast (setup.py)
557553
platformdirs==3.11.0
558554
# via
@@ -666,7 +662,9 @@ pyparsing==3.1.1
666662
# great-expectations
667663
# httplib2
668664
pyproject-hooks==1.0.0
669-
# via build
665+
# via
666+
# build
667+
# pip-tools
670668
pyspark==3.5.0
671669
# via feast (setup.py)
672670
pytest==7.4.4
@@ -795,7 +793,6 @@ six==1.16.0
795793
# isodate
796794
# kubernetes
797795
# mock
798-
# pandavro
799796
# python-dateutil
800797
# rfc3339-validator
801798
# thriftpy2
@@ -826,7 +823,9 @@ sphinxcontrib-qthelp==1.0.7
826823
sphinxcontrib-serializinghtml==1.1.10
827824
# via sphinx
828825
sqlalchemy[mypy]==1.4.51
829-
# via feast (setup.py)
826+
# via
827+
# feast (setup.py)
828+
# sqlalchemy
830829
sqlalchemy2-stubs==0.0.2a38
831830
# via sqlalchemy
832831
stack-data==0.6.3
@@ -893,7 +892,7 @@ traitlets==5.14.1
893892
# nbclient
894893
# nbconvert
895894
# nbformat
896-
trino==0.327.0
895+
trino==0.328.0
897896
# via feast (setup.py)
898897
typeguard==4.1.5
899898
# via feast (setup.py)
@@ -913,11 +912,11 @@ types-pytz==2024.1.0.20240203
913912
# via feast (setup.py)
914913
types-pyyaml==6.0.12.12
915914
# via feast (setup.py)
916-
types-redis==4.6.0.20240106
915+
types-redis==4.6.0.20240218
917916
# via feast (setup.py)
918917
types-requests==2.30.0.0
919918
# via feast (setup.py)
920-
types-setuptools==69.0.0.20240125
919+
types-setuptools==69.1.0.20240217
921920
# via feast (setup.py)
922921
types-tabulate==0.9.0.20240106
923922
# via feast (setup.py)
@@ -936,7 +935,10 @@ typing-extensions==4.9.0
936935
# pydantic-core
937936
# snowflake-connector-python
938937
# sqlalchemy2-stubs
938+
# typeguard
939939
# uvicorn
940+
tzdata==2024.1
941+
# via pandas
940942
tzlocal==5.2
941943
# via
942944
# great-expectations

‎sdk/python/requirements/py3.10-requirements.txt‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -121,11 +121,11 @@ packaging==23.2
121121
# via
122122
# dask
123123
# gunicorn
124-
pandas==1.5.3
124+
pandas==2.2.0
125125
# via
126126
# feast (setup.py)
127127
# pandavro
128-
pandavro==1.5.2
128+
pandavro==1.8.0
129129
# via feast (setup.py)
130130
partd==1.4.1
131131
# via dask
@@ -171,9 +171,7 @@ rpds-py==0.18.0
171171
# jsonschema
172172
# referencing
173173
six==1.16.0
174-
# via
175-
# pandavro
176-
# python-dateutil
174+
# via python-dateutil
177175
sniffio==1.3.0
178176
# via
179177
# anyio
@@ -212,13 +210,14 @@ typing-extensions==4.9.0
212210
# pydantic
213211
# pydantic-core
214212
# sqlalchemy2-stubs
213+
# typeguard
215214
# uvicorn
216-
urllib3==2.2.0
215+
tzdata==2024.1
216+
# via pandas
217+
urllib3==2.2.1
217218
# via requests
218219
uvicorn[standard]==0.27.1
219-
# via
220-
# feast (setup.py)
221-
# uvicorn
220+
# via feast (setup.py)
222221
uvloop==0.19.0
223222
# via uvicorn
224223
volatile==2.1.0

‎sdk/python/requirements/py3.8-ci-requirements.txt‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
#
55
# pip-compile --extra=ci --output-file=sdk/python/requirements/py3.8-ci-requirements.txt
66
#
7-
87
alabaster==0.7.13
98
# via sphinx
109
altair==4.2.2
@@ -66,11 +65,11 @@ black==22.12.0
6665
# via feast (setup.py)
6766
bleach==6.1.0
6867
# via nbconvert
69-
boto3==1.34.42
68+
boto3==1.34.44
7069
# via
7170
# feast (setup.py)
7271
# moto
73-
botocore==1.34.42
72+
botocore==1.34.44
7473
# via
7574
# boto3
7675
# moto
@@ -221,7 +220,7 @@ google-api-core[grpc]==2.17.1
221220
# google-cloud-storage
222221
google-api-python-client==2.118.0
223222
# via firebase-admin
224-
google-auth==2.27.0
223+
google-auth==2.28.0
225224
# via
226225
# google-api-core
227226
# google-api-python-client
@@ -232,10 +231,6 @@ google-auth==2.27.0
232231
google-auth-httplib2==0.2.0
233232
# via google-api-python-client
234233
google-cloud-bigquery[pandas]==3.12.0
235-
# via
236-
# feast (setup.py)
237-
# google-cloud-bigquery
238-
google-cloud-bigquery-storage==2.24.0
239234
# via feast (setup.py)
240235
google-cloud-bigquery-storage==2.24.0
241236
# via feast (setup.py)
@@ -270,7 +265,7 @@ googleapis-common-protos[grpc]==1.62.0
270265
# google-api-core
271266
# grpc-google-iam-v1
272267
# grpcio-status
273-
great-expectations==0.18.8
268+
great-expectations==0.18.9
274269
# via feast (setup.py)
275270
greenlet==3.0.3
276271
# via sqlalchemy
@@ -344,6 +339,7 @@ importlib-metadata==6.11.0
344339
# jupyterlab-server
345340
# nbconvert
346341
# sphinx
342+
# typeguard
347343
importlib-resources==6.1.1
348344
# via
349345
# feast (setup.py)
@@ -508,7 +504,7 @@ nodeenv==1.8.0
508504
# via pre-commit
509505
notebook==7.1.0
510506
# via great-expectations
511-
notebook-shim==0.2.3
507+
notebook-shim==0.2.4
512508
# via
513509
# jupyterlab
514510
# notebook
@@ -555,7 +551,7 @@ pandas==1.5.3
555551
# great-expectations
556552
# pandavro
557553
# snowflake-connector-python
558-
pandavro==1.5.2
554+
pandavro==1.8.0
559555
# via feast (setup.py)
560556
pandocfilters==1.5.1
561557
# via nbconvert
@@ -571,7 +567,7 @@ pexpect==4.9.0
571567
# via ipython
572568
pickleshare==0.7.5
573569
# via ipython
574-
pip-tools==7.3.0
570+
pip-tools==7.4.0
575571
# via feast (setup.py)
576572
pkgutil-resolve-name==1.3.10
577573
# via jsonschema
@@ -687,7 +683,9 @@ pyparsing==3.1.1
687683
# great-expectations
688684
# httplib2
689685
pyproject-hooks==1.0.0
690-
# via build
686+
# via
687+
# build
688+
# pip-tools
691689
pyspark==3.5.0
692690
# via feast (setup.py)
693691
pytest==7.4.4
@@ -819,7 +817,6 @@ six==1.16.0
819817
# isodate
820818
# kubernetes
821819
# mock
822-
# pandavro
823820
# python-dateutil
824821
# rfc3339-validator
825822
# thriftpy2
@@ -850,7 +847,9 @@ sphinxcontrib-qthelp==1.0.3
850847
sphinxcontrib-serializinghtml==1.1.5
851848
# via sphinx
852849
sqlalchemy[mypy]==1.4.51
853-
# via feast (setup.py)
850+
# via
851+
# feast (setup.py)
852+
# sqlalchemy
854853
sqlalchemy2-stubs==0.0.2a38
855854
# via sqlalchemy
856855
stack-data==0.6.3
@@ -917,7 +916,7 @@ traitlets==5.14.1
917916
# nbclient
918917
# nbconvert
919918
# nbformat
920-
trino==0.327.0
919+
trino==0.328.0
921920
# via feast (setup.py)
922921
typeguard==4.1.5
923922
# via feast (setup.py)
@@ -937,18 +936,19 @@ types-pytz==2024.1.0.20240203
937936
# via feast (setup.py)
938937
types-pyyaml==6.0.12.12
939938
# via feast (setup.py)
940-
types-redis==4.6.0.20240106
939+
types-redis==4.6.0.20240218
941940
# via feast (setup.py)
942941
types-requests==2.30.0.0
943942
# via feast (setup.py)
944-
types-setuptools==69.0.0.20240125
943+
types-setuptools==69.1.0.20240217
945944
# via feast (setup.py)
946945
types-tabulate==0.9.0.20240106
947946
# via feast (setup.py)
948947
types-urllib3==1.26.25.14
949948
# via types-requests
950949
typing-extensions==4.9.0
951950
# via
951+
# annotated-types
952952
# anyio
953953
# async-lru
954954
# azure-core
@@ -963,7 +963,10 @@ typing-extensions==4.9.0
963963
# snowflake-connector-python
964964
# sqlalchemy2-stubs
965965
# starlette
966+
# typeguard
966967
# uvicorn
968+
tzdata==2024.1
969+
# via pandas
967970
tzlocal==5.2
968971
# via
969972
# great-expectations

0 commit comments

Comments
 (0)