Skip to content

fix: Upgrade the pyarrow to latest v14.0.1 for CVE-2023-47248. #3841

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
achals merged 1 commit into from
Nov 18, 2023
Merged

fix: Upgrade the pyarrow to latest v14.0.1 for CVE-2023-47248. #3841

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions sdk/python/feast/infra/offline_stores/bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,11 @@ def write_logged_features(
return

with tempfile.TemporaryFile() as parquet_temp_file:
pyarrow.parquet.write_table(table=data, where=parquet_temp_file)
# In Pyarrow v13.0, the parquet version was upgraded to v2.6 from v2.4.
# Set the coerce_timestamps to "us"(microseconds) for backward compatibility.
pyarrow.parquet.write_table(
table=data, where=parquet_temp_file, coerce_timestamps="us"
)

parquet_temp_file.seek(0)

Expand Down Expand Up @@ -400,7 +404,11 @@ def offline_write_batch(
)

with tempfile.TemporaryFile() as parquet_temp_file:
pyarrow.parquet.write_table(table=table, where=parquet_temp_file)
# In Pyarrow v13.0, the parquet version was upgraded to v2.6 from v2.4.
# Set the coerce_timestamps to "us"(microseconds) for backward compatibility.
pyarrow.parquet.write_table(
table=table, where=parquet_temp_file, coerce_timestamps="us"
)

parquet_temp_file.seek(0)

Expand Down
4 changes: 3 additions & 1 deletion sdk/python/feast/infra/utils/aws_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,9 @@ def upload_arrow_table_to_redshift(
else:
# Write the PyArrow Table on disk in Parquet format and upload it to S3
with tempfile.TemporaryFile(suffix=".parquet") as parquet_temp_file:
pq.write_table(table, parquet_temp_file)
# In Pyarrow v13.0, the parquet version was upgraded to v2.6 from v2.4.
# Set the coerce_timestamps to "us"(microseconds) for backward compatibility.
pq.write_table(table, parquet_temp_file, coerce_timestamps="us")
parquet_temp_file.seek(0)
s3_resource.Object(bucket, key).put(Body=parquet_temp_file)

Expand Down
74 changes: 42 additions & 32 deletions sdk/python/requirements/py3.10-ci-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,11 @@ black==22.12.0
# via feast (setup.py)
bleach==6.1.0
# via nbconvert
boto3==1.28.80
boto3==1.29.2
# via
# feast (setup.py)
# moto
botocore==1.31.80
botocore==1.32.2
# via
# boto3
# moto
Expand Down Expand Up @@ -121,7 +121,9 @@ comm==0.2.0
# ipykernel
# ipywidgets
coverage[toml]==7.3.2
# via pytest-cov
# via
# coverage
# pytest-cov
cryptography==41.0.5
# via
# azure-identity
Expand All @@ -135,7 +137,7 @@ cryptography==41.0.5
# snowflake-connector-python
# types-pyopenssl
# types-redis
dask==2023.10.1
dask==2023.11.0
# via feast (setup.py)
db-dtypes==1.1.1
# via google-cloud-bigquery
Expand Down Expand Up @@ -177,7 +179,7 @@ fastavro==1.9.0
# via
# feast (setup.py)
# pandavro
fastjsonschema==2.18.1
fastjsonschema==2.19.0
# via nbformat
filelock==3.13.1
# via
Expand All @@ -199,7 +201,7 @@ geojson==2.5.0
# via rockset
geomet==0.2.1.post1
# via cassandra-driver
google-api-core[grpc]==2.13.0
google-api-core[grpc]==2.14.0
# via
# feast (setup.py)
# firebase-admin
Expand All @@ -211,7 +213,7 @@ google-api-core[grpc]==2.13.0
# google-cloud-datastore
# google-cloud-firestore
# google-cloud-storage
google-api-python-client==2.107.0
google-api-python-client==2.108.0
# via firebase-admin
google-auth==2.23.4
# via
Expand All @@ -224,7 +226,9 @@ google-auth==2.23.4
google-auth-httplib2==0.1.1
# via google-api-python-client
google-cloud-bigquery[pandas]==3.12.0
# via feast (setup.py)
# via
# feast (setup.py)
# google-cloud-bigquery
google-cloud-bigquery-storage==2.22.0
# via feast (setup.py)
google-cloud-bigtable==2.21.0
Expand Down Expand Up @@ -262,7 +266,7 @@ great-expectations==0.15.50
# via feast (setup.py)
greenlet==3.0.1
# via sqlalchemy
grpc-google-iam-v1==0.12.6
grpc-google-iam-v1==0.12.7
# via google-cloud-bigtable
grpcio==1.59.2
# via
Expand Down Expand Up @@ -298,7 +302,7 @@ hazelcast-python-client==5.3.0
# via feast (setup.py)
hiredis==2.2.3
# via feast (setup.py)
httpcore==1.0.1
httpcore==1.0.2
# via httpx
httplib2==0.22.0
# via
Expand Down Expand Up @@ -368,15 +372,15 @@ jsonpointer==2.4
# via
# jsonpatch
# jsonschema
jsonschema[format-nongpl]==4.19.2
jsonschema[format-nongpl]==4.20.0
# via
# altair
# feast (setup.py)
# great-expectations
# jupyter-events
# jupyterlab-server
# nbformat
jsonschema-specifications==2023.7.1
jsonschema-specifications==2023.11.1
# via jsonschema
jupyter-client==8.6.0
# via
Expand All @@ -396,7 +400,7 @@ jupyter-events==0.9.0
# via jupyter-server
jupyter-lsp==2.2.0
# via jupyterlab
jupyter-server==2.10.0
jupyter-server==2.10.1
# via
# jupyter-lsp
# jupyterlab
Expand All @@ -409,7 +413,7 @@ jupyterlab==4.0.8
# via notebook
jupyterlab-pygments==0.2.2
# via nbconvert
jupyterlab-server==2.25.0
jupyterlab-server==2.25.1
# via
# jupyterlab
# notebook
Expand All @@ -419,7 +423,7 @@ kubernetes==20.13.0
# via feast (setup.py)
locket==1.0.0
# via partd
makefun==1.15.1
makefun==1.15.2
# via great-expectations
markupsafe==2.1.3
# via
Expand All @@ -446,7 +450,7 @@ mock==2.0.0
# via feast (setup.py)
moreorless==0.4.0
# via bowler
moto==4.2.7
moto==4.2.9
# via feast (setup.py)
msal==1.25.0
# via
Expand Down Expand Up @@ -564,7 +568,7 @@ pre-commit==3.3.1
# via feast (setup.py)
prometheus-client==0.18.0
# via jupyter-server
prompt-toolkit==3.0.39
prompt-toolkit==3.0.41
# via ipython
proto-plus==1.22.3
# via
Expand Down Expand Up @@ -610,7 +614,7 @@ py-cpuinfo==9.0.0
# via pytest-benchmark
py4j==0.10.9.7
# via pyspark
pyarrow==10.0.1
pyarrow==14.0.1
# via
# db-dtypes
# feast (setup.py)
Expand Down Expand Up @@ -683,7 +687,7 @@ pytest-ordering==0.6
# via feast (setup.py)
pytest-timeout==1.4.2
# via feast (setup.py)
pytest-xdist==3.3.1
pytest-xdist==3.4.0
# via feast (setup.py)
python-dateutil==2.8.2
# via
Expand Down Expand Up @@ -723,7 +727,7 @@ pyzmq==25.1.1
# jupyter-server
redis==4.6.0
# via feast (setup.py)
referencing==0.30.2
referencing==0.31.0
# via
# jsonschema
# jsonschema-specifications
Expand Down Expand Up @@ -751,7 +755,7 @@ requests==2.31.0
# trino
requests-oauthlib==1.3.1
# via kubernetes
responses==0.24.0
responses==0.24.1
# via moto
rfc3339-validator==0.1.4
# via
Expand All @@ -763,7 +767,7 @@ rfc3986-validator==0.1.1
# jupyter-events
rockset==2.1.0
# via feast (setup.py)
rpds-py==0.12.0
rpds-py==0.13.0
# via
# jsonschema
# referencing
Expand Down Expand Up @@ -798,8 +802,10 @@ sniffio==1.3.0
# httpx
snowballstemmer==2.2.0
# via sphinx
snowflake-connector-python[pandas]==3.4.0
# via feast (setup.py)
snowflake-connector-python[pandas]==3.5.0
# via
# feast (setup.py)
# snowflake-connector-python
sortedcontainers==2.4.0
# via snowflake-connector-python
soupsieve==2.5
Expand All @@ -825,8 +831,10 @@ sphinxcontrib-qthelp==1.0.6
sphinxcontrib-serializinghtml==1.1.9
# via sphinx
sqlalchemy[mypy]==1.4.50
# via feast (setup.py)
sqlalchemy2-stubs==0.0.2a36
# via
# feast (setup.py)
# sqlalchemy
sqlalchemy2-stubs==0.0.2a37
# via sqlalchemy
stack-data==0.6.3
# via ipython
Expand All @@ -836,7 +844,7 @@ tabulate==0.9.0
# via feast (setup.py)
tenacity==8.2.3
# via feast (setup.py)
terminado==0.17.1
terminado==0.18.0
# via
# jupyter-server
# jupyter-server-terminals
Expand All @@ -858,7 +866,7 @@ tomli==2.0.1
# pip-tools
# pyproject-hooks
# pytest
tomlkit==0.12.2
tomlkit==0.12.3
# via snowflake-connector-python
toolz==0.12.0
# via
Expand Down Expand Up @@ -916,7 +924,7 @@ types-redis==4.6.0.10
# via feast (setup.py)
types-requests==2.30.0.0
# via feast (setup.py)
types-setuptools==68.2.0.0
types-setuptools==68.2.0.1
# via feast (setup.py)
types-tabulate==0.9.0.3
# via feast (setup.py)
Expand Down Expand Up @@ -955,7 +963,9 @@ urllib3==1.26.18
# rockset
# snowflake-connector-python
uvicorn[standard]==0.24.0.post1
# via feast (setup.py)
# via
# feast (setup.py)
# uvicorn
uvloop==0.19.0
# via uvicorn
virtualenv==20.23.0
Expand All @@ -966,7 +976,7 @@ volatile==2.1.0
# via bowler
watchfiles==0.21.0
# via uvicorn
wcwidth==0.2.9
wcwidth==0.2.10
# via prompt-toolkit
webcolors==1.13
# via jsonschema
Expand All @@ -987,7 +997,7 @@ wheel==0.41.3
# via pip-tools
widgetsnbextension==4.0.9
# via ipywidgets
wrapt==1.15.0
wrapt==1.16.0
# via testcontainers
xmltodict==0.13.0
# via moto
Expand Down
Loading