Skip to content

fix: Update the pyarrow to latest v14.0.1 regarding the CVE-2023-47248. #3835

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
shuchu wants to merge 14 commits into from
Closed

fix: Update the pyarrow to latest v14.0.1 regarding the CVE-2023-47248. #3835

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
9d671f1
fix: Update the pyarrow to latest v14.0.1 regarding the CVE-2023-47248.
shuchu Nov 14, 2023
9b4ec66
fix: Update the requirements file of python3.10.
shuchu Nov 15, 2023
0e0a5ad
fix: force the timestamp's datatype resolution to ms for bigquery whi…
shuchu Nov 16, 2023
fe61d0d
fix: Update the timestamp resolution to us which is exact same as usi…
shuchu Nov 16, 2023
e357d79
fix: Add debug info to check the entity_df before send to redshift.
shuchu Nov 17, 2023
20e55c4
fix: add debug to check the loaded pyarrow table file.
shuchu Nov 17, 2023
fbb5b20
fix: debug the transformation from pyarrow table to pandas df.
shuchu Nov 17, 2023
059914b
fix: roll pyarrow back.
shuchu Nov 17, 2023
f032d7a
fix: force test fail to check table and pandas df.
shuchu Nov 17, 2023
a2e217f
fix: debug the transformation from parquet file to pyarrow table.
shuchu Nov 17, 2023
0e43126
fix: debug the parquet file.
shuchu Nov 17, 2023
19d30ef
fix: test 14.0.1 parquet file.
shuchu Nov 17, 2023
ffe6255
fix: test the parquet reader of pandas instead of pyarrow.
shuchu Nov 17, 2023
dc2630e
fix: use datetime resolution us while writing test data into redshift.
shuchu Nov 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion sdk/python/feast/infra/offline_stores/bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,9 @@ def offline_write_batch(
)

with tempfile.TemporaryFile() as parquet_temp_file:
pyarrow.parquet.write_table(table=table, where=parquet_temp_file)
pyarrow.parquet.write_table(
table=table, where=parquet_temp_file, coerce_timestamps="us"
)

parquet_temp_file.seek(0)

Expand Down
28 changes: 25 additions & 3 deletions sdk/python/feast/infra/utils/aws_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,7 @@ def upload_arrow_table_to_redshift(
else:
# Write the PyArrow Table on disk in Parquet format and upload it to S3
with tempfile.TemporaryFile(suffix=".parquet") as parquet_temp_file:
pq.write_table(table, parquet_temp_file)
pq.write_table(table, parquet_temp_file, coerce_timestamps="us")
parquet_temp_file.seek(0)
s3_resource.Object(bucket, key).put(Body=parquet_temp_file)

Expand Down Expand Up @@ -566,7 +566,19 @@ def unload_redshift_query_to_pa(
with tempfile.TemporaryDirectory() as temp_dir:
download_s3_directory(s3_resource, bucket, key, temp_dir)
delete_s3_directory(s3_resource, bucket, key)
return pq.read_table(temp_dir)

# Debug
print("debug temp file using pandas. ")
df = pd.read_parquet(temp_dir)
# parquet_file = pq.ParquetDataset(temp_dir)
# df = parquet_file.read().to_pandas()
print(df)

print("transfer to table.")
table = pq.read_table(temp_dir)
print(table)

return table


def unload_redshift_query_to_df(
Expand All @@ -592,7 +604,17 @@ def unload_redshift_query_to_df(
iam_role,
query,
)
return table.to_pandas()

# Debug
print("Debug ------")
print(table.schema)
print(table)
print("Debug ++++++")

df = table.to_pandas()
print(df)
print("Debug ++++++")
return df


def get_lambda_function(lambda_client, function_name: str) -> Optional[Dict]:
Expand Down
46 changes: 23 additions & 23 deletions sdk/python/requirements/py3.10-ci-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,11 @@ black==22.12.0
# via feast (setup.py)
bleach==6.1.0
# via nbconvert
boto3==1.28.80
boto3==1.29.0
# via
# feast (setup.py)
# moto
botocore==1.31.80
botocore==1.32.0
# via
# boto3
# moto
Expand Down Expand Up @@ -135,7 +135,7 @@ cryptography==41.0.5
# snowflake-connector-python
# types-pyopenssl
# types-redis
dask==2023.10.1
dask==2023.11.0
# via feast (setup.py)
db-dtypes==1.1.1
# via google-cloud-bigquery
Expand Down Expand Up @@ -177,7 +177,7 @@ fastavro==1.9.0
# via
# feast (setup.py)
# pandavro
fastjsonschema==2.18.1
fastjsonschema==2.19.0
# via nbformat
filelock==3.13.1
# via
Expand All @@ -199,7 +199,7 @@ geojson==2.5.0
# via rockset
geomet==0.2.1.post1
# via cassandra-driver
google-api-core[grpc]==2.13.0
google-api-core[grpc]==2.14.0
# via
# feast (setup.py)
# firebase-admin
Expand All @@ -211,7 +211,7 @@ google-api-core[grpc]==2.13.0
# google-cloud-datastore
# google-cloud-firestore
# google-cloud-storage
google-api-python-client==2.107.0
google-api-python-client==2.108.0
# via firebase-admin
google-auth==2.23.4
# via
Expand Down Expand Up @@ -262,7 +262,7 @@ great-expectations==0.15.50
# via feast (setup.py)
greenlet==3.0.1
# via sqlalchemy
grpc-google-iam-v1==0.12.6
grpc-google-iam-v1==0.12.7
# via google-cloud-bigtable
grpcio==1.59.2
# via
Expand Down Expand Up @@ -298,7 +298,7 @@ hazelcast-python-client==5.3.0
# via feast (setup.py)
hiredis==2.2.3
# via feast (setup.py)
httpcore==1.0.1
httpcore==1.0.2
# via httpx
httplib2==0.22.0
# via
Expand Down Expand Up @@ -376,7 +376,7 @@ jsonschema[format-nongpl]==4.19.2
# jupyter-events
# jupyterlab-server
# nbformat
jsonschema-specifications==2023.7.1
jsonschema-specifications==2023.11.1
# via jsonschema
jupyter-client==8.6.0
# via
Expand Down Expand Up @@ -409,7 +409,7 @@ jupyterlab==4.0.8
# via notebook
jupyterlab-pygments==0.2.2
# via nbconvert
jupyterlab-server==2.25.0
jupyterlab-server==2.25.1
# via
# jupyterlab
# notebook
Expand All @@ -419,7 +419,7 @@ kubernetes==20.13.0
# via feast (setup.py)
locket==1.0.0
# via partd
makefun==1.15.1
makefun==1.15.2
# via great-expectations
markupsafe==2.1.3
# via
Expand All @@ -446,7 +446,7 @@ mock==2.0.0
# via feast (setup.py)
moreorless==0.4.0
# via bowler
moto==4.2.7
moto==4.2.8
# via feast (setup.py)
msal==1.25.0
# via
Expand Down Expand Up @@ -564,7 +564,7 @@ pre-commit==3.3.1
# via feast (setup.py)
prometheus-client==0.18.0
# via jupyter-server
prompt-toolkit==3.0.39
prompt-toolkit==3.0.41
# via ipython
proto-plus==1.22.3
# via
Expand Down Expand Up @@ -610,7 +610,7 @@ py-cpuinfo==9.0.0
# via pytest-benchmark
py4j==0.10.9.7
# via pyspark
pyarrow==10.0.1
pyarrow==14.0.1
# via
# db-dtypes
# feast (setup.py)
Expand Down Expand Up @@ -683,7 +683,7 @@ pytest-ordering==0.6
# via feast (setup.py)
pytest-timeout==1.4.2
# via feast (setup.py)
pytest-xdist==3.3.1
pytest-xdist==3.4.0
# via feast (setup.py)
python-dateutil==2.8.2
# via
Expand Down Expand Up @@ -723,7 +723,7 @@ pyzmq==25.1.1
# jupyter-server
redis==4.6.0
# via feast (setup.py)
referencing==0.30.2
referencing==0.31.0
# via
# jsonschema
# jsonschema-specifications
Expand Down Expand Up @@ -751,7 +751,7 @@ requests==2.31.0
# trino
requests-oauthlib==1.3.1
# via kubernetes
responses==0.24.0
responses==0.24.1
# via moto
rfc3339-validator==0.1.4
# via
Expand Down Expand Up @@ -798,7 +798,7 @@ sniffio==1.3.0
# httpx
snowballstemmer==2.2.0
# via sphinx
snowflake-connector-python[pandas]==3.4.0
snowflake-connector-python[pandas]==3.5.0
# via feast (setup.py)
sortedcontainers==2.4.0
# via snowflake-connector-python
Expand Down Expand Up @@ -826,7 +826,7 @@ sphinxcontrib-serializinghtml==1.1.9
# via sphinx
sqlalchemy[mypy]==1.4.50
# via feast (setup.py)
sqlalchemy2-stubs==0.0.2a36
sqlalchemy2-stubs==0.0.2a37
# via sqlalchemy
stack-data==0.6.3
# via ipython
Expand All @@ -836,7 +836,7 @@ tabulate==0.9.0
# via feast (setup.py)
tenacity==8.2.3
# via feast (setup.py)
terminado==0.17.1
terminado==0.18.0
# via
# jupyter-server
# jupyter-server-terminals
Expand Down Expand Up @@ -916,7 +916,7 @@ types-redis==4.6.0.10
# via feast (setup.py)
types-requests==2.30.0.0
# via feast (setup.py)
types-setuptools==68.2.0.0
types-setuptools==68.2.0.1
# via feast (setup.py)
types-tabulate==0.9.0.3
# via feast (setup.py)
Expand Down Expand Up @@ -966,7 +966,7 @@ volatile==2.1.0
# via bowler
watchfiles==0.21.0
# via uvicorn
wcwidth==0.2.9
wcwidth==0.2.10
# via prompt-toolkit
webcolors==1.13
# via jsonschema
Expand All @@ -987,7 +987,7 @@ wheel==0.41.3
# via pip-tools
widgetsnbextension==4.0.9
# via ipywidgets
wrapt==1.15.0
wrapt==1.16.0
# via testcontainers
xmltodict==0.13.0
# via moto
Expand Down
16 changes: 8 additions & 8 deletions sdk/python/requirements/py3.10-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ cloudpickle==3.0.0
# via dask
colorama==0.4.6
# via feast (setup.py)
dask==2023.10.1
dask==2023.11.0
# via feast (setup.py)
dill==0.3.7
# via feast (setup.py)
Expand Down Expand Up @@ -72,7 +72,7 @@ h11==0.14.0
# via
# httpcore
# uvicorn
httpcore==1.0.1
httpcore==1.0.2
# via httpx
httptools==0.6.1
# via uvicorn
Expand All @@ -93,7 +93,7 @@ jinja2==3.1.2
# via feast (setup.py)
jsonschema==4.19.2
# via feast (setup.py)
jsonschema-specifications==2023.7.1
jsonschema-specifications==2023.11.1
# via jsonschema
locket==1.0.0
# via partd
Expand All @@ -103,7 +103,7 @@ mmh3==4.0.1
# via feast (setup.py)
moreorless==0.4.0
# via bowler
mypy==1.6.1
mypy==1.7.0
# via sqlalchemy
mypy-extensions==1.0.0
# via mypy
Expand Down Expand Up @@ -137,7 +137,7 @@ protobuf==4.23.3
# grpcio-tools
# mypy-protobuf
# proto-plus
pyarrow==11.0.0
pyarrow==14.0.1
# via feast (setup.py)
pydantic==1.10.13
# via
Expand All @@ -156,7 +156,7 @@ pyyaml==6.0.1
# dask
# feast (setup.py)
# uvicorn
referencing==0.30.2
referencing==0.31.0
# via
# jsonschema
# jsonschema-specifications
Expand All @@ -176,7 +176,7 @@ sniffio==1.3.0
# httpx
sqlalchemy[mypy]==1.4.50
# via feast (setup.py)
sqlalchemy2-stubs==0.0.2a36
sqlalchemy2-stubs==0.0.2a37
# via sqlalchemy
starlette==0.27.0
# via fastapi
Expand Down Expand Up @@ -205,7 +205,7 @@ typing-extensions==4.8.0
# pydantic
# sqlalchemy2-stubs
# uvicorn
urllib3==2.0.7
urllib3==2.1.0
# via requests
uvicorn[standard]==0.24.0.post1
# via feast (setup.py)
Expand Down
4 changes: 2 additions & 2 deletions sdk/python/requirements/py3.8-ci-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -630,7 +630,7 @@ py-cpuinfo==9.0.0
# via pytest-benchmark
py4j==0.10.9.7
# via pyspark
pyarrow==10.0.1
pyarrow==14.0.1
# via
# db-dtypes
# feast (setup.py)
Expand Down Expand Up @@ -821,7 +821,7 @@ sniffio==1.3.0
# httpx
snowballstemmer==2.2.0
# via sphinx
snowflake-connector-python[pandas]==3.4.0
snowflake-connector-python[pandas]==3.5.0
# via feast (setup.py)
sortedcontainers==2.4.0
# via snowflake-connector-python
Expand Down
2 changes: 1 addition & 1 deletion sdk/python/requirements/py3.8-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ protobuf==4.23.3
# grpcio-tools
# mypy-protobuf
# proto-plus
pyarrow==11.0.0
pyarrow==14.0.1
# via feast (setup.py)
pydantic==1.10.13
# via
Expand Down
Loading