Skip to content

fix: Add spark to lambda dockerfile #2480

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 13 commits into from
Apr 4, 2022
Merged

fix: Add spark to lambda dockerfile #2480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
92caab0
add spark to lambda dockerfile
achals Apr 2, 2022
4aef576
add *args
achals Apr 2, 2022
cba63fb
Add *args
felixwang9817 Apr 2, 2022
01a5680
Pin protobuf==3.19.4
felixwang9817 Apr 2, 2022
e534b16
Remove *args
felixwang9817 Apr 2, 2022
886f450
Add a range
achals Apr 3, 2022
b8b94e1
Add a todo
achals Apr 4, 2022
47eb5a1
cleanup prints
achals Apr 4, 2022
4c0d78e
lock deps
achals Apr 4, 2022
997165e
lock deps correctly
achals Apr 4, 2022
992c3b8
fix lint
achals Apr 4, 2022
6badd47
fix lint take 2
achals Apr 4, 2022
dfe125e
Undo general updates
achals Apr 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ COPY go go
COPY README.md README.md

# Install Feast for AWS with Lambda dependencies
RUN pip3 install -e 'sdk/python[aws,redis]'
# TODO(achals): The additional spark deps should be removed. Details at https://github.com/feast-dev/feast/pull/2480.
RUN pip3 install -e 'sdk/python[aws,redis,spark]'
RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}"

# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
Expand Down
8 changes: 5 additions & 3 deletions sdk/python/feast/infra/offline_stores/bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,12 +338,14 @@ def to_bigquery(

def _to_arrow_internal(self) -> pyarrow.Table:
with self._query_generator() as query:
return self._execute_query(query).to_arrow()
q = self._execute_query(query=query)
assert q
return q.to_arrow()

@log_exceptions_and_usage
def _execute_query(
self, query, job_config=None, timeout: int = 1800
) -> bigquery.job.query.QueryJob:
) -> Optional[bigquery.job.query.QueryJob]:
bq_job = self.client.query(query, job_config=job_config)

if job_config and job_config.dry_run:
Expand Down Expand Up @@ -426,7 +428,7 @@ def _get_table_reference_for_new_entity(
dataset.location = dataset_location if dataset_location else "US"

try:
client.get_dataset(dataset)
client.get_dataset(dataset.reference)
except NotFound:
# Only create the dataset if it does not exist
client.create_dataset(dataset, exists_ok=True)
Expand Down
8 changes: 5 additions & 3 deletions sdk/python/feast/proto_json.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
JsonObject = Any


# TODO: These methods need to be updated when bumping the version of protobuf.
# https://github.com/feast-dev/feast/issues/2484
def _patch_proto_json_encoding(
proto_type: Type[ProtoMessage],
to_json_object: Callable[[_Printer, ProtoMessage], JsonObject],
Expand Down Expand Up @@ -68,7 +70,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
return value

def from_json_object(
parser: _Parser, value: JsonObject, message: ProtoMessage
parser: _Parser, value: JsonObject, message: ProtoMessage,
) -> None:
if value is None:
message.null_val = 0
Expand Down Expand Up @@ -140,7 +142,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
return [printer._MessageToJsonObject(item) for item in message.val]

def from_json_object(
parser: _Parser, value: JsonObject, message: ProtoMessage
parser: _Parser, value: JsonObject, message: ProtoMessage,
) -> None:
array = value if isinstance(value, list) else value["val"]
for item in array:
Expand Down Expand Up @@ -181,7 +183,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject:
return list(message.val)

def from_json_object(
parser: _Parser, value: JsonObject, message: ProtoMessage
parser: _Parser, value: JsonObject, message: ProtoMessage,
) -> None:
array = value if isinstance(value, list) else value["val"]
message.val.extend(array)
Expand Down
4 changes: 2 additions & 2 deletions sdk/python/requirements/py3.7-ci-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ mypy==0.931
# via feast (setup.py)
mypy-extensions==0.4.3
# via mypy
mypy-protobuf==3.1.0
mypy-protobuf==3.1
# via feast (setup.py)
nbclient==0.5.11
# via nbconvert
Expand Down Expand Up @@ -854,4 +854,4 @@ zipp==3.7.0

# The following packages are considered to be unsafe in a requirements file:
# pip
# setuptools
# setuptools
8 changes: 4 additions & 4 deletions sdk/python/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"mmh3",
"pandas>=1.0.0",
"pandavro==1.5.*",
"protobuf>=3.10",
"protobuf>=3.10,<3.20",
Copy link
Collaborator

@felixwang9817 felixwang9817 Apr 4, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: do we want to also update py3.7-requirements.txt and other files?

Copy link
Member Author

@achals achals Apr 4, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh yeah good point

Copy link
Member Author

@achals achals Apr 4, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just read through what the make lock-python-*dependencies command does and existing requirements file - I don't think any changes are needed here.

"proto-plus<1.19.7",
"pyarrow>=4.0.0",
"pydantic>=1.0.0",
Expand Down Expand Up @@ -111,7 +111,7 @@
"mock==2.0.0",
"moto",
"mypy==0.931",
"mypy-protobuf==3.1.0",
"mypy-protobuf==3.1",
"avro==1.10.0",
"gcsfs",
"urllib3>=1.25.4",
Expand Down Expand Up @@ -150,7 +150,7 @@
+ GE_REQUIRED
)

DEV_REQUIRED = ["mypy-protobuf>=3.1.0", "grpcio-testing==1.*"] + CI_REQUIRED
DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED

# Get git repo root directory
repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent)
Expand Down Expand Up @@ -422,7 +422,7 @@ def run(self):
"setuptools_scm",
"grpcio",
"grpcio-tools==1.34.0",
"mypy-protobuf==3.1.0",
"mypy-protobuf==3.1",
"sphinx!=4.0.0",
],
package_data={
Expand Down