Skip to content

Commit d59a91c

Browse files
achalsachals
authored
chore: Update python packaging to use build to produce sdists/wheels (#2630)
* chore: Update python packaging to use build and produce sdists/wheels Signed-off-by: Achal Shah <achals@gmail.com> * D'oh Signed-off-by: Achal Shah <achals@gmail.com> * fix path Signed-off-by: Achal Shah <achals@gmail.com> * fix path Signed-off-by: Achal Shah <achals@gmail.com> * fix path again Signed-off-by: Achal Shah <achals@gmail.com> * fix Dockerfile Signed-off-by: Achal Shah <achals@gmail.com> * invoke build_ext in build Signed-off-by: Achal Shah <achals@gmail.com> * fix src_dir and dest_dir Signed-off-by: Achal Shah <achals@gmail.com> * fix dockerfiles Signed-off-by: Achal Shah <achals@gmail.com> * CR Signed-off-by: Achal Shah <achals@gmail.com>
1 parent ccad832 commit d59a91c

File tree

8 files changed

+69
-28
lines changed

8 files changed

+69
-28
lines changed

‎MANIFEST.in‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
prune sdk/python/tests
2+
prune ui
3+
prune java/
4+
prune docs
5+
prune infra
6+
prune examples

‎Makefile‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -37,24 +37,24 @@ build: protos build-java build-docker
3737
# Python SDK
3838

3939
install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies
40-
cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt
41-
cd sdk/python && COMPILE_GO=true python setup.py develop
40+
python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
41+
COMPILE_GO=true python setup.py develop
4242

4343
lock-python-ci-dependencies:
44-
cd sdk/python && python -m piptools compile -U --extra ci --output-file requirements/py$(PYTHON)-ci-requirements.txt
44+
python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
4545

4646
package-protos:
4747
cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
4848

4949
compile-protos-python:
50-
cd sdk/python && python setup.py build_python_protos
50+
python setup.py build_python_protos
5151

5252
install-python:
53-
cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-requirements.txt
54-
cd sdk/python && python setup.py develop
53+
python -m piptools sync sdk/python/requirements/py$(PYTHON)-requirements.txt
54+
python setup.py develop
5555

5656
lock-python-dependencies:
57-
cd sdk/python && python -m piptools compile -U --output-file requirements/py$(PYTHON)-requirements.txt
57+
python -m piptools compile -U --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt
5858

5959
benchmark-python:
6060
FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests
@@ -164,14 +164,14 @@ install-protoc-dependencies:
164164
pip install grpcio-tools==1.44.0 mypy-protobuf==3.1.0
165165

166166
compile-protos-go: install-go-proto-dependencies install-protoc-dependencies
167-
cd sdk/python && python setup.py build_go_protos
167+
python setup.py build_go_protos
168168

169169
compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies
170-
cd sdk/python && COMPILE_GO=True python setup.py build_ext --inplace
170+
COMPILE_GO=True python setup.py build_ext --inplace
171171

172172
# Needs feast package to setup the feature store
173173
test-go: compile-protos-go
174-
pip install -e "sdk/python[ci]"
174+
pip install -e ".[ci]"
175175
go test ./...
176176

177177
format-go:

sdk/python/pyproject.toml renamed to pyproject.toml

File renamed without changes.

‎sdk/python/README.md‎

Whitespace-only changes.

‎sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@ COPY sdk/python/feast/infra/feature_servers/aws_lambda/app.py ${LAMBDA_TASK_ROOT
77
COPY sdk/python sdk/python
88
COPY protos protos
99
COPY go go
10+
COPY setup.py setup.py
11+
COPY pyproject.toml pyproject.toml
1012
COPY README.md README.md
1113

1214
# Install Feast for AWS with Lambda dependencies
1315
# TODO(felixwang9817): Remove Snowflake dependencies once lazy loading of offline stores is supported.
1416
# See https://github.com/feast-dev/feast/issues/2566 for more details.
15-
RUN pip3 install -e 'sdk/python[aws,redis,snowflake]'
17+
RUN pip3 install -e '.[aws,redis,snowflake]'
1618
RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}"
1719

1820
# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)

‎sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,13 @@ COPY sdk/python/feast/infra/feature_servers/gcp_cloudrun/app.py ./app.py
1313
# Copy necessary parts of the Feast codebase
1414
COPY sdk/python ./sdk/python
1515
COPY protos ./protos
16+
COPY setup.py setup.py
17+
COPY pyproject.toml pyproject.toml
1618
COPY README.md ./README.md
1719

1820
# Install production dependencies.
1921
RUN pip install --no-cache-dir \
20-
-e 'sdk/python[gcp,redis]' \
22+
-e '.[gcp,redis]' \
2123
-r ./sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt
2224

2325
# Run the web service on container startup. Here we use the gunicorn

‎setup.cfg‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[isort]
2+
src_paths = feast,tests
3+
multi_line_output=3
4+
include_trailing_comma=True
5+
force_grid_wrap=0
6+
use_parentheses=True
7+
line_length=88
8+
skip=feast/protos,feast/embedded_go/lib
9+
known_first_party=feast,feast_serving_server,feast_core_server
10+
default_section=THIRDPARTY
11+
12+
[flake8]
13+
ignore = E203, E266, E501, W503
14+
max-line-length = 88
15+
max-complexity = 20
16+
select = B,C,E,F,W,T4
17+
exclude = .git,__pycache__,docs/conf.py,dist,feast/protos,feast/embedded_go/lib
18+
19+
[mypy]
20+
files=feast,tests
21+
ignore_missing_imports=true
22+
exclude=feast/embedded_go/lib

sdk/python/setup.py renamed to setup.py

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
"Jinja2>=2.0.0",
5858
"jsonschema",
5959
"mmh3",
60-
"numpy<1.22", # 1.22 drops support for python 3.7.
60+
"numpy<1.22",
6161
"pandas>=1.0.0",
6262
"pandavro==1.5.*",
6363
"protobuf>=3.10,<3.20",
@@ -124,6 +124,7 @@
124124

125125
CI_REQUIRED = (
126126
[
127+
"build",
127128
"cryptography==3.4.8",
128129
"flake8",
129130
"black==19.10b0",
@@ -179,7 +180,7 @@
179180
DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED
180181

181182
# Get git repo root directory
182-
repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent)
183+
repo_root = str(pathlib.Path(__file__).resolve().parent)
183184

184185
# README file from Feast repo root directory
185186
README_FILE = os.path.join(repo_root, "README.md")
@@ -195,11 +196,12 @@
195196

196197
# Only set use_scm_version if git executable exists (setting this variable causes pip to use git under the hood)
197198
if shutil.which("git"):
198-
use_scm_version = {"root": "../..", "relative_to": __file__, "tag_regex": TAG_REGEX}
199+
use_scm_version = {"root": ".", "relative_to": __file__, "tag_regex": TAG_REGEX}
199200
else:
200201
use_scm_version = None
201202

202203
PROTO_SUBDIRS = ["core", "serving", "types", "storage"]
204+
PYTHON_CODE_PREFIX = "sdk/python"
203205

204206

205207
class BuildPythonProtosCommand(Command):
@@ -214,7 +216,7 @@ def initialize_options(self):
214216
] # find_executable("protoc")
215217
self.proto_folder = os.path.join(repo_root, "protos")
216218
self.python_folder = os.path.join(
217-
os.path.dirname(__file__) or os.getcwd(), "feast/protos"
219+
os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos"
218220
)
219221
self.sub_folders = PROTO_SUBDIRS
220222

@@ -223,7 +225,7 @@ def finalize_options(self):
223225

224226
def _generate_python_protos(self, path: str):
225227
proto_files = glob.glob(os.path.join(self.proto_folder, path))
226-
Path(self.python_folder).mkdir(exist_ok=True)
228+
Path(self.python_folder).mkdir(parents=True, exist_ok=True)
227229
subprocess.check_call(
228230
self.python_protoc
229231
+ [
@@ -244,19 +246,17 @@ def run(self):
244246
self._generate_python_protos(f"feast/{sub_folder}/*.proto")
245247
# We need the __init__ files for each of the generated subdirs
246248
# so that they are regular packages, and don't need the `--namespace-packages` flags
247-
# when being typechecked using mypy. BUT, we need to exclude `types` because that clashes
248-
# with an existing module in the python standard library.
249-
if sub_folder == "types":
250-
continue
249+
# when being typechecked using mypy.
251250
with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'):
252251
pass
253252

253+
254254
with open(f"{self.python_folder}/__init__.py", 'w'):
255255
pass
256256
with open(f"{self.python_folder}/feast/__init__.py", 'w'):
257257
pass
258258

259-
for path in Path("feast/protos").rglob("*.py"):
259+
for path in Path(self.python_folder).rglob("*.py"):
260260
for folder in self.sub_folders:
261261
# Read in the file
262262
with open(path, "r") as file:
@@ -355,6 +355,7 @@ class BuildCommand(build_py):
355355

356356
def run(self):
357357
self.run_command("build_python_protos")
358+
self.run_command("build_ext")
358359
if os.getenv("COMPILE_GO", "false").lower() == "true":
359360
_ensure_go_and_proto_toolchain()
360361
self.run_command("build_go_protos")
@@ -421,10 +422,17 @@ def copy_extensions_to_source(self):
421422
modpath = fullname.split('.')
422423
package = '.'.join(modpath[:-1])
423424
package_dir = build_py.get_package_dir(package)
424-
src = os.path.join(self.build_lib, package_dir)
425+
426+
src_dir = dest_dir = package_dir
427+
428+
if src_dir.startswith(PYTHON_CODE_PREFIX):
429+
src_dir = package_dir[len(PYTHON_CODE_PREFIX):]
430+
src_dir = src_dir.lstrip("/")
431+
432+
src_dir = os.path.join(self.build_lib, src_dir)
425433

426434
# copy whole directory
427-
copy_tree(src, package_dir)
435+
copy_tree(src_dir, dest_dir)
428436

429437

430438
setup(
@@ -435,7 +443,8 @@ def copy_extensions_to_source(self):
435443
long_description_content_type="text/markdown",
436444
python_requires=REQUIRES_PYTHON,
437445
url=URL,
438-
packages=find_packages(exclude=("tests",)),
446+
packages=find_packages(where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")),
447+
package_dir={"": PYTHON_CODE_PREFIX},
439448
install_requires=REQUIRED,
440449
# https://stackoverflow.com/questions/28509965/setuptools-development-requirements
441450
# Install dev requirements with: pip install -e .[dev]
@@ -474,9 +483,9 @@ def copy_extensions_to_source(self):
474483
],
475484
package_data={
476485
"": [
477-
"protos/feast/**/*.proto",
478-
"protos/feast/third_party/grpc/health/v1/*.proto",
479-
"feast/protos/feast/**/*.py",
486+
"sdk/python/**/*.proto",
487+
"sdk/python/**/*.py",
488+
"protos/**/*.proto",
480489
],
481490
},
482491
cmdclass={

0 commit comments

Comments
 (0)