feast-dev · feast-ci-bot · May 3, 2022 · May 2, 2022 · May 2, 2022 · May 2, 2022
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,6 @@
+prune sdk/python/tests
+prune ui
+prune java/
+prune docs
+prune infra
+prune examples
diff --git a/Makefile b/Makefile
@@ -37,24 +37,24 @@ build: protos build-java build-docker
 # Python SDK
 
 install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies
-	cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt
-	cd sdk/python && COMPILE_GO=true python setup.py develop
+	python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
+	COMPILE_GO=true python setup.py develop
 
 lock-python-ci-dependencies:
-	cd sdk/python && python -m piptools compile -U --extra ci --output-file requirements/py$(PYTHON)-ci-requirements.txt
+	python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
 
 package-protos:
 	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
 
 compile-protos-python:
-	cd sdk/python && python setup.py build_python_protos
+	python setup.py build_python_protos
 
 install-python:
-	cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-requirements.txt
-	cd sdk/python && python setup.py develop
+	python -m piptools sync sdk/python/requirements/py$(PYTHON)-requirements.txt
+	python setup.py develop
 
 lock-python-dependencies:
-	cd sdk/python && python -m piptools compile -U --output-file requirements/py$(PYTHON)-requirements.txt
+	python -m piptools compile -U --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt
 
 benchmark-python:
 	FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data sdk/python/tests
@@ -164,14 +164,14 @@ install-protoc-dependencies:
 	pip install grpcio-tools==1.44.0 mypy-protobuf==3.1.0
 
 compile-protos-go: install-go-proto-dependencies install-protoc-dependencies
-	cd sdk/python && python setup.py build_go_protos
+	python setup.py build_go_protos
 
 compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies
-	cd sdk/python && COMPILE_GO=True python setup.py build_ext --inplace
+	COMPILE_GO=True python setup.py build_ext --inplace
 
 # Needs feast package to setup the feature store
 test-go: compile-protos-go
-	pip install -e "sdk/python[ci]"
+	pip install -e ".[ci]"
 	go test ./...
 
 format-go:

diff --git a/sdk/python/pyproject.toml → pyproject.toml b/sdk/python/pyproject.toml → pyproject.toml
@@ -7,12 +7,14 @@ COPY sdk/python/feast/infra/feature_servers/aws_lambda/app.py ${LAMBDA_TASK_ROOT
 COPY sdk/python sdk/python
 COPY protos protos
 COPY go go
+COPY setup.py setup.py
+COPY pyproject.toml pyproject.toml
 COPY README.md README.md
 
 # Install Feast for AWS with Lambda dependencies
 # TODO(felixwang9817): Remove Snowflake dependencies once lazy loading of offline stores is supported.
 # See https://github.com/feast-dev/feast/issues/2566 for more details.
-RUN pip3 install -e 'sdk/python[aws,redis,snowflake]'
+RUN pip3 install -e '.[aws,redis,snowflake]'
 RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}"
 
 # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)

@@ -13,11 +13,13 @@ COPY sdk/python/feast/infra/feature_servers/gcp_cloudrun/app.py ./app.py
 # Copy necessary parts of the Feast codebase
 COPY sdk/python ./sdk/python
 COPY protos ./protos
+COPY setup.py setup.py
+COPY pyproject.toml pyproject.toml
 COPY README.md ./README.md
 
 # Install production dependencies.
 RUN pip install --no-cache-dir \
-    -e 'sdk/python[gcp,redis]' \
+    -e '.[gcp,redis]' \
     -r ./sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt
 
 # Run the web service on container startup. Here we use the gunicorn

diff --git a/setup.cfg b/setup.cfg
@@ -0,0 +1,22 @@
+[isort]
+src_paths = feast,tests
+multi_line_output=3
+include_trailing_comma=True
+force_grid_wrap=0
+use_parentheses=True
+line_length=88
+skip=feast/protos,feast/embedded_go/lib
+known_first_party=feast,feast_serving_server,feast_core_server
+default_section=THIRDPARTY
+
+[flake8]
+ignore = E203, E266, E501, W503
+max-line-length = 88
+max-complexity = 20
+select = B,C,E,F,W,T4
+exclude = .git,__pycache__,docs/conf.py,dist,feast/protos,feast/embedded_go/lib
+
+[mypy]
+files=feast,tests
+ignore_missing_imports=true
+exclude=feast/embedded_go/lib
diff --git a/sdk/python/setup.py → setup.py b/sdk/python/setup.py → setup.py
@@ -57,7 +57,7 @@
     "Jinja2>=2.0.0",
     "jsonschema",
     "mmh3",
-    "numpy<1.22",  # 1.22 drops support for python 3.7.
+    "numpy<1.22",
     "pandas>=1.0.0",
     "pandavro==1.5.*",
     "protobuf>=3.10,<3.20",
@@ -124,6 +124,7 @@
 
 CI_REQUIRED = (
     [
+        "build",
         "cryptography==3.4.8",
         "flake8",
         "black==19.10b0",
@@ -179,7 +180,7 @@
 DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED
 
 # Get git repo root directory
-repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent)
+repo_root = str(pathlib.Path(__file__).resolve().parent)
 
 # README file from Feast repo root directory
 README_FILE = os.path.join(repo_root, "README.md")
@@ -195,11 +196,12 @@
 
 # Only set use_scm_version if git executable exists (setting this variable causes pip to use git under the hood)
 if shutil.which("git"):
-    use_scm_version = {"root": "../..", "relative_to": __file__, "tag_regex": TAG_REGEX}
+    use_scm_version = {"root": ".", "relative_to": __file__, "tag_regex": TAG_REGEX}
 else:
     use_scm_version = None
 
 PROTO_SUBDIRS = ["core", "serving", "types", "storage"]
+PYTHON_CODE_PREFIX = "sdk/python"
 
 
 class BuildPythonProtosCommand(Command):
@@ -214,7 +216,7 @@ def initialize_options(self):
         ]  # find_executable("protoc")
         self.proto_folder = os.path.join(repo_root, "protos")
         self.python_folder = os.path.join(
-            os.path.dirname(__file__) or os.getcwd(), "feast/protos"
+            os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos"
         )
         self.sub_folders = PROTO_SUBDIRS
 
@@ -223,7 +225,7 @@ def finalize_options(self):
 
     def _generate_python_protos(self, path: str):
         proto_files = glob.glob(os.path.join(self.proto_folder, path))
-        Path(self.python_folder).mkdir(exist_ok=True)
+        Path(self.python_folder).mkdir(parents=True, exist_ok=True)
         subprocess.check_call(
             self.python_protoc
             + [
@@ -244,19 +246,17 @@ def run(self):
             self._generate_python_protos(f"feast/{sub_folder}/*.proto")
             # We need the __init__ files for each of the generated subdirs
             # so that they are regular packages, and don't need the `--namespace-packages` flags
-            # when being typechecked using mypy. BUT, we need to exclude `types` because that clashes
-            # with an existing module in the python standard library.
-            if sub_folder == "types":
-                continue
+            # when being typechecked using mypy.
             with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'):
                 pass
 
+
         with open(f"{self.python_folder}/__init__.py", 'w'):
             pass
         with open(f"{self.python_folder}/feast/__init__.py", 'w'):
             pass
 
-        for path in Path("feast/protos").rglob("*.py"):
+        for path in Path(self.python_folder).rglob("*.py"):
             for folder in self.sub_folders:
                 # Read in the file
                 with open(path, "r") as file:
@@ -355,6 +355,7 @@ class BuildCommand(build_py):
 
     def run(self):
         self.run_command("build_python_protos")
+        self.run_command("build_ext")
         if os.getenv("COMPILE_GO", "false").lower() == "true":
             _ensure_go_and_proto_toolchain()
             self.run_command("build_go_protos")
@@ -421,10 +422,17 @@ def copy_extensions_to_source(self):
             modpath = fullname.split('.')
             package = '.'.join(modpath[:-1])
             package_dir = build_py.get_package_dir(package)
-            src = os.path.join(self.build_lib, package_dir)
+
+            src_dir = dest_dir = package_dir
+
+            if src_dir.startswith(PYTHON_CODE_PREFIX):
+                src_dir = package_dir[len(PYTHON_CODE_PREFIX):]
+            src_dir = src_dir.lstrip("/")
+
+            src_dir = os.path.join(self.build_lib, src_dir)
 
             # copy whole directory
-            copy_tree(src, package_dir)
+            copy_tree(src_dir, dest_dir)
 
 
 setup(
@@ -435,7 +443,8 @@ def copy_extensions_to_source(self):
     long_description_content_type="text/markdown",
     python_requires=REQUIRES_PYTHON,
     url=URL,
-    packages=find_packages(exclude=("tests",)),
+    packages=find_packages(where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")),
+    package_dir={"": PYTHON_CODE_PREFIX},
     install_requires=REQUIRED,
     # https://stackoverflow.com/questions/28509965/setuptools-development-requirements
     # Install dev requirements with: pip install -e .[dev]
@@ -474,9 +483,9 @@ def copy_extensions_to_source(self):
     ],
     package_data={
         "": [
-            "protos/feast/**/*.proto",
-            "protos/feast/third_party/grpc/health/v1/*.proto",
-            "feast/protos/feast/**/*.py",
+            "sdk/python/**/*.proto",
+            "sdk/python/**/*.py",
+            "protos/**/*.proto",
         ],
     },
     cmdclass={