feast-dev · tokoko · Sep 22, 2024 · Sep 20, 2024 · Sep 20, 2024 · Sep 20, 2024
diff --git a/.gitignore b/.gitignore
@@ -185,7 +185,6 @@ dmypy.json
 
 # Protos
 sdk/python/docs/html
-sdk/python/feast/protos/
 sdk/go/protos/
 go/protos/
 

diff --git a/Makefile b/Makefile
@@ -40,29 +40,20 @@ build: protos build-java build-docker
 install-python-ci-dependencies:
 	python -m piptools sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
 	pip install --no-deps -e .
-	python setup.py build_python_protos --inplace
 
 install-python-ci-dependencies-uv:
 	uv pip sync --system sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
 	uv pip install --system --no-deps -e .
-	python setup.py build_python_protos --inplace
 
 install-python-ci-dependencies-uv-venv:
 	uv pip sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
 	uv pip install --no-deps -e .
-	python setup.py build_python_protos --inplace
-
-install-protoc-dependencies:
-	pip install "protobuf<5" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1"
 
 lock-python-ci-dependencies:
 	uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
 
-package-protos:
-	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
-
-compile-protos-python: install-protoc-dependencies
-	python setup.py build_python_protos --inplace
+compile-protos-python:
+	python infra/scripts/generate_protos.py
 
 install-python:
 	python -m piptools sync sdk/python/requirements/py$(PYTHON_VERSION)-requirements.txt

@@ -0,0 +1,80 @@
+import os
+import sys
+import glob
+import subprocess
+from pathlib import Path
+
+repo_root = str(Path(__file__).resolve().parent)
+
+PROTO_SUBDIRS = ["core", "registry", "serving", "types", "storage"]
+PYTHON_CODE_PREFIX = "sdk/python"
+
+class BuildPythonProtosCommand:
+    description = "Builds the proto files into Python files."
+    user_options = [
+        ("inplace", "i", "Write generated proto files to source directory."),
+    ]
+
+    def __init__(self):
+        self.python_protoc = [
+            sys.executable,
+            "-m",
+            "grpc_tools.protoc",
+        ]
+        self.proto_folder = "protos"
+        self.sub_folders = PROTO_SUBDIRS
+        self.inplace = 0
+
+    @property
+    def python_folder(self):
+        return "sdk/python/feast/protos"
+
+    def _generate_python_protos(self, path: str):
+        proto_files = glob.glob(os.path.join(self.proto_folder, path))
+        Path(self.python_folder).mkdir(parents=True, exist_ok=True)
+        subprocess.check_call(
+            self.python_protoc
+            + [
+                "-I",
+                self.proto_folder,
+                "--python_out",
+                self.python_folder,
+                "--grpc_python_out",
+                self.python_folder,
+                "--mypy_out",
+                self.python_folder,
+            ]
+            + proto_files
+        )
+
+    def run(self):
+        for sub_folder in self.sub_folders:
+            self._generate_python_protos(f"feast/{sub_folder}/*.proto")
+            # We need the __init__ files for each of the generated subdirs
+            # so that they are regular packages, and don't need the `--namespace-packages` flags
+            # when being typechecked using mypy.
+            with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", "w"):
+                pass
+
+        with open(f"{self.python_folder}/__init__.py", "w"):
+            pass
+        with open(f"{self.python_folder}/feast/__init__.py", "w"):
+            pass
+
+        for path in Path(self.python_folder).rglob("*.py"):
+            for folder in self.sub_folders:
+                # Read in the file
+                with open(path, "r") as file:
+                    filedata = file.read()
+
+                # Replace the target string
+                filedata = filedata.replace(
+                    f"from feast.{folder}", f"from feast.protos.feast.{folder}"
+                )
+
+                # Write the file out again
+                with open(path, "w") as file:
+                    file.write(filedata)
+
+if __name__ == "__main__":
+    BuildPythonProtosCommand().run()
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,8 +1,5 @@
 [build-system]
 requires = [
-  "protobuf<5",
-  "grpcio-tools>=1.56.2,<2",
-  "mypy-protobuf>=3.1",
   "pybindgen==0.22.0",
   "setuptools>=60",
   "setuptools_scm>=6.2",

@@ -0,0 +1,42 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.duration_pb2
+import google.protobuf.message
+import sys
+
+if sys.version_info >= (3, 8):
+    import typing as typing_extensions
+else:
+    import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+class Aggregation(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    COLUMN_FIELD_NUMBER: builtins.int
+    FUNCTION_FIELD_NUMBER: builtins.int
+    TIME_WINDOW_FIELD_NUMBER: builtins.int
+    SLIDE_INTERVAL_FIELD_NUMBER: builtins.int
+    column: builtins.str
+    function: builtins.str
+    @property
+    def time_window(self) -> google.protobuf.duration_pb2.Duration: ...
+    @property
+    def slide_interval(self) -> google.protobuf.duration_pb2.Duration: ...
+    def __init__(
+        self,
+        *,
+        column: builtins.str = ...,
+        function: builtins.str = ...,
+        time_window: google.protobuf.duration_pb2.Duration | None = ...,
+        slide_interval: google.protobuf.duration_pb2.Duration | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["slide_interval", b"slide_interval", "time_window", b"time_window"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["column", b"column", "function", b"function", "slide_interval", b"slide_interval", "time_window", b"time_window"]) -> None: ...
+
+global___Aggregation = Aggregation
@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
@@ -0,0 +1,143 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+
+Copyright 2020 The Feast Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.message
+import sys
+
+if sys.version_info >= (3, 8):
+    import typing as typing_extensions
+else:
+    import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+class FileFormat(google.protobuf.message.Message):
+    """Defines the file format encoding the features/entity data in files"""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    class ParquetFormat(google.protobuf.message.Message):
+        """Defines options for the Parquet data format"""
+
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        def __init__(
+            self,
+        ) -> None: ...
+
+    class DeltaFormat(google.protobuf.message.Message):
+        """Defines options for delta data format"""
+
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        def __init__(
+            self,
+        ) -> None: ...
+
+    PARQUET_FORMAT_FIELD_NUMBER: builtins.int
+    DELTA_FORMAT_FIELD_NUMBER: builtins.int
+    @property
+    def parquet_format(self) -> global___FileFormat.ParquetFormat: ...
+    @property
+    def delta_format(self) -> global___FileFormat.DeltaFormat: ...
+    def __init__(
+        self,
+        *,
+        parquet_format: global___FileFormat.ParquetFormat | None = ...,
+        delta_format: global___FileFormat.DeltaFormat | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["delta_format", b"delta_format", "format", b"format", "parquet_format", b"parquet_format"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["delta_format", b"delta_format", "format", b"format", "parquet_format", b"parquet_format"]) -> None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["format", b"format"]) -> typing_extensions.Literal["parquet_format", "delta_format"] | None: ...
+
+global___FileFormat = FileFormat
+
+class StreamFormat(google.protobuf.message.Message):
+    """Defines the data format encoding features/entity data in data streams"""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    class ProtoFormat(google.protobuf.message.Message):
+        """Defines options for the protobuf data format"""
+
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        CLASS_PATH_FIELD_NUMBER: builtins.int
+        class_path: builtins.str
+        """Classpath to the generated Java Protobuf class that can be used to decode
+        Feature data from the obtained stream message
+        """
+        def __init__(
+            self,
+            *,
+            class_path: builtins.str = ...,
+        ) -> None: ...
+        def ClearField(self, field_name: typing_extensions.Literal["class_path", b"class_path"]) -> None: ...
+
+    class AvroFormat(google.protobuf.message.Message):
+        """Defines options for the avro data format"""
+
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        SCHEMA_JSON_FIELD_NUMBER: builtins.int
+        schema_json: builtins.str
+        """Optional if used in a File DataSource as schema is embedded in avro file.
+        Specifies the schema of the Avro message as JSON string.
+        """
+        def __init__(
+            self,
+            *,
+            schema_json: builtins.str = ...,
+        ) -> None: ...
+        def ClearField(self, field_name: typing_extensions.Literal["schema_json", b"schema_json"]) -> None: ...
+
+    class JsonFormat(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        SCHEMA_JSON_FIELD_NUMBER: builtins.int
+        schema_json: builtins.str
+        def __init__(
+            self,
+            *,
+            schema_json: builtins.str = ...,
+        ) -> None: ...
+        def ClearField(self, field_name: typing_extensions.Literal["schema_json", b"schema_json"]) -> None: ...
+
+    AVRO_FORMAT_FIELD_NUMBER: builtins.int
+    PROTO_FORMAT_FIELD_NUMBER: builtins.int
+    JSON_FORMAT_FIELD_NUMBER: builtins.int
+    @property
+    def avro_format(self) -> global___StreamFormat.AvroFormat: ...
+    @property
+    def proto_format(self) -> global___StreamFormat.ProtoFormat: ...
+    @property
+    def json_format(self) -> global___StreamFormat.JsonFormat: ...
+    def __init__(
+        self,
+        *,
+        avro_format: global___StreamFormat.AvroFormat | None = ...,
+        proto_format: global___StreamFormat.ProtoFormat | None = ...,
+        json_format: global___StreamFormat.JsonFormat | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["avro_format", b"avro_format", "format", b"format", "json_format", b"json_format", "proto_format", b"proto_format"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["avro_format", b"avro_format", "format", b"format", "json_format", b"json_format", "proto_format", b"proto_format"]) -> None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["format", b"format"]) -> typing_extensions.Literal["avro_format", "proto_format", "json_format"] | None: ...
+
+global___StreamFormat = StreamFormat
@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+