feast-dev · franciscojavierarceo · May 29, 2024 · May 27, 2024 · May 27, 2024 · May 27, 2024
diff --git a/Makefile b/Makefile
@@ -80,19 +80,16 @@ test-python-unit:
 	python -m pytest -n 8 --color=yes sdk/python/tests
 
 test-python-integration:
-	python -m pytest -n 8 --integration -k "(not snowflake or not test_historical_features_main) and not minio_registry" --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests
+	python -m pytest -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread \
+		-k "(not snowflake or not test_historical_features_main)" \
+		sdk/python/tests
 
 test-python-integration-local:
-	@(docker info > /dev/null 2>&1 && \
-		FEAST_IS_LOCAL_TEST=True \
-		FEAST_LOCAL_ONLINE_CONTAINER=True \
-		python -m pytest -n 8 --color=yes --integration \
-			-k "not gcs_registry and \
- 				not s3_registry and \
- 				not test_lambda_materialization and \
- 				not test_snowflake_materialization" \
-		sdk/python/tests \
-	) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
+	FEAST_IS_LOCAL_TEST=True \
+	FEAST_LOCAL_ONLINE_CONTAINER=True \
+	python -m pytest -n 8 --color=yes --integration --durations=5 --dist loadgroup \
+		-k "not test_lambda_materialization and not test_snowflake_materialization" \
+		sdk/python/tests
 
 test-python-integration-container:
 	@(docker info > /dev/null 2>&1 && \

@@ -141,7 +141,8 @@ def __init__(self, project_name: str, *args, **kwargs):
         self.minio = MinioContainer()
         self.minio.start()
         client = self.minio.get_client()
-        client.make_bucket("test")
+        if not client.bucket_exists("test"):
+            client.make_bucket("test")
         host_ip = self.minio.get_container_host_ip()
         exposed_port = self.minio.get_exposed_port(self.minio.port)
         self.endpoint_url = f"http://{host_ip}:{exposed_port}"

@@ -11,68 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-import time
 from datetime import timedelta
 from tempfile import mkstemp
 
 import pytest
 from pytest_lazyfixture import lazy_fixture
 
-from feast import FileSource
-from feast.data_format import ParquetFormat
 from feast.entity import Entity
 from feast.feature_store import FeatureStore
 from feast.feature_view import FeatureView
-from feast.field import Field
-from feast.infra.offline_stores.file import FileOfflineStoreConfig
-from feast.infra.online_stores.dynamodb import DynamoDBOnlineStoreConfig
 from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig
 from feast.repo_config import RepoConfig
-from feast.types import Array, Bytes, Float64, Int64, String
+from feast.types import Float64, Int64, String
 from tests.utils.data_source_test_creator import prep_file_source
 
 
-@pytest.mark.integration
-@pytest.mark.parametrize(
-    "test_feature_store",
-    [
-        lazy_fixture("feature_store_with_gcs_registry"),
-        lazy_fixture("feature_store_with_s3_registry"),
-    ],
-)
-def test_apply_entity_integration(test_feature_store):
-    entity = Entity(
-        name="driver_car_id",
-        description="Car driver id",
-        tags={"team": "matchmaking"},
-    )
-
-    # Register Entity
-    test_feature_store.apply([entity])
-
-    entities = test_feature_store.list_entities()
-
-    entity = entities[0]
-    assert (
-        len(entities) == 1
-        and entity.name == "driver_car_id"
-        and entity.description == "Car driver id"
-        and "team" in entity.tags
-        and entity.tags["team"] == "matchmaking"
-    )
-
-    entity = test_feature_store.get_entity("driver_car_id")
-    assert (
-        entity.name == "driver_car_id"
-        and entity.description == "Car driver id"
-        and "team" in entity.tags
-        and entity.tags["team"] == "matchmaking"
-    )
-
-    test_feature_store.teardown()
-
-
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_feature_store",
@@ -109,81 +62,6 @@ def test_feature_view_inference_success(test_feature_store, dataframe_source):
         test_feature_store.teardown()
 
 
-@pytest.mark.integration
-@pytest.mark.parametrize(
-    "test_feature_store",
-    [
-        lazy_fixture("feature_store_with_gcs_registry"),
-        lazy_fixture("feature_store_with_s3_registry"),
-    ],
-)
-def test_apply_feature_view_integration(test_feature_store):
-    # Create Feature Views
-    batch_source = FileSource(
-        file_format=ParquetFormat(),
-        path="file://feast/*",
-        timestamp_field="ts_col",
-        created_timestamp_column="timestamp",
-    )
-
-    entity = Entity(name="fs1_my_entity_1", join_keys=["test"])
-
-    fv1 = FeatureView(
-        name="my_feature_view_1",
-        schema=[
-            Field(name="fs1_my_feature_1", dtype=Int64),
-            Field(name="fs1_my_feature_2", dtype=String),
-            Field(name="fs1_my_feature_3", dtype=Array(String)),
-            Field(name="fs1_my_feature_4", dtype=Array(Bytes)),
-            Field(name="test", dtype=Int64),
-        ],
-        entities=[entity],
-        tags={"team": "matchmaking"},
-        source=batch_source,
-        ttl=timedelta(minutes=5),
-    )
-
-    # Register Feature View
-    test_feature_store.apply([fv1, entity])
-
-    feature_views = test_feature_store.list_feature_views()
-
-    # List Feature Views
-    assert (
-        len(feature_views) == 1
-        and feature_views[0].name == "my_feature_view_1"
-        and feature_views[0].features[0].name == "fs1_my_feature_1"
-        and feature_views[0].features[0].dtype == Int64
-        and feature_views[0].features[1].name == "fs1_my_feature_2"
-        and feature_views[0].features[1].dtype == String
-        and feature_views[0].features[2].name == "fs1_my_feature_3"
-        and feature_views[0].features[2].dtype == Array(String)
-        and feature_views[0].features[3].name == "fs1_my_feature_4"
-        and feature_views[0].features[3].dtype == Array(Bytes)
-        and feature_views[0].entities[0] == "fs1_my_entity_1"
-    )
-
-    feature_view = test_feature_store.get_feature_view("my_feature_view_1")
-    assert (
-        feature_view.name == "my_feature_view_1"
-        and feature_view.features[0].name == "fs1_my_feature_1"
-        and feature_view.features[0].dtype == Int64
-        and feature_view.features[1].name == "fs1_my_feature_2"
-        and feature_view.features[1].dtype == String
-        and feature_view.features[2].name == "fs1_my_feature_3"
-        and feature_view.features[2].dtype == Array(String)
-        and feature_view.features[3].name == "fs1_my_feature_4"
-        and feature_view.features[3].dtype == Array(Bytes)
-        and feature_view.entities[0] == "fs1_my_entity_1"
-    )
-
-    test_feature_store.delete_feature_view("my_feature_view_1")
-    feature_views = test_feature_store.list_feature_views()
-    assert len(feature_views) == 0
-
-    test_feature_store.teardown()
-
-
 @pytest.fixture
 def feature_store_with_local_registry():
     fd, registry_path = mkstemp()
@@ -197,46 +75,3 @@ def feature_store_with_local_registry():
             entity_key_serialization_version=2,
         )
     )
-
-
-@pytest.fixture
-def feature_store_with_gcs_registry():
-    from google.cloud import storage
-
-    storage_client = storage.Client()
-    bucket_name = f"feast-registry-test-{int(time.time() * 1000)}"
-    bucket = storage_client.bucket(bucket_name)
-    bucket = storage_client.create_bucket(bucket)
-    bucket.add_lifecycle_delete_rule(
-        age=14
-    )  # delete buckets automatically after 14 days
-    bucket.patch()
-    bucket.blob("registry.db")
-
-    return FeatureStore(
-        config=RepoConfig(
-            registry=f"gs://{bucket_name}/registry.db",
-            project="default",
-            provider="gcp",
-            entity_key_serialization_version=2,
-        )
-    )
-
-
-@pytest.fixture
-def feature_store_with_s3_registry():
-    aws_registry_path = os.getenv(
-        "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries"
-    )
-    return FeatureStore(
-        config=RepoConfig(
-            registry=f"{aws_registry_path}/{int(time.time() * 1000)}/registry.db",
-            project="default",
-            provider="aws",
-            online_store=DynamoDBOnlineStoreConfig(
-                region=os.getenv("AWS_REGION", "us-west-2")
-            ),
-            offline_store=FileOfflineStoreConfig(),
-            entity_key_serialization_version=2,
-        )
-    )
@@ -7,7 +7,9 @@
 from assertpy import assertpy
 
 from feast.feature_store import FeatureStore
-from tests.integration.feature_repos.repo_configuration import Environment
+from tests.integration.feature_repos.universal.data_sources.file import (
+    FileDataSourceCreator,
+)
 from tests.utils.basic_read_write_test import basic_rw_test
 from tests.utils.cli_repo_creator import CliRunner, get_example_repo
 from tests.utils.e2e_test_validation import (
@@ -17,8 +19,7 @@
 
 
 @pytest.mark.integration
-@pytest.mark.universal_offline_stores
-def test_universal_cli(environment: Environment):
+def test_universal_cli():
     project = f"test_universal_cli_{str(uuid.uuid4()).replace('-', '')[:8]}"
     runner = CliRunner()
 
@@ -28,9 +29,9 @@ def test_universal_cli(environment: Environment):
             feature_store_yaml = make_feature_store_yaml(
                 project,
                 repo_path,
-                environment.data_source_creator,
-                environment.provider,
-                environment.online_store,
+                FileDataSourceCreator("project"),
+                "local",
+                {"type": "sqlite"},
             )
 
             repo_config = repo_path / "feature_store.yaml"
@@ -115,8 +116,7 @@ def test_universal_cli(environment: Environment):
 
 
 @pytest.mark.integration
-@pytest.mark.universal_offline_stores
-def test_odfv_apply(environment) -> None:
+def test_odfv_apply() -> None:
     project = f"test_odfv_apply{str(uuid.uuid4()).replace('-', '')[:8]}"
     runner = CliRunner()
 
@@ -126,9 +126,9 @@ def test_odfv_apply(environment) -> None:
             feature_store_yaml = make_feature_store_yaml(
                 project,
                 repo_path,
-                environment.data_source_creator,
-                environment.provider,
-                environment.online_store,
+                FileDataSourceCreator("project"),
+                "local",
+                {"type": "sqlite"},
             )
 
             repo_config = repo_path / "feature_store.yaml"