Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -484,8 +484,8 @@ test-python-universal-cassandra: ## Run Python Cassandra integration tests
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.cassandra_online_store.cassandra_repo_configuration \
PYTEST_PLUGINS=sdk.python.tests.universal.feature_repos.universal.online_store.cassandra \
python -m pytest -x --integration \
sdk/python/tests/integration/offline_store/test_feature_logging.py \
--ignore=sdk/python/tests/integration/offline_store/test_validation.py \
sdk/python/tests/universal/offline_store/test_feature_logging.py \
--ignore=sdk/python/tests/universal/offline_store/test_dqm_validation.py \
-k "not test_snowflake and \
not test_spark_materialization_consistency and \
not test_universal_materialization"
Expand Down Expand Up @@ -574,7 +574,7 @@ test-python-universal-milvus-online: ## Run Python Milvus online store integrati
PYTEST_PLUGINS=sdk.python.tests.universal.feature_repos.universal.online_store.milvus \
python -m pytest -n 8 --integration \
-k "test_retrieve_online_milvus_documents" \
sdk/python/tests --ignore=sdk/python/tests/integration/offline_store/test_dqm_validation.py
sdk/python/tests --ignore=sdk/python/tests/universal/offline_store/test_dqm_validation.py

test-python-universal-singlestore-online: ## Run Python Singlestore online store integration tests
PYTHONPATH='.' \
Expand All @@ -594,7 +594,7 @@ test-python-universal-qdrant-online: ## Run Python Qdrant online store integrati
PYTEST_PLUGINS=sdk.python.tests.universal.feature_repos.universal.online_store.qdrant \
python -m pytest -n 8 --integration \
-k "test_retrieve_online_documents" \
sdk/python/tests/integration/online_store/test_universal_online.py
sdk/python/tests/universal/online_store/test_universal_online.py

# To use Couchbase as an offline store, you need to create an Couchbase Capella Columnar cluster on cloud.couchbase.com.
# Modify environment variables COUCHBASE_COLUMNAR_CONNECTION_STRING, COUCHBASE_COLUMNAR_USER, and COUCHBASE_COLUMNAR_PASSWORD
Expand Down
2 changes: 1 addition & 1 deletion docs/how-to-guides/adding-or-reusing-tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ Docstring tests are primarily smoke tests to make sure imports and setup functio
Let's look at a sample test using the universal repo:

{% tabs %}
{% tab code="sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py" %}
{% tab code="sdk/python/tests/universal/offline_store/test_universal_historical_retrieval.py" %}
```python
@pytest.mark.integration
@pytest.mark.universal_offline_stores
Expand Down
2 changes: 1 addition & 1 deletion sdk/python/tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ Docstring tests are primarily smoke tests to make sure imports and setup functio
Let's look at a sample test using the universal repo:

{% tabs %}
{% tab code="sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py" %}
{% tab code="sdk/python/tests/universal/offline_store/test_universal_historical_retrieval.py" %}
```python
@pytest.mark.integration
@pytest.mark.universal_offline_stores
Expand Down
170 changes: 0 additions & 170 deletions sdk/python/tests/integration/online_store/conftest.py
Original file line number Diff line number Diff line change
@@ -1,173 +1,3 @@
"""
Fixtures for online-store integration tests.
"""

from typing import Dict

import pytest
from testcontainers.core.container import DockerContainer
from testcontainers.core.waiting_utils import wait_for_logs

from tests.universal.feature_repos.universal.online_store_creator import (
OnlineStoreCreator,
)


class _SharedDbDynamoDBOnlineStoreCreator(OnlineStoreCreator):
"""DynamoDB Local container started with ``-sharedDb -inMemory``.

Why ``-sharedDb``
-----------------
DynamoDB Local 2.x namespaces tables by the **access key ID** in the
request signature. In CI, the sync ``boto3`` client and the async
``aiobotocore`` client can resolve credentials from *different* sources
(env vars, credential file, ``credential_process``, container IAM role,
etc.) even after ``monkeypatch.setenv`` has set fake keys—because the
credential chain is evaluated lazily and various caches may hold stale
values.

When the two clients end up using *different* access keys, the sync
client creates tables in namespace A while the async client queries
namespace B, which is empty → ``ResourceNotFoundException``.

``-sharedDb`` collapses all namespaces into a single in-memory database,
making table visibility completely independent of which credentials each
client uses. This is the correct setting for integration tests that want
to verify async read/write behaviour without caring about credential
isolation.
"""

def __init__(self, project_name: str, **kwargs):
super().__init__(project_name)
self.container = (
DockerContainer("amazon/dynamodb-local:latest")
.with_exposed_ports("8000")
.with_command("-jar DynamoDBLocal.jar -sharedDb -inMemory")
)

def create_online_store(self) -> Dict[str, str]:
self.container.start()
wait_for_logs(
container=self.container,
predicate="Initializing DynamoDB Local with the following configuration:",
timeout=10,
)
exposed_port = self.container.get_exposed_port("8000")
return {
"type": "dynamodb",
"endpoint_url": f"http://localhost:{exposed_port}",
"region": "us-west-2",
}

def teardown(self):
self.container.stop()


@pytest.fixture
async def dynamodb_local_environment(monkeypatch, worker_id):
"""Isolated, self-contained Environment for DynamoDB async tests.

Root cause of the async credential failures
-------------------------------------------
DynamoDB Local 2.x isolates tables **per access key ID**. In CI,
``boto3`` (sync, used to provision tables via ``store.apply()``) and
``aiobotocore`` (async, used for reads/writes in the test body) may
resolve credentials from *different* sources even when ``monkeypatch``
has set fake static keys—the credential chain is evaluated lazily and
caches may hold stale values from a real AWS session configured in the
runner environment.

When the two clients end up using different access key IDs they land in
different DynamoDB Local namespaces:

* sync client → namespace ``KEY_A`` → tables exist ✓
* async client → namespace ``KEY_B`` → tables not found → ``ResourceNotFoundException``

Fix: ``_SharedDbDynamoDBOnlineStoreCreator``
--------------------------------------------
The isolated container is started with ``-sharedDb -inMemory``. In
shared-DB mode DynamoDB Local stores *all* tables in a single namespace
regardless of the access key, so sync and async clients always see the
same tables.

Why async + ``await fs.initialize()`` before yielding
-----------------------------------------------------
Calling ``await fs.initialize()`` eagerly creates the ``aiobotocore``
client inside this fixture's event loop (the *same* loop the test will
run in). This pre-caches:

1. ``FeatureStore._provider`` so the identical ``DynamoDBOnlineStore``
instance is reused for the entire test.
2. The aiobotocore client, which is now unambiguously pointed at our
isolated container's ``endpoint_url``.

Yields
------
tuple[Environment, TestData]
``(environment, (entities, datasets, data_sources))``
"""
from feast.infra.online_stores.dynamodb import DynamoDBOnlineStore
from tests.universal.feature_repos.integration_test_repo_config import (
IntegrationTestRepoConfig,
)
from tests.universal.feature_repos.repo_configuration import (
construct_test_environment,
construct_universal_test_data,
)
from tests.universal.feature_repos.universal.data_sources.file import (
FileDataSourceCreator,
)

# Set fake static credentials before any boto client is created.
# These are accepted by DynamoDB Local regardless of validity.
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fakeaccesskey000000")
monkeypatch.setenv(
"AWS_SECRET_ACCESS_KEY", "fakesecretkey0000000000000000000000000000"
)
monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False)
monkeypatch.delenv("AWS_SECURITY_TOKEN", raising=False)
# Prevent IMDS from injecting real session tokens on EC2-backed runners.
monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")
# Disable the container credentials provider (ECS/EKS IAM roles).
monkeypatch.delenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", raising=False)
monkeypatch.delenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", raising=False)
# Ensure no profile redirects boto to a different credential source.
monkeypatch.delenv("AWS_PROFILE", raising=False)
monkeypatch.delenv("AWS_DEFAULT_PROFILE", raising=False)

# Reset class-level boto3 client caches so that no stale client from a
# previous test in this worker bleeds into our isolated environment.
DynamoDBOnlineStore._dynamodb_client = None
DynamoDBOnlineStore._dynamodb_resource = None

config = IntegrationTestRepoConfig(
provider="local",
offline_store_creator=FileDataSourceCreator,
online_store_creator=_SharedDbDynamoDBOnlineStoreCreator,
online_store=None,
)

environment = construct_test_environment(
config,
fixture_request=None,
worker_id=worker_id,
)
environment.setup()

# FileDataSourceCreator writes only local Parquet files — no AWS calls.
universal_test_data = construct_universal_test_data(environment)

# Eagerly initialise the aiobotocore client in *this* event loop so it
# is guaranteed to point at our container and is reused throughout the
# test body without lazy-init surprises.
await environment.feature_store.initialize()

yield environment, universal_test_data

# Cleanly shut down the async client before the container disappears.
await environment.feature_store.close()
environment.teardown()

# Flush class-level caches so the next test starts completely fresh.
DynamoDBOnlineStore._dynamodb_client = None
DynamoDBOnlineStore._dynamodb_resource = None
Empty file.
Empty file.
94 changes: 94 additions & 0 deletions sdk/python/tests/universal/online_store/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
"""
Fixtures for universal online-store tests.
"""

from typing import Dict

import pytest
from testcontainers.core.container import DockerContainer
from testcontainers.core.waiting_utils import wait_for_logs

from tests.universal.feature_repos.universal.online_store_creator import (
OnlineStoreCreator,
)


class _SharedDbDynamoDBOnlineStoreCreator(OnlineStoreCreator):
"""DynamoDB Local container started with ``-sharedDb -inMemory``."""

def __init__(self, project_name: str, **kwargs):
super().__init__(project_name)
self.container = (
DockerContainer("amazon/dynamodb-local:latest")
.with_exposed_ports("8000")
.with_command("-jar DynamoDBLocal.jar -sharedDb -inMemory")
)

def create_online_store(self) -> Dict[str, str]:
self.container.start()
wait_for_logs(
container=self.container,
predicate="Initializing DynamoDB Local with the following configuration:",
timeout=10,
)
exposed_port = self.container.get_exposed_port("8000")
return {
"type": "dynamodb",
"endpoint_url": f"http://localhost:{exposed_port}",
"region": "us-west-2",
}

def teardown(self):
self.container.stop()


@pytest.fixture
async def dynamodb_local_environment(monkeypatch, worker_id):
"""Isolated, self-contained Environment for DynamoDB async tests."""
from feast.infra.online_stores.dynamodb import DynamoDBOnlineStore
from tests.universal.feature_repos.integration_test_repo_config import (
IntegrationTestRepoConfig,
)
from tests.universal.feature_repos.repo_configuration import (
construct_test_environment,
construct_universal_test_data,
)
from tests.universal.feature_repos.universal.data_sources.file import (
FileDataSourceCreator,
)

monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fakeaccesskey000000")
monkeypatch.setenv(
"AWS_SECRET_ACCESS_KEY", "fakesecretkey0000000000000000000000000000"
)
monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False)
monkeypatch.delenv("AWS_SECURITY_TOKEN", raising=False)
monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")
monkeypatch.delenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", raising=False)
monkeypatch.delenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", raising=False)
monkeypatch.delenv("AWS_PROFILE", raising=False)
monkeypatch.delenv("AWS_DEFAULT_PROFILE", raising=False)

DynamoDBOnlineStore._dynamodb_client = None
DynamoDBOnlineStore._dynamodb_resource = None

config = IntegrationTestRepoConfig(
provider="local",
offline_store_creator=FileDataSourceCreator,
online_store_creator=_SharedDbDynamoDBOnlineStoreCreator,
online_store=None,
)

environment = construct_test_environment(
config, fixture_request=None, worker_id=worker_id
)
environment.setup()
universal_test_data = construct_universal_test_data(environment)
await environment.feature_store.initialize()

yield environment, universal_test_data

await environment.feature_store.close()
environment.teardown()
DynamoDBOnlineStore._dynamodb_client = None
DynamoDBOnlineStore._dynamodb_resource = None