Skip to content

chore: Make loading online and offline classes lazy #2646

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 4 commits into from
May 6, 2022
Merged

chore: Make loading online and offline classes lazy #2646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c243357
chore: Make loading online and offline classes lazy
achals May 6, 2022
be4834a
fixes
achals May 6, 2022
525e230
fixes
achals May 6, 2022
dc79121
nullability fixes
achals May 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions sdk/python/feast/infra/passthrough_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,24 @@ def __init__(self, config: RepoConfig):
super().__init__(config)

self.repo_config = config
self.offline_store = get_offline_store_from_config(config.offline_store)
self.online_store = (
get_online_store_from_config(config.online_store)
if config.online_store
else None
)
self._offline_store = None
self._online_store = None

@property
def online_store(self):
if not self._online_store and self.repo_config.online_store:
self._online_store = get_online_store_from_config(
self.repo_config.online_store
)
return self._online_store

@property
def offline_store(self):
if not self._offline_store:
self._offline_store = get_offline_store_from_config(
self.repo_config.offline_store
)
return self._offline_store

def update_infra(
self,
Expand Down
71 changes: 56 additions & 15 deletions sdk/python/feast/repo_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import yaml
from pydantic import (
BaseModel,
Field,
StrictInt,
StrictStr,
ValidationError,
Expand Down Expand Up @@ -107,10 +108,10 @@ class RepoConfig(FeastBaseModel):
provider: StrictStr
""" str: local or gcp or aws """

online_store: Any
_online_config: Any = Field(alias="online_store")
""" OnlineStoreConfig: Online store configuration (optional depending on provider) """

offline_store: Any
_offline_config: Any = Field(alias="offline_store")
""" OfflineStoreConfig: Offline store configuration (optional depending on provider) """

feature_server: Optional[Any]
Expand All @@ -126,19 +127,27 @@ class RepoConfig(FeastBaseModel):
def __init__(self, **data: Any):
super().__init__(**data)

if isinstance(self.online_store, Dict):
self.online_store = get_online_config_from_type(self.online_store["type"])(
**self.online_store
)
elif isinstance(self.online_store, str):
self.online_store = get_online_config_from_type(self.online_store)()

if isinstance(self.offline_store, Dict):
self.offline_store = get_offline_config_from_type(
self.offline_store["type"]
)(**self.offline_store)
elif isinstance(self.offline_store, str):
self.offline_store = get_offline_config_from_type(self.offline_store)()
self._offline_store = None
if "offline_store" in data:
self._offline_config = data["offline_store"]
else:
if data["provider"] == "local":
self._offline_config = "file"
elif data["provider"] == "gcp":
self._offline_config = "bigquery"
elif data["provider"] == "aws":
self._offline_config = "redshift"

self._online_store = None
if "online_store" in data:
self._online_config = data["online_store"]
else:
if data["provider"] == "local":
self._online_config = "sqlite"
elif data["provider"] == "gcp":
self._online_config = "datastore"
elif data["provider"] == "aws":
self._online_config = "dynamodb"

if isinstance(self.feature_server, Dict):
self.feature_server = get_feature_server_config_from_type(
Expand All @@ -151,6 +160,35 @@ def get_registry_config(self):
else:
return self.registry

@property
def offline_store(self):
if not self._offline_store:
if isinstance(self._offline_config, Dict):
self._offline_store = get_offline_config_from_type(
self._offline_config["type"]
)(**self._offline_config)
elif isinstance(self._offline_config, str):
self._offline_store = get_offline_config_from_type(
self._offline_config
)()
elif self._offline_config:
self._offline_store = self._offline_config
return self._offline_store

@property
def online_store(self):
if not self._online_store:
if isinstance(self._online_config, Dict):
self._online_store = get_online_config_from_type(
self._online_config["type"]
)(**self._online_config)
elif isinstance(self._online_config, str):
self._online_store = get_online_config_from_type(self._online_config)()
elif self._online_config:
self._online_store = self._online_config

return self._online_store

@root_validator(pre=True)
@log_exceptions
def _validate_online_store_config(cls, values):
Expand Down Expand Up @@ -304,6 +342,9 @@ def write_to_path(self, repo_path: Path):
sort_keys=False,
)

class Config:
allow_population_by_field_name = True


class FeastConfigError(Exception):
def __init__(self, error_message, config_path):
Expand Down
6 changes: 4 additions & 2 deletions sdk/python/tests/integration/feature_repos/integration_test_repo_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ class IntegrationTestRepoConfig:
"""

provider: str = "local"
online_store: Union[str, Dict] = "sqlite"
online_store: Optional[Union[str, Dict]] = "sqlite"

offline_store_creator: Type[DataSourceCreator] = FileDataSourceCreator
online_store_creator: Optional[Type[OnlineStoreCreator]] = None
Expand All @@ -38,8 +38,10 @@ def __repr__(self) -> str:
online_store_type = self.online_store.get("redis_type", "redis")
else:
online_store_type = self.online_store["type"]
else:
elif self.online_store:
online_store_type = self.online_store.__name__
else:
online_store_type = "none"
else:
online_store_type = self.online_store_creator.__name__

Expand Down
1 change: 1 addition & 0 deletions sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def repo_config():
project=PROJECT,
provider=PROVIDER,
online_store=DynamoDBOnlineStoreConfig(region=REGION),
# online_store={"type": "dynamodb", "region": REGION},
offline_store=FileOfflineStoreConfig(),
)

Expand Down