Skip to content

feat(server): Remote Materialization #6649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ntkathole merged 13 commits into from
Aug 6, 2026
+619 −58
Merged

feat(server): Remote Materialization #6649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
f61be7d
feat(server): Add async materialization via ?async=true query param
aniketpalu Jul 28, 2026
9e121fc
Merge branch 'master' into feat/remote-materialize-v2
aniketpalu Jul 28, 2026
6c3aaa7
feat(server): Add async materialization via ?async=true query param
aniketpalu Jul 28, 2026
1384c60
feat(sdk): Auto-delegate materialize to feature server when online_st…
aniketpalu Jul 28, 2026
9a97b70
fix: Conflicts resolved
aniketpalu Jul 28, 2026
b5cb173
fix(server): Let store own MATERIALIZING transitions on async path
aniketpalu Jul 28, 2026
dc6f965
Merge branch 'master' into feat/remote-materialize-v2
aniketpalu Jul 30, 2026
d8e7a99
fix: Address #6649 review — run_async, race reserve, version threading
aniketpalu Jul 31, 2026
059c2d2
Merge branch 'master' into feat/remote-materialize-v2
aniketpalu Aug 2, 2026
c602fe3
fix(server): Only reset MATERIALIZING FVs on async materialize failure
aniketpalu Aug 5, 2026
7909fc4
fix: Default run_async to False and cover version forwarding
aniketpalu Aug 5, 2026
b8aa6e7
Merge branch 'master' into feat/remote-materialize-v2
aniketpalu Aug 6, 2026
a2810c8
fix(server): Use dedicated executor for async materialize
aniketpalu Aug 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
299 changes: 251 additions & 48 deletions sdk/python/feast/feature_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import time
import traceback
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager
from datetime import datetime
from importlib import resources as importlib_resources
Expand All @@ -31,6 +32,7 @@
from fastapi import (
Depends,
FastAPI,
Query,
Request,
Response,
WebSocket,
Expand All @@ -41,7 +43,7 @@
from fastapi.logger import logger
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, field_validator
from pydantic import BaseModel, Field, field_validator

import feast
from feast import metrics as feast_metrics
Expand All @@ -54,6 +56,7 @@
)
from feast.feast_object import FeastObject
from feast.feature_server_utils import convert_response_to_dict
from feast.feature_view import FeatureViewState
from feast.feature_view_utils import get_feature_view_from_feature_store
from feast.filter_models import ComparisonFilter, CompoundFilter
from feast.permissions.action import WRITE, AuthzedAction
Expand Down Expand Up @@ -94,12 +97,26 @@ class MaterializeRequest(BaseModel):
feature_views: Optional[List[str]] = None
disable_event_timestamp: bool = False
full_feature_names: bool = False
version: Optional[str] = Field(
None,
description=(
"Optional version to materialize (e.g. 'v2'). Requires feature_views "
"with exactly one entry and registry.enable_online_feature_view_versioning."
),
)


class MaterializeIncrementalRequest(BaseModel):
end_ts: str
feature_views: Optional[List[str]] = None
full_feature_names: bool = False
version: Optional[str] = Field(
None,
description=(
"Optional version to materialize (e.g. 'v2'). Requires feature_views "
"with exactly one entry and registry.enable_online_feature_view_versioning."
),
)


class GetOnlineFeaturesRequest(BaseModel):
Comment on lines 97 to 122

@jyejare jyejare Jul 29, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] Missing version parameter documentation and validation

The version parameter was added to both MaterializeRequest and MaterializeIncrementalRequest but there's no documentation about what values are valid or how it affects the materialization behavior.

Suggested:

Suggested change
feature_views: Optional[List[str]] = None
disable_event_timestamp: bool = False
full_feature_names: bool = False
version: Optional[str] = None
class MaterializeIncrementalRequest(BaseModel):
end_ts: str
feature_views: Optional[List[str]] = None
full_feature_names: bool = False
version: Optional[str] = None
class GetOnlineFeaturesRequest(BaseModel):
version: Optional[str] = Field(
None,
description="Optional version to materialize (e.g., 'v2'). Requires feature_views with exactly one entry."
)

@aniketpalu aniketpalu Jul 31, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

Expand Down Expand Up @@ -333,6 +350,128 @@ async def load_static_artifacts(app: FastAPI, store):
logger.warning(f"Failed to load static artifacts: {e}")


def _authorize_materialize_views(
store: "feast.FeatureStore",
feature_view_names: Optional[List[str]],
version: Optional[str] = None,
) -> List[str]:
"""Resolve + authorize feature views for materialization.

Returns the resolved list of FV names (all eligible FVs when
feature_view_names is None).
"""
parsed_version = store._validate_materialize_version(version, feature_view_names)
feature_views_to_materialize = store._get_feature_views_to_materialize(
feature_view_names, version=parsed_version
)
for fv in feature_views_to_materialize:
assert_permissions(
resource=fv,
actions=[AuthzedAction.WRITE_ONLINE],
)
return [fv.name for fv in feature_views_to_materialize]


def _check_already_materializing(
store: "feast.FeatureStore",
fv_names: List[str],
) -> Optional[JSONResponse]:
"""Return a 409 JSONResponse if any requested FV is already MATERIALIZING."""
conflicting: List[str] = []
for fv_name in fv_names:
try:
fv = store.registry.get_feature_view(
fv_name, store.project, allow_cache=False
)
if getattr(fv, "state", None) == FeatureViewState.MATERIALIZING:
conflicting.append(fv_name)
except (FeatureViewNotFoundException, KeyError):
pass
except Exception as e:
logger.warning(
f"Unexpected error checking MATERIALIZING state for {fv_name}: {e}"
)
if conflicting:
return JSONResponse(
status_code=409,
content={
"error": (
f"Cannot start async materialization — the following feature "
f"views are already in MATERIALIZING state: {conflicting}. "
f"Use ?force=true to override."
),
"feature_views": conflicting,
},
)
return None


def _update_fv_state(
Comment on lines +397 to +409

@jyejare jyejare Jul 29, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Warning] Silent exception handling could hide real errors

The _reset_stuck_materializing_to_generated and _check_already_materializing functions silently ignore all exceptions when accessing feature views. This could hide legitimate errors like registry corruption or network issues, making debugging difficult.

Suggested:

Suggested change
content={
"error": (
f"Cannot start async materialization — the following feature "
f"views are already in MATERIALIZING state: {conflicting}. "
f"Use ?force=true to override."
),
"feature_views": conflicting,
},
)
return None
def _update_fv_state(
except (FeatureViewNotFoundException, KeyError):
# Expected when FV doesn't exist
pass
except Exception as e:
logger.warning(f"Unexpected error checking state for {fv_name}: {e}")
pass

@aniketpalu aniketpalu Jul 31, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Accepted

store: "feast.FeatureStore",
fv_names: List[str],
state: FeatureViewState,
) -> None:
"""Set FV state in the registry for each named feature view."""
for fv_name in fv_names:
try:
fv = store.registry.get_feature_view(
fv_name, store.project, allow_cache=False
)
fv.state = state
store.registry.apply_feature_view(fv, store.project)
except (FeatureViewNotFoundException, KeyError):
logger.warning(f"Feature view {fv_name} not found; skip state={state}")
except Exception as e:
logger.warning(f"Failed to set state={state} for {fv_name}: {e}")


def _reset_stuck_materializing_to_generated(
store: "feast.FeatureStore",
fv_names: List[str],
) -> None:
"""Reset FVs currently in MATERIALIZING to GENERATED (force override)."""
stuck: List[str] = []
for fv_name in fv_names:
try:
fv = store.registry.get_feature_view(
fv_name, store.project, allow_cache=False
)
if getattr(fv, "state", None) == FeatureViewState.MATERIALIZING:
stuck.append(fv_name)
except (FeatureViewNotFoundException, KeyError):
pass
except Exception as e:
logger.warning(f"Unexpected error while force-resetting {fv_name}: {e}")
if stuck:
_update_fv_state(store, stuck, FeatureViewState.GENERATED)
logger.info(
"Force reset MATERIALIZING → GENERATED for feature views: %s", stuck
)


def _parse_materialize_timestamps(
request: "MaterializeRequest",
) -> tuple:
"""Parse and validate start/end timestamps from a MaterializeRequest."""
if request.disable_event_timestamp:
now = datetime.now()
return datetime(1970, 1, 1), now

if not request.start_ts or not request.end_ts:
raise ValueError(
"start_ts and end_ts are required when disable_event_timestamp is False"
)
try:
start_date = utils.make_tzaware(parser.parse(request.start_ts))
end_date = utils.make_tzaware(parser.parse(request.end_ts))
except (ValueError, TypeError) as e:
raise ValueError(f"Invalid timestamp format: {e}") from e

if start_date >= end_date:
raise ValueError(f"start_ts ({start_date}) must be before end_ts ({end_date})")
return start_date, end_date


def get_app(
store: "feast.FeatureStore",
registry_ttl_sec: int = DEFAULT_FEATURE_SERVER_REGISTRY_TTL,
Expand Down Expand Up @@ -412,6 +551,22 @@ def get_app(
else:
logger.debug("Offline write batching is DISABLED")

# Dedicated pool for async materialize so long Spark/offline waits do not
# starve the default executor used by online serving and run_in_threadpool.
_mat_workers_raw = os.environ.get("FEAST_MATERIALIZE_MAX_WORKERS", "2")
try:
materialize_max_workers = max(1, int(_mat_workers_raw))
except ValueError:
logger.warning(
"Invalid FEAST_MATERIALIZE_MAX_WORKERS=%r; using default 2",
_mat_workers_raw,
)
materialize_max_workers = 2
materialize_executor = ThreadPoolExecutor(
max_workers=materialize_max_workers,
thread_name_prefix="feast-materialize",
)

def stop_refresh():
nonlocal shutting_down
shutting_down = True
Expand Down Expand Up @@ -445,6 +600,9 @@ async def lifespan(app: FastAPI):
stop_refresh()
if offline_batcher is not None:
offline_batcher.shutdown()
# wait=False: do not block process exit on in-flight materialize
# (same fire-and-forget contract as returning 202 mid-job).
materialize_executor.shutdown(wait=False)
await store.close()

app = FastAPI(lifespan=lifespan)
Expand Down Expand Up @@ -798,70 +956,115 @@ async def chat_ui():
return Response(content=content, media_type="text/html")

@app.post("/materialize", dependencies=[Depends(inject_user_details)])
async def materialize(request: MaterializeRequest) -> None:
async def materialize(
request: MaterializeRequest,
async_mode: bool = Query(False, alias="async"),
force: bool = Query(False),
):
with feast_metrics.track_request_latency("/materialize"):
if request.feature_views:
for feature_view in request.feature_views:
resource = await _get_feast_object(feature_view, True)
assert_permissions(
resource=resource,
actions=[AuthzedAction.WRITE_ONLINE],
)
else:
feature_views_to_materialize = store._get_feature_views_to_materialize(
None
)
for fv in feature_views_to_materialize:
assert_permissions(
resource=fv,
actions=[AuthzedAction.WRITE_ONLINE],
)
fv_names = _authorize_materialize_views(
store, request.feature_views, version=request.version
)
start_date, end_date = _parse_materialize_timestamps(request)

if request.disable_event_timestamp:
now = datetime.now()
start_date = datetime(1970, 1, 1)
end_date = now
else:
if not request.start_ts or not request.end_ts:
raise ValueError(
"start_ts and end_ts are required when disable_event_timestamp is False"
)
start_date = utils.make_tzaware(parser.parse(request.start_ts))
end_date = utils.make_tzaware(parser.parse(request.end_ts))
if async_mode:
if force:
_reset_stuck_materializing_to_generated(store, fv_names)
else:
conflict = _check_already_materializing(store, fv_names)
if conflict:
return conflict

# Reserve MATERIALIZING before 202 so concurrent requests hit 409.
# store.materialize() treats already-MATERIALIZING as a no-op.
_update_fv_state(store, fv_names, FeatureViewState.MATERIALIZING)

def _run_materialize():
try:
store.materialize(
start_date,
end_date,
fv_names,
disable_event_timestamp=request.disable_event_timestamp,
full_feature_names=request.full_feature_names,
version=request.version,
)
except Exception as e:
logger.error(
f"Async materialization failed for {fv_names}: {e}",
exc_info=True,
)
_reset_stuck_materializing_to_generated(store, fv_names)

loop = asyncio.get_running_loop()
loop.run_in_executor(materialize_executor, _run_materialize)

return JSONResponse(
status_code=202,
content={"status": "accepted", "feature_views": fv_names},
)

await run_in_threadpool(
store.materialize,
start_date,
end_date,
request.feature_views,
fv_names,
disable_event_timestamp=request.disable_event_timestamp,
full_feature_names=request.full_feature_names,
version=request.version,
)

Comment on lines 998 to 1016

@jyejare jyejare Jul 29, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Nitpick] Inconsistent parameter passing in sync mode

In the synchronous code path for materialize, the version parameter is passed but the feature_views parameter uses the original request.feature_views instead of the resolved fv_names list.

Suggested:

Suggested change
await run_in_threadpool(
store.materialize,
start_date,
end_date,
fv_names, # Use resolved names for consistency
disable_event_timestamp=request.disable_event_timestamp,
full_feature_names=request.full_feature_names,
version=request.version, # Don't forget version parameter
)

@aniketpalu aniketpalu Jul 31, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sync path now passes resolved fv_names and version=request.version (aligned with async).

@app.post("/materialize-incremental", dependencies=[Depends(inject_user_details)])
async def materialize_incremental(request: MaterializeIncrementalRequest) -> None:
async def materialize_incremental(
request: MaterializeIncrementalRequest,
async_mode: bool = Query(False, alias="async"),
force: bool = Query(False),
):
with feast_metrics.track_request_latency("/materialize-incremental"):
if request.feature_views:
for feature_view in request.feature_views:
resource = await _get_feast_object(feature_view, True)
assert_permissions(
resource=resource,
actions=[AuthzedAction.WRITE_ONLINE],
)
else:
feature_views_to_materialize = store._get_feature_views_to_materialize(
None
fv_names = _authorize_materialize_views(
store, request.feature_views, version=request.version
)
end_date = utils.make_tzaware(parser.parse(request.end_ts))

if async_mode:
if force:
_reset_stuck_materializing_to_generated(store, fv_names)
else:
conflict = _check_already_materializing(store, fv_names)
if conflict:
return conflict

_update_fv_state(store, fv_names, FeatureViewState.MATERIALIZING)

def _run_materialize_incremental():
try:
store.materialize_incremental(
end_date,
fv_names,
full_feature_names=request.full_feature_names,
version=request.version,
)
except Exception as e:
logger.error(
f"Async materialize-incremental failed for {fv_names}: {e}",
exc_info=True,
)
_reset_stuck_materializing_to_generated(store, fv_names)

loop = asyncio.get_running_loop()
loop.run_in_executor(materialize_executor, _run_materialize_incremental)

return JSONResponse(
status_code=202,
content={"status": "accepted", "feature_views": fv_names},
)
for fv in feature_views_to_materialize:
assert_permissions(
resource=fv,
actions=[AuthzedAction.WRITE_ONLINE],
)

await run_in_threadpool(
store.materialize_incremental,
utils.make_tzaware(parser.parse(request.end_ts)),
request.feature_views,
end_date,
fv_names,
full_feature_names=request.full_feature_names,
version=request.version,
)

@app.exception_handler(Exception)
Expand Down
Loading
Loading