-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Expand file tree
/
Copy pathutils.py
More file actions
83 lines (70 loc) · 2.9 KB
/
Copy pathutils.py
File metadata and controls
83 lines (70 loc) · 2.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
from datetime import datetime
from typing import Any, Mapping, Optional, Sequence
from feast.data_source import DataSource
from feast.infra.compute_engines.dag.context import ColumnInfo, ExecutionContext
from feast.infra.offline_stores.offline_store import RetrievalJob
from feast.infra.offline_stores.offline_utils import (
DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL,
infer_event_timestamp_from_entity_df,
)
ENTITY_TS_ALIAS = "__entity_event_timestamp"
ENTITY_ROW_ID = "__feast_entity_row_id"
def infer_entity_timestamp_column(entity_schema: Mapping[str, Any]) -> str:
"""Resolve the entity timestamp column used for point-in-time joins."""
if ENTITY_TS_ALIAS in entity_schema:
return ENTITY_TS_ALIAS
return infer_event_timestamp_from_entity_df(dict(entity_schema))
def find_entity_timestamp_column(columns: Sequence[str]) -> Optional[str]:
"""Find the timestamp column in an entity DataFrame schema, if present."""
if ENTITY_TS_ALIAS in columns:
return ENTITY_TS_ALIAS
if DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL in columns:
return DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL
return None
def create_offline_store_retrieval_job(
data_source: DataSource,
column_info: ColumnInfo,
context: ExecutionContext,
start_time: Optional[datetime] = None,
end_time: Optional[datetime] = None,
) -> RetrievalJob:
"""
Create a retrieval job for the offline store.
Args:
data_source: The data source to pull from.
column_info: Column information containing join keys, feature columns, and timestamps.
context:
start_time:
end_time:
Returns:
"""
offline_store = context.offline_store
pull_latest = context.repo_config.materialization_config.pull_latest_features
if pull_latest:
if not start_time or not end_time:
raise ValueError(
"start_time and end_time must be provided when pull_latest_features is True"
)
retrieval_job = offline_store.pull_latest_from_table_or_query(
config=context.repo_config,
data_source=data_source,
join_key_columns=column_info.join_keys,
feature_name_columns=column_info.feature_cols,
timestamp_field=column_info.ts_col,
created_timestamp_column=column_info.created_ts_col,
start_date=start_time,
end_date=end_time,
)
else:
# 📥 Reuse Feast's robust query resolver
retrieval_job = offline_store.pull_all_from_table_or_query(
config=context.repo_config,
data_source=data_source,
join_key_columns=column_info.join_keys,
feature_name_columns=column_info.feature_cols,
timestamp_field=column_info.ts_col,
created_timestamp_column=column_info.created_ts_col,
start_date=start_time,
end_date=end_time,
)
return retrieval_job