Skip to content

Commit 4d8abb2

Browse files
committed
Use drop_duplicates() instead of groupby (about 1.5~2x faster)
Signed-off-by: rightx2 <rightx2@gmail.com>
1 parent 99ee2ce commit 4d8abb2

1 file changed

Lines changed: 6 additions & 6 deletions

File tree

  • sdk/python/feast/infra/offline_stores

sdk/python/feast/infra/offline_stores/file.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,9 @@ def evaluate_historical_retrieval():
153153
]
154154

155155
df_to_join.sort_values(by=right_entity_key_sort_columns, inplace=True)
156-
df_to_join = df_to_join.groupby(by=right_entity_key_columns).last()
157-
df_to_join.reset_index(inplace=True)
156+
df_to_join.drop_duplicates(
157+
right_entity_key_sort_columns, keep="last", ignore_index=True, inplace=True
158+
)
158159

159160
# Select only the columns we need to join from the feature dataframe
160161
df_to_join = df_to_join[right_entity_key_columns + feature_names]
@@ -231,10 +232,9 @@ def pull_latest_from_table_or_query(
231232
(source_df[event_timestamp_column] >= start_date)
232233
& (source_df[event_timestamp_column] < end_date)
233234
]
234-
last_values_df = filtered_df.groupby(by=join_key_columns).last()
235-
236-
# make driver_id a normal column again
237-
last_values_df.reset_index(inplace=True)
235+
last_values_df = filtered_df.drop_duplicates(
236+
join_key_columns, keep="last", ignore_index=True
237+
)
238238

239239
columns_to_extract = set(join_key_columns + feature_name_columns + ts_columns)
240240
table = pyarrow.Table.from_pandas(last_values_df[columns_to_extract])

0 commit comments

Comments
 (0)