Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion pyiceberg/table/update/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,12 @@ def _(update: RemoveSnapshotsUpdate, base_metadata: TableMetadata, context: _Tab
for statistics_file in base_metadata.statistics
if statistics_file.snapshot_id in update.snapshot_ids
)
updates = itertools.chain(remove_ref_updates, remove_statistics_updates)
remove_partition_statistics_updates = (
RemovePartitionStatisticsUpdate(snapshot_id=partition_statistics_file.snapshot_id)
for partition_statistics_file in base_metadata.partition_statistics
if partition_statistics_file.snapshot_id in update.snapshot_ids
)
updates = itertools.chain(remove_ref_updates, remove_statistics_updates, remove_partition_statistics_updates)
new_metadata = base_metadata
for upd in updates:
new_metadata = _apply_table_update(upd, new_metadata, context)
Expand Down
29 changes: 28 additions & 1 deletion tests/table/test_expire_snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
import pytest

from pyiceberg.table import CommitTableResponse, Table
from pyiceberg.table.update import RemoveSnapshotsUpdate, update_table_metadata
from pyiceberg.table.statistics import PartitionStatisticsFile
from pyiceberg.table.update import RemoveSnapshotsUpdate, SetPartitionStatisticsUpdate, update_table_metadata
from pyiceberg.table.update.snapshot import ExpireSnapshots


Expand Down Expand Up @@ -316,3 +317,29 @@ def test_update_remove_snapshots_with_statistics(table_v2_with_statistics: Table
assert not any(stat.snapshot_id == REMOVE_SNAPSHOT for stat in new_metadata.statistics), (
"Statistics for removed snapshot should be gone"
)


def test_update_remove_snapshots_with_partition_statistics(table_v2: Table) -> None:
"""Removing a snapshot also removes its partition statistics, as Iceberg Java does."""
REMOVE_SNAPSHOT = 3051729675574597004
KEEP_SNAPSHOT = 3055729675574597004

metadata = update_table_metadata(
table_v2.metadata,
tuple(
SetPartitionStatisticsUpdate(
partition_statistics=PartitionStatisticsFile(
snapshot_id=snapshot_id,
statistics_path=f"s3://bucket/warehouse/partition-stats-{snapshot_id}.parquet",
file_size_in_bytes=124,
)
)
for snapshot_id in (REMOVE_SNAPSHOT, KEEP_SNAPSHOT)
),
)
assert len(metadata.partition_statistics) == 2

new_metadata = update_table_metadata(metadata, (RemoveSnapshotsUpdate(snapshot_ids=[REMOVE_SNAPSHOT]),))

assert [snapshot.snapshot_id for snapshot in new_metadata.snapshots] == [KEEP_SNAPSHOT]
assert [stat.snapshot_id for stat in new_metadata.partition_statistics] == [KEEP_SNAPSHOT]
Loading