Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions pyiceberg/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -675,6 +675,10 @@ def data_file(self) -> DataFile:
def data_file(self, value: DataFile) -> None:
self._data[4] = value

def is_live(self) -> bool:
"""Return whether the entry is live (added or existing) in the snapshot."""
return self.status in (ManifestEntryStatus.ADDED, ManifestEntryStatus.EXISTING)


PARTITION_FIELD_SUMMARY_TYPE = StructType(
NestedField(509, "contains_null", BooleanType(), required=True),
Expand Down Expand Up @@ -933,15 +937,15 @@ def fetch_manifest_entry(
next_row_id = self.first_row_id if self.content == ManifestContent.DATA else None

for entry in reader:
if discard_deleted and entry.status == ManifestEntryStatus.DELETED:
if discard_deleted and not entry.is_live():
continue
_inherit_from_manifest(entry, self)

# Inherit first row IDs, which are only valid when a data manifest has a first_row_id
data_file = entry.data_file
if next_row_id is None:
data_file.first_row_id = None
elif entry.status != ManifestEntryStatus.DELETED and data_file.first_row_id is None:
elif entry.is_live() and data_file.first_row_id is None:
data_file.first_row_id = next_row_id
next_row_id += data_file.record_count

Expand Down Expand Up @@ -1254,7 +1258,7 @@ def add_entry(self, entry: ManifestEntry) -> ManifestWriter:
self._partitions.append(entry.data_file.partition)

if (
(entry.status == ManifestEntryStatus.ADDED or entry.status == ManifestEntryStatus.EXISTING)
entry.is_live()
and entry.sequence_number is not None
and (self._min_sequence_number is None or entry.sequence_number < self._min_sequence_number)
):
Expand Down
2 changes: 1 addition & 1 deletion pyiceberg/table/update/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -973,7 +973,7 @@ def _create_manifest(self, spec_id: int, manifest_bin: list[ManifestFile]) -> Ma
elif entry.status == ManifestEntryStatus.ADDED and entry.snapshot_id == self._snapshot_producer.snapshot_id:
# added entries from this snapshot are still added, otherwise they should be existing
writer.add(entry)
elif entry.status != ManifestEntryStatus.DELETED:
elif entry.is_live():
# add all non-deleted files from the old manifest as existing files
writer.existing(entry)

Expand Down
8 changes: 8 additions & 0 deletions tests/utils/test_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,14 @@ def test_fetch_manifest_entry_with_filter(generated_manifest_entry_file: str) ->
assert len(no_match) == 0


@pytest.mark.parametrize(
"status, expected",
[(ManifestEntryStatus.ADDED, True), (ManifestEntryStatus.EXISTING, True), (ManifestEntryStatus.DELETED, False)],
)
def test_manifest_entry_is_live(status: ManifestEntryStatus, expected: bool) -> None:
assert ManifestEntry.from_args(status=status).is_live() is expected


def test_fetch_manifest_entry_inherits_first_row_id(tmp_path: Path) -> None:
"""Data files inherit first row IDs from data manifests in file order, and have them cleared otherwise."""
io = PyArrowFileIO()
Expand Down
Loading