Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1884,18 +1884,23 @@ def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[st
deletes_per_file: dict[str, list[ChunkedArray]] = {}
# DataFile equality uses only the path; DVs from different tasks may share a file.
unique_deletes: dict[_DeleteFileKey, DataFile] = {}
# Data files whose task has the delete file attached; a delete file can also list rows of other data files.
attached_paths: dict[_DeleteFileKey, set[str]] = {}
for task in tasks:
for delete_file in task.delete_files:
key = _DeleteFileKey(delete_file.file_path, delete_file.content_offset, delete_file.content_size_in_bytes)
unique_deletes.setdefault(key, delete_file)
attached_paths.setdefault(key, set()).add(task.file.file_path)
if len(unique_deletes) > 0:
executor = ExecutorFactory.get_or_create()
deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map(
lambda args: _read_deletes(*args),
[(io, delete_file) for delete_file in unique_deletes.values()],
)
for delete in deletes_per_files:
for key, delete in zip(unique_deletes, deletes_per_files, strict=True):
for file, arr in delete.items():
if file not in attached_paths[key]:
continue
if file in deletes_per_file:
deletes_per_file[file].append(arr)
else:
Expand Down
33 changes: 33 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -2253,6 +2253,39 @@ def test_read_all_delete_files_keeps_multiple_dv_content_ranges_for_same_path(tm
}


def test_read_all_delete_files_only_loads_positions_for_tasks_it_is_attached_to(tmp_path: Path) -> None:
"""A delete file that names rows of two data files only applies to the data files whose task has it attached."""
first_path, second_path = f"{tmp_path}/data-0.parquet", f"{tmp_path}/data-1.parquet"
old_path, new_path = f"{tmp_path}/old-deletes.parquet", f"{tmp_path}/new-deletes.parquet"
pq.write_table(pa.table({"file_path": [first_path, second_path], "pos": [0, 0]}), old_path)
pq.write_table(pa.table({"file_path": [first_path], "pos": [1]}), new_path)
old_deletes = DataFile.from_args(
content=DataFileContent.POSITION_DELETES, file_path=old_path, file_format=FileFormat.PARQUET, record_count=2
)
new_deletes = DataFile.from_args(
content=DataFileContent.POSITION_DELETES, file_path=new_path, file_format=FileFormat.PARQUET, record_count=1
)
first_data = DataFile.from_args(
content=DataFileContent.DATA, file_path=first_path, file_format=FileFormat.PARQUET, record_count=10
)
second_data = DataFile.from_args(
content=DataFileContent.DATA, file_path=second_path, file_format=FileFormat.PARQUET, record_count=10
)
# (delete files attached to the first task, delete files attached to the second task) -> expected positions
shapes = [
(set(), {old_deletes}, {second_path: [pa.chunked_array([[0]])]}),
({old_deletes}, {old_deletes}, {first_path: [pa.chunked_array([[0]])], second_path: [pa.chunked_array([[0]])]}),
# The first data file has a newer delete file too; the old one must not add its row 0 to it.
({new_deletes}, {old_deletes}, {first_path: [pa.chunked_array([[1]])], second_path: [pa.chunked_array([[0]])]}),
]
for first_task_deletes, second_task_deletes, expected in shapes:
tasks = [
FileScanTask(data_file=first_data, delete_files=first_task_deletes),
FileScanTask(data_file=second_data, delete_files=second_task_deletes),
]
assert _read_all_delete_files(PyArrowFileIO(), tasks) == expected


@pytest.mark.parametrize("selected_indices", [(0, 1, 2, 3, 4, 5), (0, 3, 5)])
@pytest.mark.parametrize("planning_mode", ["local", "rest"])
def test_read_all_delete_files_from_external_packed_deletion_vectors(
Expand Down
15 changes: 9 additions & 6 deletions tests/table/test_commit_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -544,11 +544,13 @@ def test_file_overwrite_validates_concurrent_file_delete(


def commit_partition_position_delete(
table: Table, partition: Record, file_path: str, pos: int, row: dict[str, Any] | None = None
table: Table, partition: Record, rows: list[tuple[str, int]], row: dict[str, Any] | None = None
) -> None:
"""PyIceberg cannot write delete files, so commit one without path bounds, as other engines write them.

If ``row`` is given, it is stored as the deleted row, which gives the delete file bounds on the data columns.
Each ``(file_path, pos)`` in ``rows`` goes into the same delete file, which may name several data files.
If ``row`` is given, it is stored as the deleted row of a single-row delete file, which gives the delete file
bounds on the data columns.
"""
import pyarrow as pa
import pyarrow.parquet as pq
Expand All @@ -571,7 +573,8 @@ def new_delete_manifest_writer(producer: _OverwriteFiles, spec: PartitionSpec) -
spec, producer.schema(), producer.new_manifest_output(), producer._snapshot_id, producer._compression
)

columns: dict[str, list[Any]] = {"file_path": [file_path], "pos": [pos]}
rows = sorted(rows) # the spec requires delete rows sorted by file_path, then pos
columns: dict[str, list[Any]] = {"file_path": [file_path for file_path, _ in rows], "pos": [pos for _, pos in rows]}
bounds = None
if row is not None:
columns["row"] = [row]
Expand All @@ -585,7 +588,7 @@ def new_delete_manifest_writer(producer: _OverwriteFiles, spec: PartitionSpec) -
file_path=delete_path,
file_format=FileFormat.PARQUET,
partition=partition,
record_count=1,
record_count=len(rows),
file_size_in_bytes=len(table.io.new_input(delete_path)),
lower_bounds=bounds,
upper_bounds=bounds,
Expand Down Expand Up @@ -628,7 +631,7 @@ def test_file_overwrite_validates_concurrent_position_delete_in_nan_partition(ca
overwrite.delete_data_file(file_to_replace)
overwrite.append_data_file(replacement_file)

commit_partition_position_delete(deleting_table, Record(float("nan")), file_to_replace.file_path, pos=0)
commit_partition_position_delete(deleting_table, Record(float("nan")), [(file_to_replace.file_path, 0)])

with pytest.raises(ValidationException, match="found new delete for replace data file"):
replacing_transaction.commit_transaction()
Expand All @@ -650,7 +653,7 @@ def test_partial_delete_validates_concurrent_delete_outside_the_filter(catalog:
deleting_table = catalog.load_table(identifier)

# The delete file stores the deleted row (2, 'b'), so its bounds can't match k = 1.
commit_partition_position_delete(table, Record(), data_file.file_path, pos=1, row={"k": 2, "v": "b"})
commit_partition_position_delete(table, Record(), [(data_file.file_path, 1)], row={"k": 2, "v": "b"})

with pytest.raises(ValidationException, match="found new delete for replace data file"):
deleting_table.delete("k = 1")
Expand Down
36 changes: 35 additions & 1 deletion tests/table/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -2470,6 +2470,40 @@ def test_overwrite_delete_data_file_on_bucket_partition(catalog: Catalog) -> Non
assert table.scan().to_arrow().num_rows == 0


def test_position_delete_respects_sequence_number_of_readded_file(catalog: Catalog) -> None:
"""A position delete that names two data files applies to a file only if the file is not newer than the delete."""
import pyarrow as pa

catalog.create_namespace("default")
schema = Schema(NestedField(1, "id", LongType(), required=False))
shapes = [
# The delete predates the re-registered file P, so it keeps P's row 0 but still removes Q's row 0.
(True, [*range(10), *range(101, 110)]),
# The delete is committed after P is re-registered, so it removes row 0 of both files.
(False, [*range(1, 10), *range(101, 110)]),
]
for index, (delete_before_readd, expected) in enumerate(shapes):
table = catalog.create_table(
f"default.readded_position_delete_{index}", schema=schema, properties={"format-version": "2"}
)
table.append(pa.table({"id": pa.array(range(10), pa.int64())}))
table.append(pa.table({"id": pa.array(range(100, 110), pa.int64())}))
p_path = next(task.file.file_path for task in table.scan(row_filter="id = 0").plan_files())
q_path = next(task.file.file_path for task in table.scan(row_filter="id = 100").plan_files())
rows = [(p_path, 0), (q_path, 0)]

if delete_before_readd:
commit_partition_position_delete(table, Record(), rows)
table.delete("id < 10") # removes P, the whole file
table.add_files([p_path]) # registers P again with a newer data sequence number
if not delete_before_readd:
commit_partition_position_delete(table, Record(), rows)

assert sorted(table.scan().to_arrow()["id"].to_pylist()) == expected
assert table.scan().count() == len(expected)
assert sorted(table.scan(row_filter="id < 10").to_arrow()["id"].to_pylist()) == [i for i in expected if i < 10]


def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None:
"""Rewriting a file applies all of its delete files, not only those whose bounds can match the delete filter."""
import pyarrow as pa
Expand All @@ -2481,7 +2515,7 @@ def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalo
data_file = next(iter(table.scan().plan_files())).file

# The delete file stores the deleted row (2, 'b'), so a scan for k = 1 skips it.
commit_partition_position_delete(table, Record(), data_file.file_path, pos=1, row={"k": 2, "v": "b"})
commit_partition_position_delete(table, Record(), [(data_file.file_path, 1)], row={"k": 2, "v": "b"})
assert [task.delete_files for task in table.scan(row_filter="k = 1").plan_files()] == [set()]

table.delete("k = 1")
Expand Down
Loading