Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 26 additions & 6 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from pydantic import Field

from pyiceberg.encryption.kms import KeyManagementClient
from pyiceberg.exceptions import CommitFailedException, ValidationException
from pyiceberg.exceptions import CommitFailedException, CommitStateUnknownException, ValidationException
from pyiceberg.expressions import AlwaysFalse, AlwaysTrue, And, BooleanExpression, EqualTo, IsNull, Or, Reference
from pyiceberg.expressions.visitors import (
ResidualEvaluator,
Expand Down Expand Up @@ -1151,6 +1151,12 @@ def commit_transaction(self) -> Table:
except CommitFailedException:
elapsed_ms = (time.monotonic() - start_time) * 1000
if attempt == num_retries or not self._snapshot_producers or elapsed_ms >= total_timeout_ms:
# The catalog can apply a commit and still report failure, for example
# when the response is lost. Check before cleaning up, since cleaning
# up a snapshot that landed would leave the table unreadable.
if self._snapshot_producers and self._attempt_landed():
self._cleanup_uncommitted_manifests()
break
raise

wait = min(min_wait_ms * (2**attempt), max_wait_ms)
Expand All @@ -1163,11 +1169,7 @@ def commit_transaction(self) -> Table:
)
time.sleep((wait + jitter) / 1000.0)

self._table.refresh()
if all(
self._table.metadata.snapshot_by_id(producer._snapshot_id) is not None
for producer in self._snapshot_producers
):
if self._attempt_landed():
# A previous attempt actually landed even though it was reported as
# failed (for example a lost response that the transport layer retried).
# The snapshot id is stable across attempts, so finding it in the
Expand Down Expand Up @@ -1220,6 +1222,24 @@ def commit_transaction(self) -> Table:

return self._table

def _attempt_landed(self) -> bool:
"""Refresh the table and return whether a snapshot written by this transaction is present.

Raises:
CommitStateUnknownException: If the table cannot be refreshed, the outcome is unknown and
the files written for the attempt must not be deleted.
"""
try:
self._table.refresh()
except Exception as e:
raise CommitStateUnknownException(
f"Commit failed and the table could not be refreshed to check whether it landed: {e}"
) from e

return all(
self._table.metadata.snapshot_by_id(producer._snapshot_id) is not None for producer in self._snapshot_producers
)

def _cleanup_uncommitted_manifests(self) -> None:
"""Clean up manifests from failed retry attempts after a successful commit."""
for producer in self._snapshot_producers:
Expand Down
75 changes: 74 additions & 1 deletion tests/table/test_commit_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from pyiceberg.catalog import Catalog
from pyiceberg.exceptions import CommitFailedException, CommitStateUnknownException, ValidationException
from pyiceberg.schema import Schema
from pyiceberg.table import TableProperties, Transaction
from pyiceberg.table import Table, TableProperties, Transaction
from pyiceberg.table.snapshots import IsolationLevel, Operation
from pyiceberg.types import LongType, NestedField, StringType

Expand Down Expand Up @@ -1292,6 +1292,79 @@ def test_negative_wait_properties_do_not_mask_commit_failure(catalog: Catalog) -
assert len(result) == 6


def _commit_outcomes(catalog: Catalog, *outcomes: str) -> Any:
"""Simulate a commit failing before ("conflict") or after ("lost") the catalog applied it."""
real_commit = catalog.commit_table
remaining = iter(outcomes)

def commit(*args: Any, **kwargs: Any) -> Any:
outcome = next(remaining, "ok")
if outcome == "conflict":
raise CommitFailedException("concurrent update")
result = real_commit(*args, **kwargs)
if outcome == "lost":
raise CommitFailedException("response lost after the commit landed")
return result

return patch.object(catalog, "commit_table", side_effect=commit)


def _assert_single_readable_snapshot(catalog: Catalog, identifier: str, expected: list[dict[str, Any]]) -> None:
table = catalog.load_table(identifier)
assert len(table.snapshots()) == 1
snapshot = table.current_snapshot()
assert snapshot is not None
assert table.io.new_input(snapshot.manifest_list).exists()
assert table.scan().to_arrow().to_pylist() == expected


def test_lost_response_on_the_last_attempt_keeps_the_landed_snapshot(catalog: Catalog) -> None:
"""A commit that lands but reports failure on the last attempt is not cleaned up."""
catalog.create_namespace("default")
table = catalog.create_table(
"default.lost_response_last_attempt",
schema=_test_schema(),
properties={TableProperties.COMMIT_NUM_RETRIES: "0"},
)

import pyarrow as pa

with _commit_outcomes(catalog, "lost"):
table.append(pa.table({"x": [1, 2, 3]}))

_assert_single_readable_snapshot(catalog, "default.lost_response_last_attempt", [{"x": 1}, {"x": 2}, {"x": 3}])


def test_lost_response_on_the_final_retry_keeps_the_landed_snapshot(catalog: Catalog) -> None:
"""A landed retry is detected even when the retry budget is exhausted."""
catalog.create_namespace("default")
table = catalog.create_table(
"default.lost_response_final_retry",
schema=_test_schema(),
properties={TableProperties.COMMIT_NUM_RETRIES: "1"},
)

import pyarrow as pa

with _commit_outcomes(catalog, "conflict", "lost"):
table.append(pa.table({"x": [1, 2, 3]}))

_assert_single_readable_snapshot(catalog, "default.lost_response_final_retry", [{"x": 1}, {"x": 2}, {"x": 3}])


def test_unreadable_commit_outcome_raises_commit_state_unknown(catalog: Catalog) -> None:
"""When the table cannot be refreshed, the outcome is unknown and no file is deleted."""
catalog.create_namespace("default")
table = catalog.create_table("default.unknown_commit_outcome", schema=_test_schema())

import pyarrow as pa

with _commit_outcomes(catalog, "lost"):
with patch.object(Table, "refresh", side_effect=RuntimeError("catalog unavailable")):
with pytest.raises(CommitStateUnknownException):
table.append(pa.table({"x": [1, 2, 3]}))


def test_transaction_writing_two_branches_detects_concurrent_change_to_either(catalog: Catalog) -> None:
"""A transaction writing two branches must assert the base snapshot of both refs.

Expand Down
Loading