Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 24 additions & 13 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -562,7 +562,7 @@ def append(
except ModuleNotFoundError as e:
raise ModuleNotFoundError("For writes PyArrow needs to be installed") from e

from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _dataframe_to_data_files
from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible

if not isinstance(df, (pa.Table, pa.RecordBatchReader)):
raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}")
Expand All @@ -576,17 +576,24 @@ def append(
)

with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files:
# For pa.Table we can short-circuit empty inputs cheaply. For a
# RecordBatchReader the stream is consumed lazily by
# _dataframe_to_data_files and an empty reader simply yields zero
# data files (the snapshot is still committed for symmetry with the
# pa.Table case where empty inputs also produce a snapshot).
if isinstance(df, pa.RecordBatchReader) or df.shape[0] > 0:
data_files = _dataframe_to_data_files(
table_metadata=self.table_metadata, write_uuid=append_files.commit_uuid, df=df, io=self._table.io
)
for data_file in data_files:
append_files.append_data_file(data_file)
self._write_append_data(df, append_files)

def _write_append_data(self, df: pa.Table | pa.RecordBatchReader, append_files: _FastAppendFiles) -> None:
import pyarrow as pa

from pyiceberg.io.pyarrow import _dataframe_to_data_files

# For pa.Table we can short-circuit empty inputs cheaply. For a
# RecordBatchReader the stream is consumed lazily by
# _dataframe_to_data_files and an empty reader simply yields zero
# data files (the snapshot is still committed for symmetry with the
# pa.Table case where empty inputs also produce a snapshot).
if isinstance(df, pa.RecordBatchReader) or df.shape[0] > 0:
data_files = _dataframe_to_data_files(
table_metadata=self.table_metadata, write_uuid=append_files.commit_uuid, df=df, io=self._table.io
)
for data_file in data_files:
append_files.append_data_file(data_file)

def dynamic_partition_overwrite(
self, df: pa.Table, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH
Expand Down Expand Up @@ -1014,7 +1021,11 @@ def upsert(
if when_not_matched_insert_all:
insert_row_cnt = len(rows_to_insert)
if rows_to_insert:
self.append(rows_to_insert, branch=branch, snapshot_properties=snapshot_properties)
with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files:
append_files.validate_no_conflicting_data(
upsert_util.create_match_filter(rows_to_insert, join_cols), case_sensitive
)
self._write_append_data(rows_to_insert, append_files)

return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt)

Expand Down
35 changes: 34 additions & 1 deletion pyiceberg/table/update/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -690,8 +690,41 @@ def _refresh_for_retry(self) -> None:


class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]):
_conflict_detection_filter: BooleanExpression | None = None
_conflict_detection_case_sensitive: bool = True

def validate_no_conflicting_data(self, data_filter: BooleanExpression, case_sensitive: bool = True) -> None:
"""Fail a retry when a concurrent commit added data matching the filter."""
self._conflict_detection_filter = data_filter
self._conflict_detection_case_sensitive = case_sensitive

def _validate_concurrency(self) -> None:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This check runs regardless of the table's isolation level. The update half of the upsert only runs _validate_added_data_files under serializable (see the base _validate_concurrency), so with write.update.isolation-level=snapshot the two halves of a single upsert behave differently: the update half tolerates concurrently added files, while the insert half rejects them. This matches Java, where MERGE only validates conflicting data under serializable isolation. Should we check isolation_level(Operation.OVERWRITE) here so both halves follow the same setting?

"""Skip validation; appends do not conflict with other operations."""
"""Skip validation for plain appends.

An append that declares a conflict detection filter fails when a concurrent commit added
data matching it, instead of replaying the append on retry.
"""
if self._conflict_detection_filter is None or self._commit_window is None or self._commit_window.is_empty():
return

from pyiceberg.table.snapshots import IsolationLevel
from pyiceberg.table.update.validate import _validate_added_data_files

table = self._transaction._table
if table.metadata.isolation_level(Operation.OVERWRITE) != IsolationLevel.SERIALIZABLE:
return

catalog_head = self._commit_window.head
if catalog_head is None:
return

_validate_added_data_files(
table,
catalog_head,
self._conflict_detection_filter,
self._commit_window.base,
self._conflict_detection_case_sensitive,
)

def _existing_manifests(self) -> list[ManifestFile]:
"""To determine if there are any existing manifest files.
Expand Down
59 changes: 56 additions & 3 deletions tests/table/test_upsert.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,17 @@
from pyarrow import Table as pa_table

from pyiceberg.catalog import Catalog
from pyiceberg.exceptions import NoSuchTableError
from pyiceberg.exceptions import NoSuchTableError, ValidationException
from pyiceberg.expressions import AlwaysTrue, And, EqualTo, Reference
from pyiceberg.expressions.literals import LongLiteral
from pyiceberg.io.pyarrow import schema_to_pyarrow
from pyiceberg.partitioning import PartitionField, PartitionSpec
from pyiceberg.schema import Schema
from pyiceberg.table import Table, UpsertResult
from pyiceberg.table import Table, TableProperties, UpsertResult
from pyiceberg.table.snapshots import Operation
from pyiceberg.table.upsert_util import create_match_filter, get_rows_to_update
from pyiceberg.transforms import DayTransform
from pyiceberg.types import IntegerType, NestedField, StringType, StructType, TimestampType
from pyiceberg.types import IntegerType, LongType, NestedField, StringType, StructType, TimestampType
from tests.catalog.test_base import InMemoryCatalog


Expand Down Expand Up @@ -427,6 +427,59 @@ def test_upsert_into_empty_table(catalog: Catalog) -> None:
assert upd.rows_inserted == 4


def test_upsert_retry_with_a_concurrently_inserted_key(catalog: Catalog) -> None:
identifier = "default.test_upsert_retry_with_a_concurrently_inserted_key"
_drop_table(catalog, identifier)

schema = Schema(
NestedField(1, "k", LongType(), required=True),
NestedField(2, "v", StringType(), required=False),
identifier_field_ids=[1],
)
arrow_schema = schema_to_pyarrow(schema)
catalog.create_table(identifier, schema=schema).append(pa.Table.from_pylist([{"k": 1, "v": "a"}], schema=arrow_schema))

writer_a = catalog.load_table(identifier)
writer_b = catalog.load_table(identifier)
writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema))

with pytest.raises(ValidationException):
writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema))

assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [
{"k": 1, "v": "a"},
{"k": 2, "v": "b"},
]


def test_upsert_retry_under_snapshot_isolation_does_not_validate_inserted_keys(catalog: Catalog) -> None:
"""The insert half follows the update isolation level, which tolerates concurrently added rows."""
identifier = "default.test_upsert_retry_snapshot_isolation"
_drop_table(catalog, identifier)

schema = Schema(
NestedField(1, "k", LongType(), required=True),
NestedField(2, "v", StringType(), required=False),
identifier_field_ids=[1],
)
arrow_schema = schema_to_pyarrow(schema)
catalog.create_table(identifier, schema=schema, properties={TableProperties.WRITE_UPDATE_ISOLATION_LEVEL: "snapshot"}).append(
pa.Table.from_pylist([{"k": 1, "v": "a"}], schema=arrow_schema)
)

writer_a = catalog.load_table(identifier)
writer_b = catalog.load_table(identifier)
writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema))

writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema))

assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [
{"k": 1, "v": "a"},
{"k": 2, "v": "a"},
{"k": 2, "v": "b"},
]


def test_create_match_filter_single_condition() -> None:
"""
Test create_match_filter with a composite key where the source yields exactly one unique key.
Expand Down
Loading