Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions pyiceberg/table/upsert_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,18 @@ def has_duplicate_rows(df: pyarrow_table, join_cols: list[str]) -> bool:
return len(df.select(join_cols).group_by(join_cols).aggregate([([], "count_all")]).filter(pc.field("count_all") > 1)) > 0


def _struct_values_differ(source_values: pa.ChunkedArray, target_values: pa.ChunkedArray) -> pa.ChunkedArray:
"""Compare two struct columns field by field, as PyArrow cannot compare the structs themselves."""
# A null struct has null fields, so the struct itself decides those rows
changed = pc.not_equal(pc.is_null(source_values), pc.is_null(target_values))
for field in source_values.type:
changed = pc.or_(
changed,
_values_differ(pc.struct_field(source_values, field.name), pc.struct_field(target_values, field.name)),
)
return changed


def _values_differ(
source_values: pa.Array | pa.ChunkedArray, target_values: pa.Array | pa.ChunkedArray
) -> pa.Array | pa.ChunkedArray:
Expand All @@ -61,6 +73,14 @@ def _values_differ(
equal = pc.fill_null(pc.equal(source_values, target_values), False)
except (pa.ArrowNotImplementedError, pa.ArrowInvalid, pa.ArrowTypeError):
# PyArrow has no comparison kernel for nested types
if pa.types.is_struct(source_values.type) and source_values.type == target_values.type:
return _struct_values_differ(source_values, target_values)

# Different types: cast the source to the target type and compare again, e.g. a naive against
# a zoned timestamp, or a struct of string against the struct of large_string a scan returns
if source_values.type != target_values.type:
return _values_differ(source_values.cast(target_values.type), target_values)

source_list = source_values.to_pylist()
target_list = target_values.to_pylist()
return pa.array(
Expand Down
99 changes: 99 additions & 0 deletions tests/table/test_upsert.py
Original file line number Diff line number Diff line change
Expand Up @@ -955,3 +955,102 @@ def test_get_rows_to_update_compares_nested_values() -> None:
target = pa.table({"pk": [1, 2], "value": pa.array([{"a": 1}, {"a": 3}])})

assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [2], "value": [{"a": 2}]}


def test_get_rows_to_update_compares_columns_of_a_different_type() -> None:
"""A naive and a zoned timestamp of the same instant are not a change."""
target = pa.table({"id": pa.array([1, 2], pa.int32()), "ts": pa.array([1, 2], pa.timestamp("us", tz="UTC"))})
source = pa.table({"id": pa.array([1, 2], pa.int32()), "ts": pa.array([1, 9], pa.timestamp("us"))})

assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2]


def test_get_rows_to_update_compares_a_struct_that_is_null() -> None:
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field("nested", pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())])),
]
)
target = pa.Table.from_pylist(
[
{"id": 1, "nested": None},
{"id": 2, "nested": None},
{"id": 3, "nested": {"a": 1, "b": "x"}},
{"id": 4, "nested": {"a": 1, "b": "x"}},
{"id": 5, "nested": {"a": 1, "b": None}},
{"id": 6, "nested": {"a": None, "b": None}},
{"id": 7, "nested": None},
],
schema=schema,
)
source = pa.Table.from_pylist(
[
{"id": 1, "nested": None}, # unchanged
{"id": 2, "nested": {"a": 1, "b": "x"}}, # null to a struct
{"id": 3, "nested": None}, # a struct to null
{"id": 4, "nested": {"a": 1, "b": "y"}}, # one field differs
{"id": 5, "nested": {"a": 1, "b": None}}, # unchanged, with a null field
{"id": 6, "nested": None}, # a struct of null fields to null
{"id": 7, "nested": {"a": None, "b": None}}, # null to a struct of null fields
],
schema=schema,
)

assert get_rows_to_update(source, target, ["id"]).sort_by("id").column("id").to_pylist() == [2, 3, 4, 6, 7]


def test_get_rows_to_update_compares_a_struct_whose_fields_are_in_another_order() -> None:
source_type = pa.struct([pa.field("a", pa.int64()), pa.field("b", pa.string())])
target_type = pa.struct([pa.field("b", pa.string()), pa.field("a", pa.int64())])
source = pa.table({"id": pa.array([1, 2]), "nested": pa.array([{"a": 1, "b": "x"}, {"a": 2, "b": "y"}], type=source_type)})
target = pa.table({"id": pa.array([1, 2]), "nested": pa.array([{"a": 1, "b": "x"}, {"a": 9, "b": "y"}], type=target_type)})

assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2]


def test_get_rows_to_update_compares_a_nested_struct() -> None:
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field("outer", pa.struct([pa.field("inner", pa.struct([pa.field("value", pa.int32())]))])),
]
)
target = pa.Table.from_pylist(
[
{"id": 1, "outer": {"inner": {"value": 1}}},
{"id": 2, "outer": {"inner": {"value": 1}}},
{"id": 3, "outer": {"inner": None}},
{"id": 4, "outer": {"inner": {"value": 1}}},
],
schema=schema,
)
source = pa.Table.from_pylist(
[
{"id": 1, "outer": {"inner": {"value": 1}}}, # unchanged
{"id": 2, "outer": {"inner": {"value": 2}}}, # the innermost value differs
{"id": 3, "outer": {"inner": {"value": 1}}}, # a null inner struct to a struct
{"id": 4, "outer": {"inner": None}}, # a struct to a null inner struct
],
schema=schema,
)

assert get_rows_to_update(source, target, ["id"]).sort_by("id").column("id").to_pylist() == [2, 3, 4]


def test_get_rows_to_update_compares_a_struct_read_as_large_string() -> None:
"""A scan reads the strings of a struct as large_string, while a dataframe holds them as string."""
source = pa.table(
{
"id": pa.array([1, 2], pa.int32()),
"nested": pa.array([{"a": "x"}, {"a": "y"}], type=pa.struct([pa.field("a", pa.string())])),
}
)
target = pa.table(
{
"id": pa.array([1, 2], pa.int32()),
"nested": pa.array([{"a": "x"}, {"a": "z"}], type=pa.struct([pa.field("a", pa.large_string())])),
}
)

assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2]
Loading