Skip to content

Commit ed68cc3

Browse files
committed
Compare struct upsert columns field by field in PyArrow
Since #4334 the upsert comparison runs column by column in PyArrow, but struct columns still fall back to Python, a dict per row. Structs are now compared field by field, recursing into nested structs. A scan returns the strings of a struct as large_string while a dataframe holds string, so the two struct types differ in every real upsert. When PyArrow cannot compare two columns of different types, the source column is cast to the target type first, which lets those structs take the PyArrow path. Upserting 30 struct columns over 200k rows goes from 5.5s to 2.9s.
1 parent 9eb8395 commit ed68cc3

2 files changed

Lines changed: 118 additions & 0 deletions

File tree

‎pyiceberg/table/upsert_util.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,18 @@ def has_duplicate_rows(df: pyarrow_table, join_cols: list[str]) -> bool:
5353
return len(df.select(join_cols).group_by(join_cols).aggregate([([], "count_all")]).filter(pc.field("count_all") > 1)) > 0
5454

5555

56+
def _struct_values_differ(source_values: pa.ChunkedArray, target_values: pa.ChunkedArray) -> pa.ChunkedArray:
57+
"""Compare two struct columns field by field, as PyArrow cannot compare the structs themselves."""
58+
# A null struct has null fields, so the struct itself decides those rows
59+
changed = pc.not_equal(pc.is_null(source_values), pc.is_null(target_values))
60+
for field in source_values.type:
61+
changed = pc.or_(
62+
changed,
63+
_values_differ(pc.struct_field(source_values, field.name), pc.struct_field(target_values, field.name)),
64+
)
65+
return changed
66+
67+
5668
def _values_differ(
5769
source_values: pa.Array | pa.ChunkedArray, target_values: pa.Array | pa.ChunkedArray
5870
) -> pa.Array | pa.ChunkedArray:
@@ -61,6 +73,13 @@ def _values_differ(
6173
equal = pc.fill_null(pc.equal(source_values, target_values), False)
6274
except (pa.ArrowNotImplementedError, pa.ArrowInvalid, pa.ArrowTypeError):
6375
# PyArrow has no comparison kernel for nested types
76+
if pa.types.is_struct(source_values.type) and source_values.type == target_values.type:
77+
return _struct_values_differ(source_values, target_values)
78+
79+
# e.g. a struct of string against the struct of large_string a scan returns
80+
if source_values.type != target_values.type:
81+
return _values_differ(source_values.cast(target_values.type), target_values)
82+
6483
source_list = source_values.to_pylist()
6584
target_list = target_values.to_pylist()
6685
return pa.array(

‎tests/table/test_upsert.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -955,3 +955,102 @@ def test_get_rows_to_update_compares_nested_values() -> None:
955955
target = pa.table({"pk": [1, 2], "value": pa.array([{"a": 1}, {"a": 3}])})
956956

957957
assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [2], "value": [{"a": 2}]}
958+
959+
960+
def test_get_rows_to_update_compares_columns_of_a_different_type() -> None:
961+
"""A naive and a zoned timestamp of the same instant are not a change."""
962+
target = pa.table({"id": pa.array([1, 2], pa.int32()), "ts": pa.array([1, 2], pa.timestamp("us", tz="UTC"))})
963+
source = pa.table({"id": pa.array([1, 2], pa.int32()), "ts": pa.array([1, 9], pa.timestamp("us"))})
964+
965+
assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2]
966+
967+
968+
def test_get_rows_to_update_compares_a_struct_that_is_null() -> None:
969+
schema = pa.schema(
970+
[
971+
pa.field("id", pa.int32()),
972+
pa.field("nested", pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())])),
973+
]
974+
)
975+
target = pa.Table.from_pylist(
976+
[
977+
{"id": 1, "nested": None},
978+
{"id": 2, "nested": None},
979+
{"id": 3, "nested": {"a": 1, "b": "x"}},
980+
{"id": 4, "nested": {"a": 1, "b": "x"}},
981+
{"id": 5, "nested": {"a": 1, "b": None}},
982+
{"id": 6, "nested": {"a": None, "b": None}},
983+
{"id": 7, "nested": None},
984+
],
985+
schema=schema,
986+
)
987+
source = pa.Table.from_pylist(
988+
[
989+
{"id": 1, "nested": None}, # unchanged
990+
{"id": 2, "nested": {"a": 1, "b": "x"}}, # null to a struct
991+
{"id": 3, "nested": None}, # a struct to null
992+
{"id": 4, "nested": {"a": 1, "b": "y"}}, # one field differs
993+
{"id": 5, "nested": {"a": 1, "b": None}}, # unchanged, with a null field
994+
{"id": 6, "nested": None}, # a struct of null fields to null
995+
{"id": 7, "nested": {"a": None, "b": None}}, # null to a struct of null fields
996+
],
997+
schema=schema,
998+
)
999+
1000+
assert get_rows_to_update(source, target, ["id"]).sort_by("id").column("id").to_pylist() == [2, 3, 4, 6, 7]
1001+
1002+
1003+
def test_get_rows_to_update_compares_a_struct_whose_fields_are_in_another_order() -> None:
1004+
source_type = pa.struct([pa.field("a", pa.int64()), pa.field("b", pa.string())])
1005+
target_type = pa.struct([pa.field("b", pa.string()), pa.field("a", pa.int64())])
1006+
source = pa.table({"id": pa.array([1, 2]), "nested": pa.array([{"a": 1, "b": "x"}, {"a": 2, "b": "y"}], type=source_type)})
1007+
target = pa.table({"id": pa.array([1, 2]), "nested": pa.array([{"a": 1, "b": "x"}, {"a": 9, "b": "y"}], type=target_type)})
1008+
1009+
assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2]
1010+
1011+
1012+
def test_get_rows_to_update_compares_a_nested_struct() -> None:
1013+
schema = pa.schema(
1014+
[
1015+
pa.field("id", pa.int32()),
1016+
pa.field("outer", pa.struct([pa.field("inner", pa.struct([pa.field("value", pa.int32())]))])),
1017+
]
1018+
)
1019+
target = pa.Table.from_pylist(
1020+
[
1021+
{"id": 1, "outer": {"inner": {"value": 1}}},
1022+
{"id": 2, "outer": {"inner": {"value": 1}}},
1023+
{"id": 3, "outer": {"inner": None}},
1024+
{"id": 4, "outer": {"inner": {"value": 1}}},
1025+
],
1026+
schema=schema,
1027+
)
1028+
source = pa.Table.from_pylist(
1029+
[
1030+
{"id": 1, "outer": {"inner": {"value": 1}}}, # unchanged
1031+
{"id": 2, "outer": {"inner": {"value": 2}}}, # the innermost value differs
1032+
{"id": 3, "outer": {"inner": {"value": 1}}}, # a null inner struct to a struct
1033+
{"id": 4, "outer": {"inner": None}}, # a struct to a null inner struct
1034+
],
1035+
schema=schema,
1036+
)
1037+
1038+
assert get_rows_to_update(source, target, ["id"]).sort_by("id").column("id").to_pylist() == [2, 3, 4]
1039+
1040+
1041+
def test_get_rows_to_update_compares_a_struct_read_as_large_string() -> None:
1042+
"""A scan reads the strings of a struct as large_string, while a dataframe holds them as string."""
1043+
source = pa.table(
1044+
{
1045+
"id": pa.array([1, 2], pa.int32()),
1046+
"nested": pa.array([{"a": "x"}, {"a": "y"}], type=pa.struct([pa.field("a", pa.string())])),
1047+
}
1048+
)
1049+
target = pa.table(
1050+
{
1051+
"id": pa.array([1, 2], pa.int32()),
1052+
"nested": pa.array([{"a": "x"}, {"a": "z"}], type=pa.struct([pa.field("a", pa.large_string())])),
1053+
}
1054+
)
1055+
1056+
assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2]

0 commit comments

Comments
 (0)