@@ -955,3 +955,102 @@ def test_get_rows_to_update_compares_nested_values() -> None:
955955 target = pa .table ({"pk" : [1 , 2 ], "value" : pa .array ([{"a" : 1 }, {"a" : 3 }])})
956956
957957 assert get_rows_to_update (source , target , ["pk" ]).to_pydict () == {"pk" : [2 ], "value" : [{"a" : 2 }]}
958+
959+
960+ def test_get_rows_to_update_compares_columns_of_a_different_type () -> None :
961+ """A naive and a zoned timestamp of the same instant are not a change."""
962+ target = pa .table ({"id" : pa .array ([1 , 2 ], pa .int32 ()), "ts" : pa .array ([1 , 2 ], pa .timestamp ("us" , tz = "UTC" ))})
963+ source = pa .table ({"id" : pa .array ([1 , 2 ], pa .int32 ()), "ts" : pa .array ([1 , 9 ], pa .timestamp ("us" ))})
964+
965+ assert get_rows_to_update (source , target , ["id" ]).column ("id" ).to_pylist () == [2 ]
966+
967+
968+ def test_get_rows_to_update_compares_a_struct_that_is_null () -> None :
969+ schema = pa .schema (
970+ [
971+ pa .field ("id" , pa .int32 ()),
972+ pa .field ("nested" , pa .struct ([pa .field ("a" , pa .int32 ()), pa .field ("b" , pa .string ())])),
973+ ]
974+ )
975+ target = pa .Table .from_pylist (
976+ [
977+ {"id" : 1 , "nested" : None },
978+ {"id" : 2 , "nested" : None },
979+ {"id" : 3 , "nested" : {"a" : 1 , "b" : "x" }},
980+ {"id" : 4 , "nested" : {"a" : 1 , "b" : "x" }},
981+ {"id" : 5 , "nested" : {"a" : 1 , "b" : None }},
982+ {"id" : 6 , "nested" : {"a" : None , "b" : None }},
983+ {"id" : 7 , "nested" : None },
984+ ],
985+ schema = schema ,
986+ )
987+ source = pa .Table .from_pylist (
988+ [
989+ {"id" : 1 , "nested" : None }, # unchanged
990+ {"id" : 2 , "nested" : {"a" : 1 , "b" : "x" }}, # null to a struct
991+ {"id" : 3 , "nested" : None }, # a struct to null
992+ {"id" : 4 , "nested" : {"a" : 1 , "b" : "y" }}, # one field differs
993+ {"id" : 5 , "nested" : {"a" : 1 , "b" : None }}, # unchanged, with a null field
994+ {"id" : 6 , "nested" : None }, # a struct of null fields to null
995+ {"id" : 7 , "nested" : {"a" : None , "b" : None }}, # null to a struct of null fields
996+ ],
997+ schema = schema ,
998+ )
999+
1000+ assert get_rows_to_update (source , target , ["id" ]).sort_by ("id" ).column ("id" ).to_pylist () == [2 , 3 , 4 , 6 , 7 ]
1001+
1002+
1003+ def test_get_rows_to_update_compares_a_struct_whose_fields_are_in_another_order () -> None :
1004+ source_type = pa .struct ([pa .field ("a" , pa .int64 ()), pa .field ("b" , pa .string ())])
1005+ target_type = pa .struct ([pa .field ("b" , pa .string ()), pa .field ("a" , pa .int64 ())])
1006+ source = pa .table ({"id" : pa .array ([1 , 2 ]), "nested" : pa .array ([{"a" : 1 , "b" : "x" }, {"a" : 2 , "b" : "y" }], type = source_type )})
1007+ target = pa .table ({"id" : pa .array ([1 , 2 ]), "nested" : pa .array ([{"a" : 1 , "b" : "x" }, {"a" : 9 , "b" : "y" }], type = target_type )})
1008+
1009+ assert get_rows_to_update (source , target , ["id" ]).column ("id" ).to_pylist () == [2 ]
1010+
1011+
1012+ def test_get_rows_to_update_compares_a_nested_struct () -> None :
1013+ schema = pa .schema (
1014+ [
1015+ pa .field ("id" , pa .int32 ()),
1016+ pa .field ("outer" , pa .struct ([pa .field ("inner" , pa .struct ([pa .field ("value" , pa .int32 ())]))])),
1017+ ]
1018+ )
1019+ target = pa .Table .from_pylist (
1020+ [
1021+ {"id" : 1 , "outer" : {"inner" : {"value" : 1 }}},
1022+ {"id" : 2 , "outer" : {"inner" : {"value" : 1 }}},
1023+ {"id" : 3 , "outer" : {"inner" : None }},
1024+ {"id" : 4 , "outer" : {"inner" : {"value" : 1 }}},
1025+ ],
1026+ schema = schema ,
1027+ )
1028+ source = pa .Table .from_pylist (
1029+ [
1030+ {"id" : 1 , "outer" : {"inner" : {"value" : 1 }}}, # unchanged
1031+ {"id" : 2 , "outer" : {"inner" : {"value" : 2 }}}, # the innermost value differs
1032+ {"id" : 3 , "outer" : {"inner" : {"value" : 1 }}}, # a null inner struct to a struct
1033+ {"id" : 4 , "outer" : {"inner" : None }}, # a struct to a null inner struct
1034+ ],
1035+ schema = schema ,
1036+ )
1037+
1038+ assert get_rows_to_update (source , target , ["id" ]).sort_by ("id" ).column ("id" ).to_pylist () == [2 , 3 , 4 ]
1039+
1040+
1041+ def test_get_rows_to_update_compares_a_struct_read_as_large_string () -> None :
1042+ """A scan reads the strings of a struct as large_string, while a dataframe holds them as string."""
1043+ source = pa .table (
1044+ {
1045+ "id" : pa .array ([1 , 2 ], pa .int32 ()),
1046+ "nested" : pa .array ([{"a" : "x" }, {"a" : "y" }], type = pa .struct ([pa .field ("a" , pa .string ())])),
1047+ }
1048+ )
1049+ target = pa .table (
1050+ {
1051+ "id" : pa .array ([1 , 2 ], pa .int32 ()),
1052+ "nested" : pa .array ([{"a" : "x" }, {"a" : "z" }], type = pa .struct ([pa .field ("a" , pa .large_string ())])),
1053+ }
1054+ )
1055+
1056+ assert get_rows_to_update (source , target , ["id" ]).column ("id" ).to_pylist () == [2 ]
0 commit comments