Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 15 additions & 33 deletions pyiceberg/table/name_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,35 +187,20 @@ def __init__(self, updates: dict[int, NestedField], adds: dict[int, list[NestedF
self._adds = adds

@staticmethod
def _remove_reassigned_names(field: MappedField, assignments: dict[str, int]) -> MappedField | None:
removed_names = set()
for name in field.names:
if (assigned_id := assignments.get(name)) and assigned_id != field.field_id:
removed_names.add(name)

remaining_names = [f for f in field.names if f not in removed_names]
if remaining_names:
return MappedField(field_id=field.field_id, names=remaining_names, fields=field.fields)
else:
return None
def _check_names_not_reassigned(fields: list[MappedField], assignments: dict[str, int], operation: str) -> None:
for field in fields:
for name in field.names:
if (assigned_id := assignments.get(name)) is not None and assigned_id != field.field_id:
raise ValueError(f"Cannot {operation} '{name}': the name mapping already assigns it to another field")

def _add_new_fields(self, mapped_fields: list[MappedField], parent_id: int) -> list[MappedField]:
if fields_to_add := self._adds.get(parent_id):
fields: list[MappedField] = []
new_fields: list[MappedField] = []

for add in fields_to_add:
new_fields.append(
MappedField(field_id=add.field_id, names=[add.name], fields=visit(add.field_type, _CreateMapping()))
)

reassignments = {f.name: f.field_id for f in fields_to_add}
fields = [
updated_field
for field in mapped_fields
if (updated_field := self._remove_reassigned_names(field, reassignments)) is not None
] + new_fields
return fields
new_fields = [
MappedField(field_id=add.field_id, names=[add.name], fields=visit(add.field_type, _CreateMapping()))
for add in fields_to_add
]
self._check_names_not_reassigned(mapped_fields, {field.name: field.field_id for field in fields_to_add}, "add column")
return mapped_fields + new_fields
else:
return mapped_fields

Expand All @@ -225,14 +210,11 @@ def mapping(self, nm: NameMapping, field_results: list[MappedField]) -> list[Map
def fields(self, struct: list[MappedField], field_results: list[MappedField]) -> list[MappedField]:
reassignments: dict[str, int] = {
update.name: update.field_id
for f in field_results
if f.field_id is not None and (update := self._updates.get(f.field_id))
}
return [
updated_field
for field in field_results
if (updated_field := self._remove_reassigned_names(field, reassignments)) is not None
]
if field.field_id is not None and (update := self._updates.get(field.field_id))
}
self._check_names_not_reassigned(field_results, reassignments, "rename column to")
return field_results

def field(self, field: MappedField, field_result: list[MappedField]) -> MappedField:
if field.field_id is None:
Expand Down
22 changes: 22 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from typing import Any

import pyarrow as pa
import pyarrow.parquet as pq
import pytest
from pydantic_core import ValidationError
from pytest_lazy_fixtures import lf
Expand Down Expand Up @@ -338,6 +339,27 @@ def test_write_pyarrow_schema(catalog: Catalog, test_table_identifier: Identifie
table.append(pyarrow_table)


def test_add_column_refuses_name_still_used_by_name_mapping(
catalog_with_warehouse: Catalog,
tmp_path: Path,
) -> None:
imported = tmp_path / "imported.parquet"
pq.write_table(pa.table({"a": [1, 3]}), imported)

catalog_with_warehouse.create_namespace("default")
table = catalog_with_warehouse.create_table(("default", "imported"), Schema(NestedField(1, "a", LongType())))
table.add_files([f"file://{imported}"])

with table.update_schema() as update:
update.rename_column("a", "b")

with pytest.raises(ValueError, match="name mapping already assigns it to another field"):
with table.update_schema() as update:
update.add_column("a", LongType())

assert table.scan().to_arrow().to_pydict() == {"b": [1, 3]}


@pytest.mark.parametrize(
"schema,expected",
[
Expand Down
26 changes: 24 additions & 2 deletions tests/table/test_name_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ def test_update_mapping(table_name_mapping_nested: NameMapping) -> None:
updates = {1: NestedField(1, "foo_update", StringType(), True)}
adds = {
-1: [NestedField(18, "add_18", StringType(), True)],
15: [NestedField(19, "name", StringType(), True), NestedField(20, "add_20", StringType(), True)],
15: [NestedField(19, "add_19", StringType(), True), NestedField(20, "add_20", StringType(), True)],
}

expected = NameMapping(
Expand Down Expand Up @@ -345,8 +345,9 @@ def test_update_mapping(table_name_mapping_nested: NameMapping) -> None:
field_id=15,
names=["person"],
fields=[
MappedField(field_id=16, names=["name"]),
MappedField(field_id=17, names=["age"]),
MappedField(field_id=19, names=["name"]),
MappedField(field_id=19, names=["add_19"]),
MappedField(field_id=20, names=["add_20"]),
],
),
Expand All @@ -356,6 +357,27 @@ def test_update_mapping(table_name_mapping_nested: NameMapping) -> None:
assert update_mapping(table_name_mapping_nested, updates, adds) == expected


def test_update_mapping_refuses_reassigned_name_on_add(table_name_mapping_nested: NameMapping) -> None:
adds = {-1: [NestedField(18, "foo", StringType(), True)]}

with pytest.raises(ValueError, match="name mapping already assigns it to another field"):
update_mapping(table_name_mapping_nested, {}, adds)


def test_update_mapping_refuses_reassigned_name_on_nested_add(table_name_mapping_nested: NameMapping) -> None:
adds = {15: [NestedField(19, "name", StringType(), True)]}

with pytest.raises(ValueError, match="name mapping already assigns it to another field"):
update_mapping(table_name_mapping_nested, {}, adds)


def test_update_mapping_refuses_reassigned_name_on_rename(table_name_mapping_nested: NameMapping) -> None:
updates = {2: NestedField(2, "foo", StringType(), True)}

with pytest.raises(ValueError, match="name mapping already assigns it to another field"):
update_mapping(table_name_mapping_nested, updates, {})


def test_mapping_using_by_visitor(table_schema_nested: Schema, table_name_mapping_nested: NameMapping) -> None:
schema_without_ids = Schema(
NestedField(field_id=0, name="foo", field_type=StringType(), required=False),
Expand Down
Loading