Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions pyiceberg/table/inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from pyiceberg.manifest import DataFile, DataFileContent, ManifestContent, ManifestFile, PartitionFieldSummary
from pyiceberg.partitioning import PartitionSpec
from pyiceberg.table.snapshots import Snapshot, ancestors_of
from pyiceberg.types import PrimitiveType
from pyiceberg.types import IcebergType, PrimitiveType
from pyiceberg.utils.concurrent import ExecutorFactory
from pyiceberg.utils.singleton import _convert_to_hashable_type

Expand Down Expand Up @@ -553,13 +553,23 @@ def _generate_manifests_table(self, snapshot: Snapshot | None, is_all_manifests_
"""
import pyarrow as pa

partition_types_by_spec_id: dict[int, dict[int, IcebergType]] = {}

def _partition_summaries_to_rows(
spec: PartitionSpec, partition_summaries: list[PartitionFieldSummary]
) -> list[dict[str, Any]]:
partition_field_types = partition_types_by_spec_id.get(spec.spec_id)
if partition_field_types is None:
partition_field_types = {
struct_field.field_id: struct_field.field_type
for struct_field in self.tbl.metadata.specs_struct([spec.spec_id]).fields
}
partition_types_by_spec_id[spec.spec_id] = partition_field_types

rows = []
for i, field_summary in enumerate(partition_summaries):
field = spec.fields[i]
partition_field_type = spec.partition_type(self.tbl.schema()).fields[i].field_type
partition_field_type = partition_field_types[field.field_id]
lower_bound = (
(
field.transform.to_human_string(
Expand Down
16 changes: 12 additions & 4 deletions pyiceberg/table/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
IcebergRootModel,
Properties,
)
from pyiceberg.types import NestedField, StructType, transform_dict_value_to_str
from pyiceberg.types import IcebergType, NestedField, StructType, UnknownType, transform_dict_value_to_str
from pyiceberg.utils.config import Config
from pyiceberg.utils.datetime import datetime_to_millis

Expand Down Expand Up @@ -298,6 +298,16 @@ def specs(self) -> dict[int, PartitionSpec]:
"""Return a dict the partition specs this table."""
return {spec.spec_id: spec for spec in self.partition_specs}

def _resolve_field_type(self, field_id: int) -> IcebergType:
"""Resolve a field's type from the current schema or historical schemas."""
if field := self.schema()._lazy_id_to_field.get(field_id):
return field.field_type
# Historical partition specs may reference columns dropped in later schema evolutions.
for schema in reversed(self.schemas):
if field := schema._lazy_id_to_field.get(field_id):
return field.field_type
return UnknownType()

def specs_struct(self, spec_ids: Iterable[int] | None = None) -> StructType:
"""Produce a struct of the combined PartitionSpecs.

Expand All @@ -316,13 +326,11 @@ def specs_struct(self, spec_ids: Iterable[int] | None = None) -> StructType:
# Collect all the fields
struct_fields = {field.field_id: field for spec in selected_specs for field in spec.fields}

schema = self.schema()

nested_fields = []
# Sort them by field_id in order to get a deterministic output
for field_id in sorted(struct_fields):
field = struct_fields[field_id]
source_type = schema.find_type(field.source_id)
source_type = self._resolve_field_type(field.source_id)
result_type = field.transform.result_type(source_type)
nested_fields.append(NestedField(field_id=field.field_id, name=field.name, type=result_type, required=False))

Expand Down
69 changes: 69 additions & 0 deletions tests/table/test_inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,3 +138,72 @@ def test_inspect_bounds_after_type_promotion(catalog: InMemoryCatalog) -> None:
partition_summaries = tbl.inspect.manifests().to_pydict()["partition_summaries"][0]
assert partition_summaries[0]["lower_bound"] == str(values[0]), name
assert partition_summaries[0]["upper_bound"] == str(values[-1]), name


def test_inspect_after_dropping_old_partition_column(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(2, "p", StringType(), required=False),
NestedField(3, "q", StringType(), required=False),
)
spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"))
tbl = catalog.create_table("default.dropped_partition_col", schema, partition_spec=spec)
tbl.append(pa.table({"id": [1, 2], "p": ["a", "b"], "q": ["x", "y"]}))

with tbl.update_spec() as spec_update:
spec_update.remove_field("p")
with tbl.update_schema() as schema_update:
schema_update.delete_column("p")

assert tbl.inspect.partitions().to_pydict()["partition"] == [{"p": "a"}, {"p": "b"}]
for table_view in [
tbl.inspect.entries(),
tbl.inspect.files(),
tbl.inspect.data_files(),
tbl.inspect.all_files(),
tbl.inspect.all_data_files(),
]:
assert table_view.num_rows == 2
assert tbl.inspect.delete_files().num_rows == 0
assert tbl.inspect.all_delete_files().num_rows == 0

for manifest_table in [tbl.inspect.manifests(), tbl.inspect.all_manifests()]:
assert manifest_table.to_pydict()["partition_summaries"][0] == [
{"contains_null": False, "contains_nan": False, "lower_bound": "a", "upper_bound": "b"}
]

# Appending under an unpartitioned spec and a newly evolved partition column also stays readable
tbl.append(pa.table({"id": [3], "q": ["z"]}))
with tbl.update_spec() as spec_update:
spec_update.add_identity("q")
tbl.append(pa.table({"id": [4], "q": ["w"]}))

assert sorted(
[(row["p"], row["q"]) for row in tbl.inspect.partitions().to_pydict()["partition"]],
key=lambda item: (item[0] or "", item[1] or ""),
) == [(None, None), (None, "w"), ("a", None), ("b", None)]
assert tbl.inspect.entries().num_rows == 4
assert tbl.inspect.files().num_rows == 4
assert tbl.inspect.all_files().num_rows == 4


def test_inspect_after_promoting_then_dropping_partition_column(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(2, "v", IntegerType(), required=False),
)
spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "v"))
tbl = catalog.create_table("default.promoted_then_dropped", schema, partition_spec=spec)
tbl.append(pa.table({"id": pa.array([1, 2], pa.int64()), "v": pa.array([10, 20], pa.int32())}))

with tbl.update_schema() as schema_update:
schema_update.update_column("v", field_type=LongType())
with tbl.update_spec() as spec_update:
spec_update.remove_field("v")
with tbl.update_schema() as schema_update:
schema_update.delete_column("v")

assert sorted(row["v"] for row in tbl.inspect.partitions().to_pydict()["partition"]) == [10, 20]
assert tbl.inspect.manifests().to_pydict()["partition_summaries"][0] == [
{"contains_null": False, "contains_nan": False, "lower_bound": "10", "upper_bound": "20"}
]
50 changes: 50 additions & 0 deletions tests/table/test_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -992,3 +992,53 @@ def test_v3_metadata_with_encryption_keys_serializes_them(example_table_metadata
assert table_metadata.model_dump(mode="json")["encryption-keys"] == [
{"key-id": "a", "encrypted-key-metadata": "a2V5", "properties": {}}
]


def test_specs_struct_resolves_dropped_partition_source_from_historical_schemas() -> None:
initial_schema = Schema(
NestedField(1, "id", LongType()),
NestedField(2, "p", StringType()),
NestedField(3, "v", IntegerType()),
schema_id=0,
)
promoted_schema = Schema(
NestedField(1, "id", LongType()),
NestedField(2, "p", StringType()),
NestedField(3, "v", LongType()),
schema_id=1,
)
dropped_schema = Schema(NestedField(1, "id", LongType()), schema_id=2)

spec_p = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"), spec_id=0)
spec_v = PartitionSpec(PartitionField(3, 1001, IdentityTransform(), "v"), spec_id=1)

table_metadata = TableMetadataV2(
location="s3://bucket/test/location",
schemas=[initial_schema, promoted_schema, dropped_schema],
current_schema_id=2,
partition_specs=[spec_p, spec_v, PartitionSpec(spec_id=2)],
default_spec_id=2,
last_column_id=3,
last_partition_id=1001,
)

assert table_metadata.specs_struct() == StructType(
NestedField(1000, "p", StringType(), required=False),
NestedField(1001, "v", LongType(), required=False),
)


def test_specs_struct_falls_back_to_unknown_when_source_missing_from_all_schemas() -> None:
table_metadata = TableMetadataV2(
location="s3://bucket/test/location",
schemas=[Schema(NestedField(1, "id", LongType()), schema_id=0)],
current_schema_id=0,
partition_specs=[PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"), spec_id=0)],
default_spec_id=0,
last_column_id=2,
last_partition_id=1000,
)

assert table_metadata.specs_struct() == StructType(
NestedField(1000, "p", UnknownType(), required=False),
)
Loading