Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 49 additions & 13 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@
from pyiceberg.table.deletion_vector import read_deletion_vectors
from pyiceberg.table.locations import load_location_provider
from pyiceberg.table.metadata import TableMetadata
from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID
from pyiceberg.table.name_mapping import NameMapping, apply_name_mapping
from pyiceberg.transforms import IdentityTransform, TruncateTransform
from pyiceberg.typedef import EMPTY_DICT, Properties, Record, TableVersion
Expand Down Expand Up @@ -214,6 +215,8 @@
LIST_ELEMENT_NAME = "element"
MAP_KEY_NAME = "key"
MAP_VALUE_NAME = "value"
# Temporary column that tracks the position of each row in its file while rows are filtered
_POSITION_COLUMN = "__iceberg_file_position"
DOC = "doc"
UTC_ALIASES = {"UTC", "+00:00", "Etc/UTC", "Z"}
ADLS_SCHEMES = frozenset({"abfs", "abfss", "wasb", "wasbs"})
Expand Down Expand Up @@ -1793,12 +1796,18 @@ def _task_to_record_batches(

file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False)

# Row IDs are derived from the position of each row in the file, so rows must not be filtered before they are counted
row_id_index = next(
(index for index, field in enumerate(projected_schema.fields) if field.field_id == ROW_ID.field_id), None
)
filter_after_read = bool(positional_deletes) or row_id_index is not None

fragment_scanner = ds.Scanner.from_fragment(
fragment=fragment,
schema=physical_schema,
# This will push down the query to Arrow.
# But in case there are positional deletes, we have to apply them first
filter=pyarrow_filter if not positional_deletes else None,
# But in case there are positional deletes or row IDs, rows must be counted before filtering
filter=pyarrow_filter if not filter_after_read else None,
columns=[col.name for col in file_project_schema.columns],
)

Expand All @@ -1808,34 +1817,58 @@ def _task_to_record_batches(
next_index = next_index + len(batch)
current_index = next_index - len(batch)
current_batch = batch
positions = pa.array(range(current_index, next_index), type=pa.int64()) if row_id_index is not None else None

if positional_deletes:
# Create the mask of indices that we're interested in
indices = _combine_positional_deletes(positional_deletes, current_index, current_index + len(batch))
current_batch = current_batch.take(indices)
if pyarrow_filter is not None:
# Temporary fix until PyArrow 21 is the minimum supported version
# (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises
# IndexError on PyArrow <21 when the result is empty; Table.filter does not.
table = pa.Table.from_batches([current_batch])
table = table.filter(pyarrow_filter)
if table.num_rows == 0:
current_batch = current_batch.slice(0, 0)
else:
current_batch = table.combine_chunks().to_batches()[0]
if positions is not None:
positions = positions.take(indices)

if filter_after_read and pyarrow_filter is not None:
# Temporary fix until PyArrow 21 is the minimum supported version
# (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises
# IndexError on PyArrow <21 when the result is empty; Table.filter does not.
table = pa.Table.from_batches([current_batch])
if positions is not None:
table = table.append_column(_POSITION_COLUMN, positions)
table = table.filter(pyarrow_filter)
if positions is not None:
positions = table.column(_POSITION_COLUMN).combine_chunks()
table = table.drop_columns([_POSITION_COLUMN])
if table.num_rows == 0:
current_batch = current_batch.slice(0, 0)
else:
current_batch = table.combine_chunks().to_batches()[0]

# skip empty batches
if current_batch.num_rows == 0:
continue

yield _to_requested_schema(
result_batch = _to_requested_schema(
projected_schema,
file_project_schema,
current_batch,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
projected_missing_fields=projected_missing_fields,
allow_timestamp_tz_mismatch=True,
)
if row_id_index is not None and positions is not None:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: positions is always set when row_id_index is not None, so the positions is not None check is redundant.

result_batch = _assign_row_ids(result_batch, row_id_index, positions, task.file.first_row_id)
yield result_batch


def _assign_row_ids(batch: pa.RecordBatch, row_id_index: int, positions: pa.Array, first_row_id: int | None) -> pa.RecordBatch:
"""Replace null row IDs with the file's first row ID plus the row position.

When the data file has no first row ID, every row ID is null.
"""
if first_row_id is None:
row_ids = pa.nulls(len(batch), type=pa.int64())
else:
row_ids = pc.coalesce(batch.column(row_id_index), pc.add(positions, pa.scalar(first_row_id, type=pa.int64())))
return batch.set_column(row_id_index, batch.schema.field(row_id_index), row_ids)


@dataclass(frozen=True, slots=True)
Expand Down Expand Up @@ -1904,6 +1937,9 @@ def __init__(
*,
dictionary_columns: tuple[str, ...] = (),
) -> None:
if any(field.field_id == LAST_UPDATED_SEQUENCE_NUMBER.field_id for field in projected_schema.fields):
# The data sequence number is not carried on the scan task yet
raise NotImplementedError(f"Reading {LAST_UPDATED_SEQUENCE_NUMBER.name} is not yet supported")
self._table_metadata = table_metadata
self._io = io
self._projected_schema = projected_schema
Expand Down
17 changes: 8 additions & 9 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
from pyiceberg.table.locations import LocationProvider, load_location_provider
from pyiceberg.table.maintenance import MaintenanceTable
from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadata
from pyiceberg.table.metadata_columns import project_schema, split_selected_fields
from pyiceberg.table.name_mapping import NameMapping
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef
from pyiceberg.table.snapshots import (
Expand Down Expand Up @@ -2302,10 +2303,7 @@ def projection(self) -> Schema:
else:
raise ValueError(f"Snapshot not found: {self.snapshot_id}")

if "*" in self.selected_fields:
return current_schema

return current_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive)
return project_schema(current_schema, self.selected_fields, self.case_sensitive, self.table_metadata.format_version)

def use_ref(self: S, name: str) -> S:
if self.snapshot_id is not None:
Expand Down Expand Up @@ -2550,9 +2548,11 @@ def _plan_files_server_side(self) -> Iterable[FileScanTask]:
if self.table_identifier is None:
raise ValueError("REST scan planning requires a table identifier")

# Metadata columns are not in the table schema, so the server cannot resolve them
schema_fields, _ = split_selected_fields(self.table_metadata.schema(), self.selected_fields, self.case_sensitive)
request = PlanTableScanRequest(
snapshot_id=self.snapshot_id,
select=list(self.selected_fields) if self.selected_fields != ("*",) else None,
select=list(schema_fields) if schema_fields != ("*",) else None,
filter=self.row_filter if self.row_filter != ALWAYS_TRUE else None,
case_sensitive=self.case_sensitive,
)
Expand Down Expand Up @@ -2736,10 +2736,9 @@ def to_snapshot_id_inclusive(self: IAS, to_snapshot_id: int) -> IAS:
return self.update(to_snapshot_id=to_snapshot_id)

def projection(self) -> Schema:
current_schema = self.table_metadata.schema()
if "*" in self.selected_fields:
return current_schema
return current_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive)
return project_schema(
self.table_metadata.schema(), self.selected_fields, self.case_sensitive, self.table_metadata.format_version
)

def plan_files(self) -> Iterable[FileScanTask]:
"""Plans the relevant files added between the specified snapshots."""
Expand Down
99 changes: 99 additions & 0 deletions pyiceberg/table/metadata_columns.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Metadata columns that are not part of a table schema, but can be selected in a scan."""

from __future__ import annotations

from collections.abc import Mapping
from types import MappingProxyType

from pyiceberg.schema import Schema
from pyiceberg.types import LongType, NestedField

ROW_ID = NestedField(
field_id=2147483540,
name="_row_id",
field_type=LongType(),
required=False,
doc="Implicit row ID that is automatically assigned",
)
LAST_UPDATED_SEQUENCE_NUMBER = NestedField(
field_id=2147483539,
name="_last_updated_sequence_number",
field_type=LongType(),
required=False,
doc="Sequence number when the row was last updated",
)

METADATA_COLUMNS: Mapping[str, NestedField] = MappingProxyType(
{field.name: field for field in (ROW_ID, LAST_UPDATED_SEQUENCE_NUMBER)}
)


def _find_metadata_column(name: str, case_sensitive: bool) -> NestedField | None:
if case_sensitive:
return METADATA_COLUMNS.get(name)
return next((field for field in METADATA_COLUMNS.values() if field.name.lower() == name.lower()), None)


def _has_field(schema: Schema, name: str, case_sensitive: bool) -> bool:
try:
schema.find_field(name, case_sensitive=case_sensitive)
return True
except ValueError:
return False


def split_selected_fields(
schema: Schema, selected_fields: tuple[str, ...], case_sensitive: bool
) -> tuple[tuple[str, ...], tuple[NestedField, ...]]:
"""Split selected names into schema columns and metadata columns.

A name that resolves in the schema is a schema column, even when it matches a metadata column name.
"""
schema_names: list[str] = []
metadata_fields: list[NestedField] = []
for name in selected_fields:
if name != "*" and not _has_field(schema, name, case_sensitive):
if (field := _find_metadata_column(name, case_sensitive)) is not None:
if field not in metadata_fields:
metadata_fields.append(field)
continue
schema_names.append(name)
return tuple(schema_names), tuple(metadata_fields)


def project_schema(schema: Schema, selected_fields: tuple[str, ...], case_sensitive: bool, format_version: int) -> Schema:
"""Prune the schema to the selected names, and append any selected metadata columns."""
schema_names, metadata_fields = split_selected_fields(schema, selected_fields, case_sensitive)

if metadata_fields and format_version < 3:
names = ", ".join(field.name for field in metadata_fields)
raise ValueError(
f"Cannot select {names}: row lineage requires format version 3, but the table has format version {format_version}"
)

projected = schema if "*" in schema_names else schema.select(*schema_names, case_sensitive=case_sensitive)
if not metadata_fields:
return projected

return Schema(
*projected.fields,
*metadata_fields,
schema_id=projected.schema_id,
identifier_field_ids=projected.identifier_field_ids,
)
49 changes: 49 additions & 0 deletions tests/integration/test_reads.py
Original file line number Diff line number Diff line change
Expand Up @@ -1299,6 +1299,55 @@ def test_read_first_row_ids_written_by_spark(catalog: Catalog, spark: SparkSessi
assert actual == expected


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [lf("session_catalog")])
def test_read_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> None:
identifier = "default.test_read_row_ids_written_by_spark"
spark.sql(f"DROP TABLE IF EXISTS {identifier}")
spark.sql(
f"""
CREATE TABLE {identifier} (id int, data string)
USING ICEBERG
TBLPROPERTIES (
'format-version'='3',
'write.update.mode'='copy-on-write',
'write.delete.mode'='merge-on-read',
'write.merge.mode'='merge-on-read'
)
"""
)
spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd')")
spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (5, 'e'), (6, 'f')")
# Copy-on-write rewrites the data file and stores the existing row IDs in it
spark.sql(f"UPDATE {identifier} SET data = 'x' WHERE id = 2")
# Merge-on-read adds deletion vectors, and new rows inherit their row IDs
spark.sql(f"DELETE FROM {identifier} WHERE id = 3")
spark.sql(
f"""
MERGE INTO {identifier} t
USING (SELECT * FROM VALUES (5, 'y'), (7, 'g') AS s(id, data)) s
ON t.id = s.id
WHEN MATCHED THEN UPDATE SET t.data = s.data
WHEN NOT MATCHED THEN INSERT *
"""
)
spark.sql(f"CALL rest.system.rewrite_data_files(table => '{identifier}', options => map('rewrite-all', 'true'))")
# Rows in a new file inherit row IDs from their position, so a filter must not drop rows before they are counted
spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (8, 'h'), (9, 'i'), (10, 'j')")

expected = sorted((row.id, row._row_id) for row in spark.sql(f"SELECT id, _row_id FROM {identifier}").collect())
assert all(row_id is not None for _, row_id in expected)

table = catalog.load_table(identifier)
result = table.scan(selected_fields=("id", "_row_id")).to_arrow()
assert sorted(zip(result["id"].to_pylist(), result["_row_id"].to_pylist(), strict=True)) == expected

filtered = table.scan(row_filter="id >= 9", selected_fields=("id", "_row_id")).to_arrow()
assert sorted(zip(filtered["id"].to_pylist(), filtered["_row_id"].to_pylist(), strict=True)) == [
(id_, row_id) for id_, row_id in expected if id_ >= 9
]


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [lf("session_catalog_hive"), lf("session_catalog")])
def test_filter_after_arrow_scan(catalog: Catalog) -> None:
Expand Down
Loading
Loading