Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 43 additions & 3 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -2717,10 +2717,45 @@ def parquet_path_to_id_mapping(
return result


def _nan_value_counts_from_arrow(table: pa.Table, stats_columns: dict[int, StatisticsCollector]) -> dict[int, int]:
"""Count the non-null NaN values of the top-level float and double columns.

PyArrow does not expose the NaN count of a Parquet column statistic, so the counts are taken
from the data before it is written. Columns that cannot be counted are left out, which only
means the file cannot be pruned on NaN counts.

Args:
table (pyarrow.Table): The data that is written to the file.
stats_columns (dict[int, StatisticsCollector]): The statistics gathering plan.

Returns:
A mapping of Iceberg field ID to the number of non-null NaN values.
"""
nan_value_counts: dict[int, int] = {}
for pos, arrow_field in enumerate(table.schema):
Comment thread
Fokko marked this conversation as resolved.
Resolved
if (field_id := _get_field_id(arrow_field)) is None:
continue

stats_col = stats_columns.get(field_id)
if stats_col is None:
continue

if stats_col.mode == MetricsMode(MetricModeTypes.NONE):
continue

if not isinstance(stats_col.iceberg_type, (FloatType, DoubleType)):
continue

nan_value_counts[field_id] = pc.sum(pc.is_nan(table.column(pos)), min_count=0).as_py()

return nan_value_counts


def data_file_statistics_from_parquet_metadata(
parquet_metadata: pq.FileMetaData,
stats_columns: dict[int, StatisticsCollector],
parquet_column_mapping: dict[str, int],
nan_value_counts: dict[int, int] | None = None,
) -> DataFileStatistics:
"""
Compute and return DataFileStatistics that includes the following.
Expand All @@ -2738,13 +2773,14 @@ def data_file_statistics_from_parquet_metadata(
stats_columns (Dict[int, StatisticsCollector]): The statistics gathering plan. It is required to
set the mode for column metrics collection
parquet_column_mapping (Dict[str, int]): The mapping of the parquet file name to the field ID
nan_value_counts (dict[int, int] | None): The NaN value counts computed from the data, if available
"""
column_sizes: dict[int, int] = {}
value_counts: dict[int, int] = {}
split_offsets: list[int] = []

null_value_counts: dict[int, int] = {}
nan_value_counts: dict[int, int] = {}
nan_value_counts = dict(nan_value_counts or {})

col_aggs = {}

Expand Down Expand Up @@ -2863,11 +2899,12 @@ def __init__(
) -> None:
self._output_file = output_file
self._file_schema = file_schema
self._properties = properties
self._writer: pq.ParquetWriter | None = None
self._fos: OutputStream | None = None
self._parquet_writer_kwargs = _get_parquet_writer_kwargs(properties)
self._parquet_writer_kwargs["encryption_properties"] = _parquet_encryption_properties(key_metadata)
self._stats_columns = compute_statistics_plan(file_schema, properties)
self._nan_value_counts: dict[int, int] = {}
self._row_group_size = property_as_int(
properties=properties,
property_name=TableProperties.PARQUET_ROW_GROUP_LIMIT,
Expand All @@ -2889,6 +2926,8 @@ def write(self, table: pa.Table) -> None:
raise
self._fos = fos
self._writer.write(table, row_group_size=self._row_group_size)
for field_id, count in _nan_value_counts_from_arrow(table, self._stats_columns).items():
self._nan_value_counts[field_id] = self._nan_value_counts.get(field_id, 0) + count

def close(self) -> DataFileStatistics:
if self._result is not None:
Expand All @@ -2899,8 +2938,9 @@ def close(self) -> DataFileStatistics:
self._writer.close()
self._result = data_file_statistics_from_parquet_metadata(
parquet_metadata=self._writer.writer.metadata,
stats_columns=compute_statistics_plan(self._file_schema, self._properties),
stats_columns=self._stats_columns,
parquet_column_mapping=parquet_path_to_id_mapping(self._file_schema),
nan_value_counts=self._nan_value_counts,
)
return self._result

Expand Down
2 changes: 1 addition & 1 deletion tests/integration/test_writes/test_writes.py
Original file line number Diff line number Diff line change
Expand Up @@ -1688,7 +1688,7 @@ def test_merge_manifests_file_content(session_catalog: Catalog, arrow_table_with
(11, b"\x01"),
(12, b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"),
]
assert tbl_a_data_file["nan_value_counts"] == []
assert tbl_a_data_file["nan_value_counts"] == [(6, 0), (7, 0)]
assert tbl_a_data_file["null_value_counts"] == [
(1, 1),
(2, 1),
Expand Down
59 changes: 57 additions & 2 deletions tests/io/test_format_writers.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@
import pytest

from pyiceberg.io.fileformat import FileFormatFactory, FileFormatModel
from pyiceberg.io.pyarrow import PyArrowFileIO
from pyiceberg.io.pyarrow import ParquetFormatModel, PyArrowFileIO, schema_to_pyarrow
from pyiceberg.manifest import FileFormat
from pyiceberg.schema import Schema
from pyiceberg.types import LongType, NestedField
from pyiceberg.types import DoubleType, FloatType, LongType, NestedField


@pytest.fixture(params=FileFormatFactory.available_formats(), ids=lambda f: f.name.lower())
Expand Down Expand Up @@ -138,3 +138,58 @@ def test_parquet_format_model_adds_field_id_metadata() -> None:
metadata_no_ids: dict[bytes, bytes] = {}
ParquetFormatModel().add_field_metadata(field, metadata_no_ids, include_field_ids=False)
assert metadata_no_ids == {}


def test_parquet_writer_accumulates_nan_value_counts(tmp_path: Path) -> None:
"""NaN counts from multiple write() calls are added together."""
schema = Schema(NestedField(field_id=1, name="d", field_type=DoubleType(), required=False))
arrow_schema = schema_to_pyarrow(schema, include_field_ids=True)
file_path = str(tmp_path / "test.parquet")
writer = ParquetFormatModel().create_writer(PyArrowFileIO().new_output(file_path), schema, {})

writer.write(pa.Table.from_arrays([pa.array([float("nan"), float("nan")], type=pa.float64())], schema=arrow_schema))
writer.write(pa.Table.from_arrays([pa.array([1.0, None], type=pa.float64())], schema=arrow_schema))
statistics = writer.close()

assert dict(statistics.nan_value_counts) == {1: 2}


def test_parquet_writer_skips_nan_counts_for_none_metrics_mode(tmp_path: Path) -> None:
"""A column with metrics mode none does not report a NaN count."""
schema = Schema(
NestedField(field_id=1, name="f", field_type=FloatType(), required=False),
NestedField(field_id=2, name="d", field_type=DoubleType(), required=False),
)
arrow_schema = schema_to_pyarrow(schema, include_field_ids=True)
file_path = str(tmp_path / "test.parquet")
writer = ParquetFormatModel().create_writer(
PyArrowFileIO().new_output(file_path), schema, {"write.metadata.metrics.column.f": "none"}
)

writer.write(
pa.Table.from_arrays(
[
pa.array([float("nan"), 1.0], type=pa.float32()),
pa.array([float("nan"), 1.0], type=pa.float64()),
],
schema=arrow_schema,
)
)
statistics = writer.close()

assert dict(statistics.nan_value_counts) == {2: 1}


def test_parquet_writer_default_none_metrics_mode_reports_no_nan_counts(tmp_path: Path) -> None:
"""With metrics mode none as the default, no column reports a NaN count."""
schema = Schema(NestedField(field_id=1, name="d", field_type=DoubleType(), required=False))
arrow_schema = schema_to_pyarrow(schema, include_field_ids=True)
file_path = str(tmp_path / "test.parquet")
writer = ParquetFormatModel().create_writer(
PyArrowFileIO().new_output(file_path), schema, {"write.metadata.metrics.default": "none"}
)

writer.write(pa.Table.from_arrays([pa.array([float("nan"), 1.0], type=pa.float64())], schema=arrow_schema))
statistics = writer.close()

assert dict(statistics.nan_value_counts) == {}
33 changes: 33 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3596,6 +3596,39 @@ def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None:
assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2]


def test_append_counts_nan_values(catalog: InMemoryCatalog) -> None:
Comment thread
Fokko marked this conversation as resolved.
Resolved
schema = Schema(
NestedField(id=1, name="f", field_type=FloatType(), required=False),
NestedField(id=2, name="d", field_type=DoubleType(), required=False),
)
catalog.create_namespace("default")
table = catalog.create_table("default.test_append_nan_counts", schema=schema)

table.append(
pa.table(
{
"f": pa.array([float("nan"), float("nan"), 1.0, None], type=pa.float32()),
"d": pa.array([float("nan"), 2.0, None, float("nan")], type=pa.float64()),
}
)
)

data_file = next(iter(table.scan().plan_files())).file
assert dict(data_file.nan_value_counts) == {1: 2, 2: 2}


def test_scan_prunes_files_without_nan_values(catalog: InMemoryCatalog) -> None:
schema = Schema(NestedField(id=1, name="d", field_type=DoubleType(), required=False))
catalog.create_namespace("default")
table = catalog.create_table("default.test_scan_prunes_nan_counts", schema=schema)

table.append(pa.table({"d": pa.array([float("nan"), 1.0], type=pa.float64())}))
table.append(pa.table({"d": pa.array([2.0, 3.0], type=pa.float64())}))

assert len(list(table.scan(row_filter="d IS NAN").plan_files())) == 1
assert len(list(table.scan(row_filter="d IS NOT NAN").plan_files())) == 2


def test_inspect_partition_for_nested_field(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="foo", field_type=StringType(), required=True),
Expand Down
Loading