Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -2721,6 +2721,7 @@ def data_file_statistics_from_parquet_metadata(
parquet_metadata: pq.FileMetaData,
stats_columns: dict[int, StatisticsCollector],
parquet_column_mapping: dict[str, int],
table_nan_value_counts: dict[int, int] | None = None,
) -> DataFileStatistics:
"""
Compute and return DataFileStatistics that includes the following.
Expand All @@ -2738,6 +2739,9 @@ def data_file_statistics_from_parquet_metadata(
stats_columns (Dict[int, StatisticsCollector]): The statistics gathering plan. It is required to
set the mode for column metrics collection
parquet_column_mapping (Dict[str, int]): The mapping of the parquet file name to the field ID
table_nan_value_counts (Dict[int, int] | None): NaN counts computed from the written table itself.
The parquet statistics API does not expose NaN counts, so the writer supplies them; defaults
to no NaN counts being reported.
"""
column_sizes: dict[int, int] = {}
value_counts: dict[int, int] = {}
Expand Down Expand Up @@ -2845,7 +2849,7 @@ def data_file_statistics_from_parquet_metadata(
column_sizes=column_sizes,
value_counts=value_counts,
null_value_counts=null_value_counts,
nan_value_counts=nan_value_counts,
nan_value_counts=table_nan_value_counts if table_nan_value_counts is not None else nan_value_counts,
column_aggregates=col_aggs,
split_offsets=split_offsets,
)
Expand All @@ -2866,6 +2870,7 @@ def __init__(
self._properties = properties
self._writer: pq.ParquetWriter | None = None
self._fos: OutputStream | None = None
self._nan_value_counts: dict[int, int] = {}
self._parquet_writer_kwargs = _get_parquet_writer_kwargs(properties)
self._parquet_writer_kwargs["encryption_properties"] = _parquet_encryption_properties(key_metadata)
self._row_group_size = property_as_int(
Expand All @@ -2889,6 +2894,14 @@ def write(self, table: pa.Table) -> None:
raise
self._fos = fos
self._writer.write(table, row_group_size=self._row_group_size)
# The parquet statistics API does not expose NaN counts, so they are counted from the data
# itself. `is_nan` is null on nulls, and `sum` skips nulls, matching the spec's NaN-count
# semantics of counting non-null NaN values.
for field in self._file_schema.fields:
if isinstance(field.field_type, (FloatType, DoubleType)):
nan_count = pc.sum(pc.is_nan(table.column(field.name)), min_count=0).as_py()
if nan_count:
self._nan_value_counts[field.field_id] = self._nan_value_counts.get(field.field_id, 0) + nan_count

def close(self) -> DataFileStatistics:
if self._result is not None:
Expand All @@ -2901,6 +2914,7 @@ def close(self) -> DataFileStatistics:
parquet_metadata=self._writer.writer.metadata,
stats_columns=compute_statistics_plan(self._file_schema, self._properties),
parquet_column_mapping=parquet_path_to_id_mapping(self._file_schema),
table_nan_value_counts=self._nan_value_counts,
)
return self._result

Expand Down
62 changes: 61 additions & 1 deletion tests/io/test_pyarrow_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,13 @@
STRUCT_INT32,
STRUCT_INT64,
)
from pyiceberg.expressions import IsNull
from pyiceberg.expressions import IsNull, NotNaN
from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator
from pyiceberg.io.pyarrow import (
MetricModeTypes,
MetricsMode,
ParquetFormatWriter,
PyArrowFileIO,
PyArrowStatisticsCollector,
compute_statistics_plan,
data_file_statistics_from_parquet_metadata,
Expand All @@ -67,6 +69,7 @@
from pyiceberg.typedef import Record
from pyiceberg.types import (
BooleanType,
DoubleType,
FloatType,
IntegerType,
NestedField,
Expand Down Expand Up @@ -325,6 +328,63 @@ def test_unknown_row_group_null_count_is_omitted(unknown_row_group: int) -> None
assert _InclusiveMetricsEvaluator(schema, IsNull("value")).eval(data_file)


def test_nan_value_counts_are_written(tmp_path) -> None:
schema = Schema(NestedField(1, "reading", DoubleType(), required=False))
arrow_schema = schema_to_pyarrow(schema)
table = pa.table(
{"reading": pa.array([1.5, float("nan"), float("nan"), None], type=pa.float64())},
schema=arrow_schema,
)

writer = ParquetFormatWriter(PyArrowFileIO().new_output(str(tmp_path / "nan.parquet")), file_schema=schema, properties={})
writer.write(table)
statistics = writer.close()

# Two non-null NaNs; nulls are counted separately and excluded from the NaN count
assert statistics.nan_value_counts == {1: 2}

data_file = DataFile.from_args(
file_path="nan.parquet",
file_format=FileFormat.PARQUET,
partition=Record(),
file_size_in_bytes=1,
**statistics.to_serialized_dict(),
)
# The file holds non-NaN values, so a not_nan filter must not prune it
assert _InclusiveMetricsEvaluator(schema, NotNaN("reading")).eval(data_file)


def test_nan_value_counts_are_written_across_row_groups(tmp_path) -> None:
schema = Schema(NestedField(1, "reading", DoubleType(), required=False))
arrow_schema = schema_to_pyarrow(schema)
table = pa.table(
{
"reading": pa.array(
[float("nan"), float("nan")],
type=pa.float64(),
)
},
schema=arrow_schema,
)

writer = ParquetFormatWriter(PyArrowFileIO().new_output(str(tmp_path / "allnan.parquet")), file_schema=schema, properties={})
writer.write(table.slice(0, 1))
writer.write(table.slice(1, 1))
statistics = writer.close()

assert statistics.nan_value_counts == {1: 2}

data_file = DataFile.from_args(
file_path="allnan.parquet",
file_format=FileFormat.PARQUET,
partition=Record(),
file_size_in_bytes=1,
**statistics.to_serialized_dict(),
)
# Every non-null value is NaN, so a not_nan filter prunes the file
assert not _InclusiveMetricsEvaluator(schema, NotNaN("reading")).eval(data_file)


def test_bounds() -> None:
metadata, table_metadata = construct_test_table()

Expand Down
Loading