Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions mkdocs/docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ Iceberg tables support table properties to configure table behavior.
| `write.parquet.row-group-limit` | Number of rows | 1048576 | The upper bound of the number of entries within a single row group |
| `write.parquet.page-size-bytes` | Size in bytes | 1MB | Set a target threshold for the approximate encoded size of data pages within a column chunk |
| `write.parquet.page-row-limit` | Number of rows | 20000 | Set a target threshold for the maximum number of rows within a column chunk |
| `write.parquet.page-index-enabled` | Boolean | True | Write Parquet column and offset indexes so readers can skip non-matching pages during predicate evaluation. Set to false to disable |
| `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group |
| `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression |
| `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported |
Expand Down
5 changes: 5 additions & 0 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3160,6 +3160,11 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]:
kwargs: dict[str, Any] = {
"compression": compression_codec,
"compression_level": compression_level,
"write_page_index": property_as_bool(
properties=table_properties,
property_name=TableProperties.PARQUET_PAGE_INDEX_ENABLED,
default=TableProperties.PARQUET_PAGE_INDEX_ENABLED_DEFAULT,
),
"data_page_size": property_as_int(
properties=table_properties,
property_name=TableProperties.PARQUET_PAGE_SIZE_BYTES,
Expand Down
3 changes: 3 additions & 0 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,9 @@ class TableProperties:
PARQUET_PAGE_ROW_LIMIT = "write.parquet.page-row-limit"
PARQUET_PAGE_ROW_LIMIT_DEFAULT = 20000

PARQUET_PAGE_INDEX_ENABLED = "write.parquet.page-index-enabled"
PARQUET_PAGE_INDEX_ENABLED_DEFAULT = True

PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes"
PARQUET_DICT_SIZE_BYTES_DEFAULT = 2 * 1024 * 1024 # 2 MB

Expand Down
24 changes: 12 additions & 12 deletions tests/integration/test_writes/test_writes.py
Original file line number Diff line number Diff line change
Expand Up @@ -1653,18 +1653,18 @@ def test_merge_manifests_file_content(session_catalog: Catalog, arrow_table_with
for i in range(3):
tbl_a_data_file = tbl_a_entries["data_file"][i]
assert tbl_a_data_file["column_sizes"] == [
(1, 51),
(2, 80),
(3, 130),
(4, 96),
(5, 120),
(6, 96),
(7, 120),
(8, 120),
(9, 120),
(10, 96),
(11, 80),
(12, 111),
(1, 33),
(2, 68),
(3, 76),
(4, 66),
(5, 74),
(6, 66),
(7, 74),
(8, 74),
(9, 74),
(10, 66),
(11, 68),
(12, 69),
]
assert tbl_a_data_file["content"] == 0
assert tbl_a_data_file["equality_ids"] is None
Expand Down
18 changes: 18 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3596,6 +3596,24 @@ def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None:
assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2]


def test_append_writes_parquet_page_index(catalog: InMemoryCatalog) -> None:
schema = Schema(NestedField(id=1, name="id", field_type=LongType(), required=False))
catalog.create_namespace("default")

for page_index_enabled, identifier in [
(True, "default.test_append_page_index_enabled"),
(False, "default.test_append_page_index_disabled"),
]:
properties = {} if page_index_enabled else {TableProperties.PARQUET_PAGE_INDEX_ENABLED: "false"}
table = catalog.create_table(identifier, schema=schema, properties=properties)
table.append(pa.table({"id": pa.array(range(1000), type=pa.int64())}))

file_path = next(iter(table.scan().plan_files())).file.file_path
column_chunk = pq.read_metadata(file_path).row_group(0).column(0)
assert column_chunk.has_column_index is page_index_enabled
assert column_chunk.has_offset_index is page_index_enabled


def test_inspect_partition_for_nested_field(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="foo", field_type=StringType(), required=True),
Expand Down
Loading