Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions pyiceberg/io/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@
)
from urllib.parse import urlparse

from pyiceberg.io.content_cache import ContentCache
from pyiceberg.typedef import EMPTY_DICT, Properties
from pyiceberg.utils.properties import property_as_bool, property_as_int

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -118,6 +120,15 @@ def _is_local_path(path: str) -> bool:
HF_ENDPOINT = "hf.endpoint"
HF_TOKEN = "hf.token"

IO_MANIFEST_CACHE_ENABLED = "io.manifest.cache-enabled"
IO_MANIFEST_CACHE_MAX_TOTAL_BYTES = "io.manifest.cache.max-total-bytes"
IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH = "io.manifest.cache.max-content-length"
IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS = "io.manifest.cache.expiration-interval-ms"

IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT = 100 * 1024 * 1024
IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH_DEFAULT = 8 * 1024 * 1024
IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS_DEFAULT = 60_000


@runtime_checkable
class InputStream(Protocol):
Expand Down Expand Up @@ -272,13 +283,39 @@ def create(self, overwrite: bool = False) -> OutputStream:
"""


def _content_cache_from_properties(properties: Properties) -> ContentCache | None:
if not property_as_bool(properties, IO_MANIFEST_CACHE_ENABLED, False):
return None

max_total_bytes: int = property_as_int( # type: ignore[assignment]
properties, IO_MANIFEST_CACHE_MAX_TOTAL_BYTES, IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT
)
max_content_length: int = property_as_int( # type: ignore[assignment]
properties, IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH, IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH_DEFAULT
)
expiration_interval_ms: int = property_as_int( # type: ignore[assignment]
properties, IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS, IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS_DEFAULT
)
return ContentCache(
max_total_bytes=max_total_bytes,
max_content_length=max_content_length,
expiration_interval_ms=expiration_interval_ms,
)


class FileIO(ABC):
"""A base class for FileIO implementations."""

properties: Properties

def __init__(self, properties: Properties = EMPTY_DICT):
self.properties = properties
self._content_cache = _content_cache_from_properties(properties)

@property
def content_cache(self) -> ContentCache | None:
"""Return the cache for immutable metadata file contents, when enabled."""
return self._content_cache

@abstractmethod
def new_input(self, location: str) -> InputFile:
Expand Down
95 changes: 95 additions & 0 deletions pyiceberg/io/content_cache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""A bounded in-memory cache for the bytes of immutable metadata files."""

from __future__ import annotations

import time
from collections import OrderedDict
from collections.abc import Callable


class ContentCache:
"""Cache file contents by location, bounded by total size and by entry age.

Args:
max_total_bytes: The maximum number of bytes to keep across all entries.
max_content_length: Entries larger than this are not cached.
expiration_interval_ms: Entries older than this are treated as absent.
clock: Returns the current time in seconds, replaced in tests.
"""

def __init__(
self,
max_total_bytes: int,
max_content_length: int,
expiration_interval_ms: int,
clock: Callable[[], float] = time.monotonic,
) -> None:
self.max_total_bytes = max_total_bytes
self.max_content_length = max_content_length
self.expiration_interval_seconds = expiration_interval_ms / 1000
self._clock = clock
self._entries: OrderedDict[str, tuple[bytes, float]] = OrderedDict()
self._total_bytes = 0

def get(self, location: str) -> bytes | None:
"""Return the cached bytes for a location, or None when absent or expired."""
entry = self._entries.get(location)
if entry is None:
return None

data, inserted_at = entry
if self._clock() - inserted_at >= self.expiration_interval_seconds:
self._remove(location)
return None

self._entries.move_to_end(location)
return data

def put(self, location: str, data: bytes) -> None:
"""Cache the bytes for a location, evicting the least recently used entries when needed."""
if len(data) > self.max_content_length or len(data) > self.max_total_bytes or self.max_content_length <= 0:
return

self._remove(location)
self._purge_expired()
while self._entries and self._total_bytes + len(data) > self.max_total_bytes:
_, (evicted, _) = self._entries.popitem(last=False)
self._total_bytes -= len(evicted)

self._entries[location] = (data, self._clock())
self._total_bytes += len(data)

def clear(self) -> None:
"""Remove every entry."""
self._entries.clear()
self._total_bytes = 0

def _remove(self, location: str) -> None:
entry = self._entries.pop(location, None)
if entry is not None:
self._total_bytes -= len(entry[0])

def _purge_expired(self) -> None:
now = self._clock()
for location in [
location
for location, (_, inserted_at) in self._entries.items()
if now - inserted_at >= self.expiration_interval_seconds
]:
self._remove(location)
39 changes: 36 additions & 3 deletions pyiceberg/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from collections.abc import Callable, Iterator
from copy import copy
from enum import Enum, StrEnum
from io import BytesIO
from types import TracebackType
from typing import (
Any,
Expand All @@ -37,7 +38,7 @@
from pyiceberg.encryption.io import EncryptedInputFile
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
from pyiceberg.exceptions import ValidationError
from pyiceberg.io import FileIO, InputFile, OutputFile
from pyiceberg.io import FileIO, InputFile, InputStream, OutputFile
from pyiceberg.partitioning import PartitionSpec
from pyiceberg.schema import Schema
from pyiceberg.typedef import Record, TableVersion
Expand All @@ -62,6 +63,38 @@
INITIAL_SEQUENCE_NUMBER = 0


class _MemoryInputFile(InputFile):
"""An InputFile that serves bytes already read from the file at the location."""

def __init__(self, location: str, data: bytes) -> None:
super().__init__(location)
self._data = data

def __len__(self) -> int:
return len(self._data)

def exists(self) -> bool:
return True

def open(self, seekable: bool = True) -> InputStream:
return BytesIO(self._data)


def _open_metadata_file(io: FileIO, location: str) -> InputFile:
"""Open a metadata file, reading its bytes through the content cache when the FileIO has one."""
cache = io.content_cache
if cache is None:
return io.new_input(location)

if (data := cache.get(location)) is None:
input_file = io.new_input(location)
with input_file.open() as stream:
data = stream.read()
cache.put(location, data)

return _MemoryInputFile(location, data)


class DataFileContent(int, Enum):
DATA = 0
POSITION_DELETES = 1
Expand Down Expand Up @@ -916,7 +949,7 @@ def fetch_manifest_entry(
Returns:
An Iterator of manifest entries.
"""
input_file = io.new_input(self.manifest_path)
input_file = _open_metadata_file(io, self.manifest_path)
if self.key_metadata is not None:
input_file = EncryptedInputFile(input_file, StandardKeyMetadata.from_bytes(self.key_metadata))
with AvroFile[ManifestEntry](
Expand Down Expand Up @@ -1033,7 +1066,7 @@ def _manifests(io: FileIO, manifest_list: str, key_metadata: StandardKeyMetadata
Returns:
A tuple of ManifestFile objects.
"""
file = io.new_input(manifest_list)
file = _open_metadata_file(io, manifest_list)
if key_metadata is not None:
file = EncryptedInputFile(file, key_metadata)
manifest_files = list(read_manifest_list(file))
Expand Down
Loading
Loading