Zipstack · kirtimanmishrazipstack · Apr 24, 2026 · Apr 21, 2026 · Apr 22, 2026 · Apr 22, 2026
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py
@@ -1,5 +1,51 @@
+from enum import Enum
+
+from botocore.exceptions import ClientError
+
 from unstract.connectors.exceptions import ConnectorError
 
+
+class BucketProbeDisposition(Enum):
+    """Action for a `list_objects_v2` probe failure, per S3 error code."""
+
+    DROP = "drop"  # Hide the bucket (no access, or bucket gone).
+    FAIL_OPEN = "fail_open"  # Region mismatch — keep bucket visible.
+    RETRY_FAIL_OPEN = "retry_fail_open"  # Throttled — retry once, then keep.
+
+
+# S3 `Error.Code` → probe disposition. Unlisted codes propagate.
+# `RequestTimeTooSkewed` is omitted deliberately: it's a system-wide clock
+# issue, not a bucket outcome — let it surface via `handle_s3fs_exception`.
+BUCKET_PROBE_DISPOSITION: dict[str, BucketProbeDisposition] = {
+    "AccessDenied": BucketProbeDisposition.DROP,
+    "AllAccessDisabled": BucketProbeDisposition.DROP,
+    "NoSuchBucket": BucketProbeDisposition.DROP,
+    "PermanentRedirect": BucketProbeDisposition.FAIL_OPEN,
+    "IllegalLocationConstraintException": BucketProbeDisposition.FAIL_OPEN,
+    "SlowDown": BucketProbeDisposition.RETRY_FAIL_OPEN,
+    "Throttling": BucketProbeDisposition.RETRY_FAIL_OPEN,
+    "ThrottlingException": BucketProbeDisposition.RETRY_FAIL_OPEN,
+}
+
+
+def s3_error_code(exc: BaseException) -> str:
+    """Return the S3 `Error.Code` from a `ClientError` or its s3fs-translated
+    wrapper (`PermissionError` / `FileNotFoundError` / `OSError`).
+
+    Walks `__cause__` first (explicit `raise X from original`), then falls
+    back to `__context__` (implicit chaining inside an `except` block). A
+    `seen` set guards against pathological cycles.
+    """
+    seen: set[int] = set()
+    target: BaseException | None = exc
+    while target is not None and id(target) not in seen:
+        seen.add(id(target))
+        if isinstance(target, ClientError):
+            return str(target.response.get("Error", {}).get("Code", "") or "")
+        target = target.__cause__ or target.__context__
+    return ""
+
+
 S3FS_EXC_TO_UNSTRACT_EXC: dict[str, str] = {
     # Auth errors
     "The AWS Access Key Id you provided does not exist in our records": (

diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
@@ -1,19 +1,144 @@
+import asyncio
 import logging
 import os
+import random
 from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
 from typing import Any
 
+from botocore.exceptions import ClientError
 from s3fs.core import S3FileSystem
 
 from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
 
-from .exceptions import handle_s3fs_exception
+from .exceptions import (
+    BUCKET_PROBE_DISPOSITION,
+    BucketProbeDisposition,
+    handle_s3fs_exception,
+    s3_error_code,
+)
 
 logger = logging.getLogger(__name__)
 
+# Cap concurrent per-bucket probes to avoid S3 503 SlowDown on large accounts.
+_MAX_CONCURRENT_BUCKET_PROBES = 16
+_BUCKET_PROBE_RETRY_DELAY_SECONDS = 0.5
+
+
+class _AccessFilteredS3FileSystem(S3FileSystem):  # type: ignore[misc]
+    """Lists only buckets the credentials can browse.
+
+    Probes each bucket and looks up the S3 `Error.Code` in
+    `BUCKET_PROBE_DISPOSITION`. Unlisted codes propagate.
+    """
+
+    async def _lsbuckets(self, refresh: bool = False) -> list[dict[str, Any]]:
+        buckets: list[dict[str, Any]] = await super()._lsbuckets(refresh=refresh)
+        if not buckets:
+            return buckets
+        return await self._filter_accessible_buckets(buckets)
+
+    async def _filter_accessible_buckets(
+        self, buckets: list[dict[str, Any]]
+    ) -> list[dict[str, Any]]:
+        sem = asyncio.Semaphore(_MAX_CONCURRENT_BUCKET_PROBES)
+
+        async def _probe(name: str) -> bool:
+            async with sem:
+                return await self._is_bucket_accessible(name)
+
+        results = await asyncio.gather(*(_probe(b["name"]) for b in buckets))
+        accessible = [b for b, ok in zip(buckets, results, strict=True) if ok]
+        dropped = len(buckets) - len(accessible)
+        if dropped:
+            logger.info(
+                "[S3/MinIO] Bucket filter: kept %d of %d; "
+                "dropped %d on DROP dispositions.",
+                len(accessible),
+                len(buckets),
+                dropped,
+            )
+        return accessible
+
+    async def _is_bucket_accessible(self, name: str) -> bool:
+        try:
+            await self._call_s3("list_objects_v2", Bucket=name, MaxKeys=1)
+            return True
+        except (ClientError, OSError) as exc:
+            code = s3_error_code(exc)
+            disposition = BUCKET_PROBE_DISPOSITION.get(code)
+            if disposition is BucketProbeDisposition.DROP:
+                return False
+            if disposition is BucketProbeDisposition.FAIL_OPEN:
+                logger.warning(
+                    "[S3/MinIO] Bucket %r probe returned %s; keeping in listing.",
+                    name,
+                    code,
+                )
+                return True
+            if disposition is BucketProbeDisposition.RETRY_FAIL_OPEN:
+                return await self._retry_probe_fail_open(name, code)
+            # Unclassified code: log which bucket before propagating so the
+            # gather-cancellation has a breadcrumb.
+            logger.exception(
+                "[S3/MinIO] Unclassified probe failure for bucket %r "
+                "(code=%r); propagating.",
+                name,
+                code,
+            )
+            raise
+
+    async def _retry_probe_fail_open(self, name: str, first_code: str) -> bool:
+        # Full jitter avoids correlated retries under SlowDown thundering herd.
+        await asyncio.sleep(
+            _BUCKET_PROBE_RETRY_DELAY_SECONDS
+            + random.uniform(0, _BUCKET_PROBE_RETRY_DELAY_SECONDS)
+        )
+        try:
+            await self._call_s3("list_objects_v2", Bucket=name, MaxKeys=1)
+            return True
+        except (ClientError, OSError) as retry_exc:
+            retry_code = s3_error_code(retry_exc)
+            disposition = BUCKET_PROBE_DISPOSITION.get(retry_code)
+            if disposition is BucketProbeDisposition.DROP:
+                return False
+            if disposition is BucketProbeDisposition.FAIL_OPEN:
+                logger.warning(
+                    "[S3/MinIO] Bucket %r fail-open on retry "
+                    "(first=%s, retry=%s); keeping in listing.",
+                    name,
+                    first_code,
+                    retry_code,
+                )
+                return True
+            if disposition is BucketProbeDisposition.RETRY_FAIL_OPEN:
+                # Still transient after one retry — keep to avoid flapping.
+                logger.info(
+                    "[S3/MinIO] Bucket %r still transient after retry "
+                    "(first=%s, retry=%s); keeping in listing.",
+                    name,
+                    first_code,
+                    retry_code,
+                )
+                return True
+            # Unclassified retry failure (e.g. credentials expired mid-probe):
+            # log the bucket, then re-raise so the real error surfaces.
+            logger.exception(
+                "[S3/MinIO] Unclassified retry probe failure for bucket %r "
+                "(first=%s, retry=%s); propagating.",
+                name,
+                first_code,
+                retry_code,
+            )
+            raise
+
 
 class MinioFS(UnstractFileSystem):
+    # Override with plain S3FileSystem in a subclass when the credentials are
+    # known to have full access to every bucket they list, so the per-bucket
+    # access probe in _AccessFilteredS3FileSystem can be skipped.
+    _FS_CLASS: type[S3FileSystem] = _AccessFilteredS3FileSystem
+
     def __init__(self, settings: dict[str, Any]):
         super().__init__("MinioFS/S3")
         key = (settings.get("key") or "").strip()
@@ -30,7 +155,7 @@ def __init__(self, settings: dict[str, Any]):
         if endpoint_url:
             creds["endpoint_url"] = endpoint_url
 
-        self.s3 = S3FileSystem(
+        self.s3 = self._FS_CLASS(
             anon=False,
             use_listings_cache=False,
             default_fill_cache=False,

diff --git a/unstract/connectors/src/unstract/connectors/filesystems/ucs/ucs.py b/unstract/connectors/src/unstract/connectors/filesystems/ucs/ucs.py
@@ -1,5 +1,7 @@
 import os
 
+from s3fs.core import S3FileSystem
+
 from unstract.connectors.filesystems.minio.minio import MinioFS
 
 
@@ -9,6 +11,11 @@ class UnstractCloudStorage(MinioFS):
     Implemented with Google Cloud Storage through Minio.
     """
 
+    # UCS credentials always have full access to UCS buckets, so skip the
+    # per-bucket access probe that MinioFS runs on its fsspec filesystem.
+    # The probe adds latency and could hide a bucket on a transient S3 error.
+    _FS_CLASS = S3FileSystem
+
     @staticmethod
     def get_id() -> str:
         return "pcs|b8cd25cd-4452-4d54-bd5e-e7d71459b702"