Skip to content

Commit feec382

Browse files
fix(spark): address review feedback on staging filesystem PR
Signed-off-by: abhijeet-dhumal <abhijeetdhumal652@gmail.com>
1 parent 11b765c commit feec382

1 file changed

Lines changed: 6 additions & 6 deletions

File tree

  • sdk/python/feast/infra/offline_stores/contrib/spark_offline_store

sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -492,9 +492,9 @@ def _resolve_staging_filesystem(
492492
"""Return (pyarrow filesystem, prefix-stripped paths) for staging URIs."""
493493
sample = paths[0]
494494

495-
if sample.startswith("s3://") or sample.startswith("s3a://"):
496-
import pyarrow.fs as pafs
495+
import pyarrow.fs as pafs
497496

497+
if sample.startswith("s3://") or sample.startswith("s3a://"):
498498
endpoint = os.environ.get("AWS_ENDPOINT_URL_S3") or os.environ.get(
499499
"AWS_S3_ENDPOINT", ""
500500
)
@@ -504,16 +504,16 @@ def _resolve_staging_filesystem(
504504
kwargs: Dict[str, Any] = {"region": region}
505505
if endpoint:
506506
kwargs["endpoint_override"] = (
507-
endpoint.rstrip("/").replace("https://", "").replace("http://", "")
507+
endpoint.rstrip("/")
508+
.removeprefix("https://")
509+
.removeprefix("http://")
508510
)
509511
kwargs["scheme"] = "https" if endpoint.startswith("https") else "http"
510512
fs = pafs.S3FileSystem(**kwargs)
511-
stripped = [p.replace("s3a://", "").replace("s3://", "") for p in paths]
513+
stripped = [p.removeprefix("s3a://").removeprefix("s3://") for p in paths]
512514
return fs, stripped
513515

514516
if sample.startswith("gs://"):
515-
import pyarrow.fs as pafs
516-
517517
fs = pafs.GcsFileSystem()
518518
stripped = [p[len("gs://") :] for p in paths]
519519
return fs, stripped

0 commit comments

Comments
 (0)