ARROW-16094: [Python] Address docstrings in Filesystems (Utilities) (apache#13582)

AlenkaF · web-flow · commit 39980dcdbfbc · 2022-07-20T08:23:18.000+02:00
Authored-by: Alenka Frim &lt;frim.alenka@gmail.com&gt;
Signed-off-by: Joris Van den Bossche &lt;jorisvandenbossche@gmail.com&gt;
diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx
@@ -44,6 +44,10 @@ def initialize_s3(S3LogLevel log_level=S3LogLevel.Fatal):
     ----------
     log_level : S3LogLevel
         level of logging
+
+    Examples
+    --------
+    >>> fs.initialize_s3(fs.S3LogLevel.Error) # doctest: +SKIP
     """
     cdef CS3GlobalOptions options
     options.log_level = <CS3LogLevel> log_level
@@ -70,8 +74,8 @@ def resolve_s3_region(bucket):
 
     Examples
     --------
-    >>> resolve_s3_region('ursa-labs-taxi-data')
-    'us-east-2'
+    >>> fs.resolve_s3_region('registry.opendata.aws')
+    'us-east-1'
     """
     cdef:
         c_string c_bucket
diff --git a/python/pyarrow/conftest.py b/python/pyarrow/conftest.py
@@ -17,6 +17,7 @@
 
 import pytest
 from pyarrow import Codec
+from pyarrow import fs
 
 groups = [
     'brotli',
@@ -241,3 +242,25 @@ def _docdir(request):
 
     else:
         yield
+
+
+# Define doctest_namespace for fs module docstring import
+@pytest.fixture(autouse=True)
+def add_fs(doctest_namespace, request, tmp_path):
+
+    # Trigger ONLY for the doctests
+    doctest_m = request.config.option.doctestmodules
+    doctest_c = getattr(request.config.option, "doctest_cython", False)
+
+    if doctest_m or doctest_c:
+        # fs import
+        doctest_namespace["fs"] = fs
+
+        # Creation of an object and file with data
+        local = fs.LocalFileSystem()
+        path = tmp_path / 'fileinfo.dat'
+        with local.open_output_stream(str(path)) as stream:
+            stream.write(b'data')
+        doctest_namespace["local"] = local
+        doctest_namespace["local_path"] = tmp_path
+    yield
diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py
@@ -227,16 +227,28 @@ def copy_files(source, destination,
 
     Examples
     --------
-    Copy an S3 bucket's files to a local directory:
+    Inspect an S3 bucket's files:
 
-    >>> copy_files("s3://your-bucket-name",
-    ...            "local-directory") # doctest: +SKIP
+    >>> s3, path = fs.FileSystem.from_uri(
+    ...            "s3://registry.opendata.aws/roda/ndjson/")
+    >>> selector = fs.FileSelector(path)
+    >>> s3.get_file_info(selector)
+    [<FileInfo for 'registry.opendata.aws/roda/ndjson/index.ndjson':...]
 
-    Using a FileSystem object:
+    Copy one file from S3 bucket to a local directory:
 
-    >>> copy_files("your-bucket-name", "local-directory",
-    ...            source_filesystem=S3FileSystem(...)) # doctest: +SKIP
+    >>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
+    ...               "file:///{}/index_copy.ndjson".format(local_path))
 
+    >>> fs.LocalFileSystem().get_file_info(str(local_path)+
+    ...                                    '/index_copy.ndjson')
+    <FileInfo for '.../index_copy.ndjson': type=FileType.File, size=...>
+
+    Copy file using a FileSystem object:
+
+    >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
+    ...               "file:///{}/index_copy.ndjson".format(local_path),
+    ...               source_filesystem=fs.S3FileSystem())
     """
     source_fs, source_path = _resolve_filesystem_and_path(
         source, source_filesystem
