public:

GcsTestbench() {

port_ = std::to_string(GetListenPort());

std::vector<std::string> names{"python3", "python"};

// If the build script or application developer provides a value in the PYTHON

// environment variable, then just use that.

if (const auto* env = std::getenv("PYTHON")) {

names = {env};

}

auto error = std::string(

"Could not start GCS emulator."

" Used the following list of python interpreter names:");

for (const auto& interpreter : names) {

auto exe_path = bp::search_path(interpreter);

error += " " + interpreter;

if (exe_path.empty()) {

error += " (exe not found)";

continue;

}

bp::ipstream output;

server_process_ = bp::child(exe_path, "-m", "testbench", "--port", port_, group_,

bp::std_err > output);

// Wait for message: "* Restarting with"

auto testbench_is_running = [&output, this](bp::child& process) {

std::string line;

std::chrono::time_point<std::chrono::steady_clock> end =

std::chrono::steady_clock::now() + std::chrono::seconds(10);

while (server_process_.valid() && server_process_.running() &&

std::chrono::steady_clock::now() < end) {

if (output.peek() && std::getline(output, line)) {

std::cerr << line << std::endl;

if (line.find("* Restarting with") != std::string::npos) return true;

} else {

std::this_thread::sleep_for(std::chrono::milliseconds(20));

}

}

return false;

};

if (testbench_is_running(server_process_)) break;

error += " (failed to start)";

server_process_.terminate();

server_process_.wait();

}

if (server_process_.valid() && server_process_.valid()) return;

error_ = std::move(error);

}

bool running() { return server_process_.running(); }

~GcsTestbench() override {

// Brutal shutdown, kill the full process group because the GCS testbench may launch

// additional children.

group_.terminate();

if (server_process_.valid()) {

server_process_.wait();

}

}

const std::string& port() const { return port_; }

const std::string& error() const { return error_; }

private:

std::string port_;

bp::child server_process_;

bp::group group_;

std::string error_;

protected:

void SetUp() override {

ASSERT_THAT(Testbench(), NotNull());

ASSERT_EQ(Testbench()->error(), "");

ASSERT_TRUE(Testbench()->running());

// Initialize a PRNG with a small amount of entropy.

generator_ = std::mt19937_64(std::random_device()());

bucket_name_ = RandomChars(32);

// Create a bucket and a small file in the testbench. This makes it easier to

// bootstrap GcsFileSystem and its tests.

auto client = gcs::Client(

google::cloud::Options{}

.set<gcs::RestEndpointOption>("http://127.0.0.1:" + Testbench()->port())

.set<gc::UnifiedCredentialsOption>(gc::MakeInsecureCredentials())

.set<gcs::TransferStallTimeoutOption>(std::chrono::seconds(5))

.set<gcs::RetryPolicyOption>(

gcs::LimitedTimeRetryPolicy(std::chrono::seconds(45)).clone()));

google::cloud::StatusOr<gcs::BucketMetadata> bucket = client.CreateBucketForProject(

PreexistingBucketName(), "ignored-by-testbench", gcs::BucketMetadata{});

ASSERT_TRUE(bucket.ok()) << "Failed to create bucket <" << PreexistingBucketName()

<< ">, status=" << bucket.status();

google::cloud::StatusOr<gcs::ObjectMetadata> object = client.InsertObject(

PreexistingBucketName(), PreexistingObjectName(), kLoremIpsum);

ASSERT_TRUE(object.ok()) << "Failed to create object <" << PreexistingObjectName()

<< ">, status=" << object.status();

}

std::string PreexistingBucketName() const { return bucket_name_; }

std::string PreexistingBucketPath() const { return PreexistingBucketName() + '/'; }

static std::string PreexistingObjectName() { return "test-object-name"; }

std::string PreexistingObjectPath() const {

return PreexistingBucketPath() + PreexistingObjectName();

}

std::string NotFoundObjectPath() { return PreexistingBucketPath() + "not-found"; }

GcsOptions TestGcsOptions() {

auto options = GcsOptions::Anonymous();

options.endpoint_override = "127.0.0.1:" + Testbench()->port();

options.retry_limit_seconds = 60;

return options;

}

gcs::Client GcsClient() {

return gcs::Client(

google::cloud::Options{}

.set<gcs::RestEndpointOption>("http://127.0.0.1:" + Testbench()->port())

.set<gc::UnifiedCredentialsOption>(gc::MakeInsecureCredentials()));

}

std::string RandomLine(int lineno, std::size_t width) {

auto line = std::to_string(lineno) + ": ";

line += RandomChars(width - line.size() - 1);

line += '\n';

return line;

}

std::size_t RandomIndex(std::size_t end) {

return std::uniform_int_distribution<std::size_t>(0, end - 1)(generator_);

}

std::string RandomBucketName() { return RandomChars(32); }

std::string RandomFolderName() { return RandomChars(32); }

struct Hierarchy {

std::string base_dir;

std::vector<FileInfo> contents;

};

Result<Hierarchy> CreateHierarchy(std::shared_ptr<arrow::fs::FileSystem> fs) {

const char* const kTestFolders[] = {

"b",

"b/0",

"b/0/0",

"b/1",

"b/2",

// Create some additional folders that should not appear in any listing of b/

"aa",

"ba",

"c",

};

constexpr auto kFilesPerFolder = 2;

auto base_dir = internal::ConcatAbstractPath(PreexistingBucketPath(), "b");

auto result = Hierarchy{base_dir, {}};

for (auto const* f : kTestFolders) {

const auto folder = internal::ConcatAbstractPath(PreexistingBucketPath(), f);

RETURN_NOT_OK(fs->CreateDir(folder, true));

result.contents.push_back(arrow::fs::Dir(folder));

for (int i = 0; i != kFilesPerFolder; ++i) {

const auto filename =

internal::ConcatAbstractPath(folder, "test-file-" + std::to_string(i));

CreateFile(fs.get(), filename, filename);

result.contents.push_back(arrow::fs::File(filename));

}

}

return result;

}

// Directories must appear without a trailing slash in the results.

std::vector<arrow::fs::FileInfo> static CleanupDirectoryNames(

std::vector<arrow::fs::FileInfo> expected) {

std::transform(expected.begin(), expected.end(), expected.begin(),

[](FileInfo const& info) {

if (!info.IsDirectory()) return info;

return Dir(std::string(internal::RemoveTrailingSlash(info.path())));

});

return expected;

}

private:

std::string RandomChars(std::size_t count) {

auto const fillers = std::string("abcdefghijlkmnopqrstuvwxyz0123456789");

std::uniform_int_distribution<std::size_t> d(0, fillers.size() - 1);

std::string s;

std::generate_n(std::back_inserter(s), count, [&] { return fillers[d(generator_)]; });

return s;

}

std::mt19937_64 generator_;

std::string bucket_name_;

public:

void SetUp() override {

ASSERT_NO_FATAL_FAILURE(GcsIntegrationTest::SetUp());

auto bucket_name = RandomBucketName();

gcs_fs_ = GcsFileSystem::Make(TestGcsOptions());

ASSERT_OK(gcs_fs_->CreateDir(bucket_name, true));

fs_ = std::make_shared<SubTreeFileSystem>(bucket_name, gcs_fs_);

}

protected:

std::shared_ptr<FileSystem> GetEmptyFileSystem() override { return fs_; }

bool have_implicit_directories() const override { return true; }

bool allow_write_file_over_dir() const override { return true; }

bool allow_read_dir_as_file() const override { return true; }

bool allow_move_dir() const override { return false; }

bool allow_append_to_file() const override { return false; }

bool have_directory_mtimes() const override { return false; }

bool have_flaky_directory_tree_deletion() const override { return false; }

bool have_file_metadata() const override { return true; }

std::shared_ptr<GcsFileSystem> gcs_fs_;

std::shared_ptr<FileSystem> fs_;

auto a = GcsOptions::Anonymous();

auto b = a;

b.endpoint_override = "localhost:1234";

EXPECT_TRUE(a.Equals(a));

EXPECT_TRUE(b.Equals(b));

auto c = b;

c.scheme = "https";

EXPECT_FALSE(b.Equals(c));

GcsOptions a = GcsOptions::Anonymous();

EXPECT_THAT(a.credentials.holder(), NotNull());

EXPECT_TRUE(a.credentials.anonymous());

EXPECT_EQ(a.scheme, "http");

std::string path;

ASSERT_OK_AND_ASSIGN(GcsOptions options, GcsOptions::FromUri("gs://", &path));

EXPECT_EQ(options.default_bucket_location, "");

EXPECT_EQ(options.scheme, "https");

EXPECT_EQ(options.endpoint_override, "");

EXPECT_EQ(path, "");

ASSERT_OK_AND_ASSIGN(options, GcsOptions::FromUri("gs:", &path));

EXPECT_EQ(path, "");

// Username/password is unsupported

ASSERT_RAISES(Invalid, GcsOptions::FromUri("gs://access:secret@mybucket", &path));

ASSERT_OK_AND_ASSIGN(options, GcsOptions::FromUri("gs://mybucket/", &path));

EXPECT_EQ(options.default_bucket_location, "");

EXPECT_EQ(options.scheme, "https");

EXPECT_EQ(options.endpoint_override, "");

EXPECT_EQ(path, "mybucket");

ASSERT_OK_AND_ASSIGN(options, GcsOptions::FromUri("gs://mybucket/foo/bar/", &path));

EXPECT_EQ(options.default_bucket_location, "");

EXPECT_EQ(options.scheme, "https");

EXPECT_EQ(options.endpoint_override, "");

EXPECT_EQ(path, "mybucket/foo/bar");

// Explicit default_bucket_location override

ASSERT_OK_AND_ASSIGN(

options,

GcsOptions::FromUri("gs://mybucket/foo/bar/"

"?endpoint_override=localhost&scheme=http&location=us-west2"

"&retry_limit_seconds=40.5",

&path));

EXPECT_EQ(options.default_bucket_location, "us-west2");

EXPECT_EQ(options.scheme, "http");

EXPECT_EQ(options.endpoint_override, "localhost");

EXPECT_EQ(path, "mybucket/foo/bar");

ASSERT_TRUE(options.retry_limit_seconds.has_value());

EXPECT_EQ(*options.retry_limit_seconds, 40.5);

// Missing bucket name

ASSERT_RAISES(Invalid, GcsOptions::FromUri("gs:///foo/bar/", &path));

// Invalid option

ASSERT_RAISES(Invalid, GcsOptions::FromUri("gs://mybucket/?xxx=zzz", &path));

// Invalid retry limit

ASSERT_RAISES(Invalid,

GcsOptions::FromUri("gs://foo/bar/?retry_limit_seconds=0", &path));

ASSERT_RAISES(Invalid,

GcsOptions::FromUri("gs://foo/bar/?retry_limit_seconds=-1", &path));

}

TEST(GcsFileSystem, OptionsAccessToken) {

TimePoint expiration = std::chrono::system_clock::now() + std::chrono::minutes(5);

auto a = GcsOptions::FromAccessToken(/*access_token=*/"accesst", expiration);

EXPECT_THAT(a.credentials.holder(), NotNull());

EXPECT_THAT(a.credentials.access_token(), Eq("accesst"));

EXPECT_THAT(a.credentials.expiration(), Eq(expiration));

EXPECT_EQ(a.scheme, "https");

}

TEST(GcsFileSystem, OptionsImpersonateServiceAccount) {

TimePoint expiration = std::chrono::system_clock::now() + std::chrono::minutes(5);

auto base = GcsOptions::FromAccessToken(/*access_token=*/"at", expiration);

std::string account = "invalid-sa-test-only@my-project.iam.gserviceaccount.com";

auto a = GcsOptions::FromImpersonatedServiceAccount(base.credentials, account);

EXPECT_THAT(a.credentials.holder(), NotNull());

EXPECT_THAT(a.credentials.access_token(), Eq("at"));

EXPECT_THAT(a.credentials.expiration(), Eq(expiration));

EXPECT_THAT(a.credentials.target_service_account(), Eq(account));

EXPECT_EQ(a.scheme, "https");

}

TEST(GcsFileSystem, OptionsServiceAccountCredentials) {

// While this service account key has the correct format, it cannot be used for

// authentication because the key has been deactivated on the server-side, *and* the

// account(s) involved are deleted *and* they are not the accounts or projects do not

// match its contents.

constexpr char kJsonKeyfileContents[] = R"""({

})""";

auto a = GcsOptions::FromServiceAccountCredentials(kJsonKeyfileContents);

EXPECT_THAT(a.credentials.holder(), NotNull());

EXPECT_THAT(a.credentials.json_credentials(), kJsonKeyfileContents);

EXPECT_EQ(a.scheme, "https");

}

TEST(GcsFileSystem, ToArrowStatusOK) {

Status actual = internal::ToArrowStatus(google::cloud::Status());

EXPECT_TRUE(actual.ok());

}

TEST(GcsFileSystem, ToArrowStatus) {

struct {

google::cloud::StatusCode input;

arrow::StatusCode expected;

} cases[] = {

{google::cloud::StatusCode::kCancelled, StatusCode::Cancelled},

{google::cloud::StatusCode::kUnknown, StatusCode::UnknownError},

{google::cloud::StatusCode::kInvalidArgument, StatusCode::Invalid},

{google::cloud::StatusCode::kDeadlineExceeded, StatusCode::IOError},

{google::cloud::StatusCode::kNotFound, StatusCode::IOError},

{google::cloud::StatusCode::kAlreadyExists, StatusCode::AlreadyExists},

{google::cloud::StatusCode::kPermissionDenied, StatusCode::IOError},

{google::cloud::StatusCode::kUnauthenticated, StatusCode::IOError},

{google::cloud::StatusCode::kResourceExhausted, StatusCode::CapacityError},

{google::cloud::StatusCode::kFailedPrecondition, StatusCode::IOError},

{google::cloud::StatusCode::kAborted, StatusCode::IOError},

{google::cloud::StatusCode::kOutOfRange, StatusCode::Invalid},

{google::cloud::StatusCode::kUnimplemented, StatusCode::NotImplemented},

{google::cloud::StatusCode::kInternal, StatusCode::IOError},

{google::cloud::StatusCode::kUnavailable, StatusCode::IOError},

{google::cloud::StatusCode::kDataLoss, StatusCode::IOError},

};

for (const auto& test : cases) {

auto status = google::cloud::Status(test.input, "test-message");

auto message = [&] {

std::ostringstream os;

os << status;

return os.str();

}();

SCOPED_TRACE("Testing with status=" + message);

const auto actual = arrow::fs::internal::ToArrowStatus(status);

EXPECT_EQ(actual.code(), test.expected);

EXPECT_THAT(actual.message(), HasSubstr(message));

}

}

TEST(GcsFileSystem, FileSystemCompare) {

GcsOptions a_options;

a_options.scheme = "http";

auto a = GcsFileSystem::Make(a_options);

EXPECT_THAT(a, NotNull());

EXPECT_TRUE(a->Equals(*a));

GcsOptions b_options;

b_options.scheme = "http";

b_options.endpoint_override = "localhost:1234";

auto b = GcsFileSystem::Make(b_options);

EXPECT_THAT(b, NotNull());

EXPECT_TRUE(b->Equals(*b));

EXPECT_FALSE(a->Equals(*b));

}

std::shared_ptr<const KeyValueMetadata> KeyValueMetadataForTest() {

return arrow::key_value_metadata({

{"Cache-Control", "test-only-cache-control"},

{"Content-Disposition", "test-only-content-disposition"},

{"Content-Encoding", "test-only-content-encoding"},

{"Content-Language", "test-only-content-language"},

{"Content-Type", "test-only-content-type"},

{"customTime", "2021-10-26T01:02:03.456Z"},

{"storageClass", "test-only-storage-class"},

{"key", "test-only-value"},

{"kmsKeyName", "test-only-kms-key-name"},

{"predefinedAcl", "test-only-predefined-acl"},

// computed with: /bin/echo -n "01234567" | openssl base64

{"encryptionKeyBase64", "MDEyMzQ1Njc="},

});

}

TEST(GcsFileSystem, ToEncryptionKey) {

gcs::EncryptionKey key;

ASSERT_OK_AND_ASSIGN(key,

arrow::fs::internal::ToEncryptionKey(KeyValueMetadataForTest()));

ASSERT_TRUE(key.has_value());

EXPECT_EQ(key.value().algorithm, "AES256");

EXPECT_EQ(key.value().key, "MDEyMzQ1Njc=");

// /bin/echo -n "01234567" | sha256sum | awk '{printf("%s", $1);}' |

// xxd -r -p | openssl base64

// to get the SHA256 value of the key.

EXPECT_EQ(key.value().sha256, "kkWSubED8U+DP6r7Z/SAaR8BmIqkV8AGF2n1jNRzEbw=");

gcs::EncryptionKey key;

ASSERT_OK_AND_ASSIGN(key, arrow::fs::internal::ToEncryptionKey({}));

ASSERT_FALSE(key.has_value());

gcs::KmsKeyName key;

ASSERT_OK_AND_ASSIGN(key, arrow::fs::internal::ToKmsKeyName(KeyValueMetadataForTest()));

EXPECT_EQ(key.value_or(""), "test-only-kms-key-name");

gcs::KmsKeyName key;

ASSERT_OK_AND_ASSIGN(key, arrow::fs::internal::ToKmsKeyName({}));

ASSERT_FALSE(key.has_value());

gcs::PredefinedAcl predefined;

ASSERT_OK_AND_ASSIGN(predefined,

arrow::fs::internal::ToPredefinedAcl(KeyValueMetadataForTest()));

EXPECT_EQ(predefined.value_or(""), "test-only-predefined-acl");

gcs::PredefinedAcl predefined;

ASSERT_OK_AND_ASSIGN(predefined, arrow::fs::internal::ToPredefinedAcl({}));

ASSERT_FALSE(predefined.has_value());

gcs::WithObjectMetadata metadata;

ASSERT_OK_AND_ASSIGN(metadata,

arrow::fs::internal::ToObjectMetadata(KeyValueMetadataForTest()));

ASSERT_TRUE(metadata.has_value());

EXPECT_EQ(metadata.value().cache_control(), "test-only-cache-control");

EXPECT_EQ(metadata.value().content_disposition(), "test-only-content-disposition");

EXPECT_EQ(metadata.value().content_encoding(), "test-only-content-encoding");

EXPECT_EQ(metadata.value().content_language(), "test-only-content-language");

EXPECT_EQ(metadata.value().content_type(), "test-only-content-type");

ASSERT_TRUE(metadata.value().has_custom_time());

EXPECT_THAT(metadata.value().metadata(),

UnorderedElementsAre(Pair("key", "test-only-value")));

gcs::WithObjectMetadata metadata;

ASSERT_OK_AND_ASSIGN(metadata, arrow::fs::internal::ToObjectMetadata({}));

ASSERT_FALSE(metadata.has_value());

auto metadata = arrow::fs::internal::ToObjectMetadata(arrow::key_value_metadata({

{"customTime", "invalid"},

}));

EXPECT_EQ(metadata.status().code(), StatusCode::Invalid);

EXPECT_THAT(metadata.status().message(), HasSubstr("Error parsing RFC-3339"));

const auto source = KeyValueMetadataForTest();

gcs::WithObjectMetadata tmp;

ASSERT_OK_AND_ASSIGN(tmp, arrow::fs::internal::ToObjectMetadata(source));

std::shared_ptr<const KeyValueMetadata> destination;

ASSERT_OK_AND_ASSIGN(destination, arrow::fs::internal::FromObjectMetadata(tmp.value()));

// Only a subset of the keys are configurable in gcs::ObjectMetadata, for example, the

// size and CRC32C values of an object are only settable when the metadata is received

// from the services. For those keys that should be preserved, verify they are preserved

// in the round trip.

for (auto key : {"Cache-Control", "Content-Disposition", "Content-Encoding",

"Content-Language", "Content-Type", "storageClass"}) {

SCOPED_TRACE("Testing key " + std::string(key));

ASSERT_OK_AND_ASSIGN(auto s, source->Get(key));

ASSERT_OK_AND_ASSIGN(auto d, destination->Get(key));

EXPECT_EQ(s, d);

}

// RFC-3339 formatted timestamps may differ in trivial things, e.g., the UTC timezone

// can be represented as 'Z' or '+00:00'.

ASSERT_OK_AND_ASSIGN(auto s, source->Get("customTime"));

ASSERT_OK_AND_ASSIGN(auto d, destination->Get("customTime"));

std::string err;

absl::Time ts;

absl::Time td;

ASSERT_TRUE(absl::ParseTime(absl::RFC3339_full, s, &ts, &err)) << "error=" << err;

ASSERT_TRUE(absl::ParseTime(absl::RFC3339_full, d, &td, &err)) << "error=" << err;

EXPECT_NEAR(absl::ToDoubleSeconds(ts - td), 0, 0.5);

}

TEST_F(GcsIntegrationTest, GetFileInfoBucket) {

auto fs = GcsFileSystem::Make(TestGcsOptions());

arrow::fs::AssertFileInfo(fs.get(), PreexistingBucketName(), FileType::Directory);

// URI

ASSERT_RAISES(Invalid, fs->GetFileInfo("gs://" + PreexistingBucketName()));

}

TEST_F(GcsIntegrationTest, GetFileInfoObjectWithNestedStructure) {

// Adds detailed tests to handle cases of different edge cases

// with directory naming conventions (e.g. with and without slashes).

auto fs = GcsFileSystem::Make(TestGcsOptions());

constexpr auto kObjectName = "test-object-dir/some_other_dir/another_dir/foo";

ASSERT_OK_AND_ASSIGN(

auto output,

fs->OpenOutputStream(PreexistingBucketPath() + kObjectName, /*metadata=*/{}));

const auto data = std::string(kLoremIpsum);

ASSERT_OK(output->Write(data.data(), data.size()));

ASSERT_OK(output->Close());

// 0 is immediately after "/" lexicographically, ensure that this doesn't

// cause unexpected issues.

ASSERT_OK_AND_ASSIGN(output, fs->OpenOutputStream(PreexistingBucketPath() +

"test-object-dir/some_other_dir0",

/*metadata=*/{}));

ASSERT_OK(output->Write(data.data(), data.size()));

ASSERT_OK(output->Close());

ASSERT_OK_AND_ASSIGN(

output,

fs->OpenOutputStream(PreexistingBucketPath() + kObjectName + "0", /*metadata=*/{}));

ASSERT_OK(output->Write(data.data(), data.size()));

ASSERT_OK(output->Close());

AssertFileInfo(fs.get(), PreexistingBucketPath() + kObjectName, FileType::File);

AssertFileInfo(fs.get(), PreexistingBucketPath() + kObjectName + "/",

FileType::NotFound);

AssertFileInfo(fs.get(), PreexistingBucketPath() + "test-object-dir",

FileType::Directory);

AssertFileInfo(fs.get(), PreexistingBucketPath() + "test-object-dir/",

FileType::Directory);

AssertFileInfo(fs.get(), PreexistingBucketPath() + "test-object-dir/some_other_dir",

FileType::Directory);

AssertFileInfo(fs.get(), PreexistingBucketPath() + "test-object-dir/some_other_dir/",

FileType::Directory);

AssertFileInfo(fs.get(), PreexistingBucketPath() + "test-object-di",

FileType::NotFound);

AssertFileInfo(fs.get(), PreexistingBucketPath() + "test-object-dir/some_other_di",

FileType::NotFound);

auto fs = GcsFileSystem::Make(TestGcsOptions());

auto object =

GcsClient().GetObjectMetadata(PreexistingBucketName(), PreexistingObjectName());

ASSERT_TRUE(object.ok()) << "status=" << object.status();

arrow::fs::AssertFileInfo(fs.get(), PreexistingObjectPath(), FileType::File,

object->time_created(), static_cast<int64_t>(object->size()));

// URI

ASSERT_RAISES(Invalid, fs->GetFileInfo("gs://" + PreexistingObjectName()));

auto fs = GcsFileSystem::Make(TestGcsOptions());

ASSERT_OK_AND_ASSIGN(auto hierarchy, CreateHierarchy(fs));

std::vector<arrow::fs::FileInfo> expected;

std::copy_if(hierarchy.contents.begin(), hierarchy.contents.end(),

std::back_inserter(expected), [&](const arrow::fs::FileInfo& info) {

if (!fs::internal::IsAncestorOf(hierarchy.base_dir, info.path())) {

return false;

}

return hierarchy.base_dir != info.path();

});

expected = CleanupDirectoryNames(std::move(expected));

auto selector = FileSelector();

selector.base_dir = hierarchy.base_dir;

selector.allow_not_found = false;

selector.recursive = true;

selector.max_recursion = 16;

ASSERT_OK_AND_ASSIGN(auto results, fs->GetFileInfo(selector));

EXPECT_THAT(results, UnorderedElementsAreArray(expected.begin(), expected.end()));

// URI

selector.base_dir = "gs://" + selector.base_dir;

ASSERT_RAISES(Invalid, fs->GetFileInfo(selector));

auto fs = GcsFileSystem::Make(TestGcsOptions());

ASSERT_OK_AND_ASSIGN(auto hierarchy, CreateHierarchy(fs));

std::vector<arrow::fs::FileInfo> expected;

std::copy_if(hierarchy.contents.begin(), hierarchy.contents.end(),

std::back_inserter(expected), [&](const arrow::fs::FileInfo& info) {

if (info.path() == hierarchy.base_dir) return false;

return internal::GetAbstractPathParent(info.path()).first ==

hierarchy.base_dir;

});

auto selector = FileSelector();

selector.base_dir = hierarchy.base_dir;

selector.allow_not_found = false;

selector.recursive = false;

ASSERT_OK_AND_ASSIGN(auto results, fs->GetFileInfo(selector));

EXPECT_THAT(results, UnorderedElementsAreArray(expected.begin(), expected.end()));

auto fs = GcsFileSystem::Make(TestGcsOptions());

ASSERT_OK_AND_ASSIGN(auto hierarchy, CreateHierarchy(fs));

for (const auto max_recursion : {0, 1, 2, 3}) {

SCOPED_TRACE("Testing with max_recursion=" + std::to_string(max_recursion));

const auto max_depth =

internal::Depth(internal::EnsureTrailingSlash(hierarchy.base_dir)) +

max_recursion + 1; // Add 1 because files in a directory should be included

std::vector<arrow::fs::FileInfo> expected;

std::copy_if(hierarchy.contents.begin(), hierarchy.contents.end(),

std::back_inserter(expected), [&](const arrow::fs::FileInfo& info) {

if (info.path() == hierarchy.base_dir) return false;

if (!fs::internal::IsAncestorOf(hierarchy.base_dir, info.path())) {

return false;

}

return internal::Depth(info.path()) <= max_depth;

});

expected = CleanupDirectoryNames(std::move(expected));

auto selector = FileSelector();

selector.base_dir = hierarchy.base_dir;

selector.allow_not_found = true;

selector.recursive = true;

selector.max_recursion = max_recursion;

ASSERT_OK_AND_ASSIGN(auto results, fs->GetFileInfo(selector));

EXPECT_THAT(results, UnorderedElementsAreArray(expected.begin(), expected.end()));

}

auto fs = GcsFileSystem::Make(TestGcsOptions());

auto selector = FileSelector();

selector.base_dir = NotFoundObjectPath() + "/";

selector.allow_not_found = true;

selector.recursive = true;

ASSERT_OK_AND_ASSIGN(auto results, fs->GetFileInfo(selector));

EXPECT_EQ(results.size(), 0);

auto fs = GcsFileSystem::Make(TestGcsOptions());

auto selector = FileSelector();

selector.base_dir = NotFoundObjectPath() + "/";

selector.allow_not_found = false;

selector.recursive = false;

ASSERT_RAISES(IOError, fs->GetFileInfo(selector));

auto fs = GcsFileSystem::Make(TestGcsOptions());

auto bucket_name = RandomBucketName();

ASSERT_OK(fs->CreateDir(bucket_name, false));

arrow::fs::AssertFileInfo(fs.get(), bucket_name, FileType::Directory);

auto fs = GcsFileSystem::Make(TestGcsOptions());

const auto path = PreexistingBucketPath() + RandomFolderName();

ASSERT_OK(fs->CreateDir(path, false));

arrow::fs::AssertFileInfo(fs.get(), path, FileType::Directory);

auto fs = GcsFileSystem::Make(TestGcsOptions());

const auto path = std::string("not-a-bucket/new-folder");

ASSERT_RAISES(IOError, fs->CreateDir(path, false));

auto fs = GcsFileSystem::Make(TestGcsOptions());

auto bucket_name = RandomBucketName();

ASSERT_OK(fs->CreateDir(bucket_name, true));

arrow::fs::AssertFileInfo(fs.get(), bucket_name, FileType::Directory);

auto fs = GcsFileSystem::Make(TestGcsOptions());

const auto parent = PreexistingBucketPath() + RandomFolderName();

const auto path = internal::ConcatAbstractPath(parent, "new-sub");

ASSERT_OK(fs->CreateDir(path, true));

arrow::fs::AssertFileInfo(fs.get(), path, FileType::Directory);

arrow::fs::AssertFileInfo(fs.get(), parent, FileType::Directory);

auto fs = GcsFileSystem::Make(TestGcsOptions());

auto bucket_name = RandomBucketName();

const auto parent = internal::ConcatAbstractPath(bucket_name, RandomFolderName());

const auto path = internal::ConcatAbstractPath(parent, "new-sub");

ASSERT_OK(fs->CreateDir(path, true));

arrow::fs::AssertFileInfo(fs.get(), path, FileType::Directory);

arrow::fs::AssertFileInfo(fs.get(), parent, FileType::Directory);

arrow::fs::AssertFileInfo(fs.get(), bucket_name, FileType::Directory);

auto fs = GcsFileSystem::Make(TestGcsOptions());

ASSERT_OK(fs->CreateDir("pyarrow-filesystem/", /*recursive=*/true));

ASSERT_RAISES(Invalid, fs->CreateDir("/", false));

ASSERT_OK(fs->DeleteDir("pyarrow-filesystem/"));

auto fs = GcsFileSystem::Make(TestGcsOptions());

ASSERT_OK_AND_ASSIGN(auto hierarchy, CreateHierarchy(fs));

ASSERT_OK(fs->DeleteDir(hierarchy.base_dir));

arrow::fs::AssertFileInfo(fs.get(), PreexistingBucketName(), FileType::Directory);

arrow::fs::AssertFileInfo(fs.get(), PreexistingObjectPath(), FileType::File);

for (auto const& info : hierarchy.contents) {

const auto expected_type = fs::internal::IsAncestorOf(hierarchy.base_dir, info.path())

? FileType::NotFound

: info.type();

arrow::fs::AssertFileInfo(fs.get(), info.path(), expected_type);

}

auto fs = GcsFileSystem::Make(TestGcsOptions());

ASSERT_OK_AND_ASSIGN(auto hierarchy, CreateHierarchy(fs));

ASSERT_OK(fs->DeleteDirContents(hierarchy.base_dir));

arrow::fs::AssertFileInfo(fs.get(), hierarchy.base_dir, FileType::Directory);

arrow::fs::AssertFileInfo(fs.get(), PreexistingBucketName(), FileType::Directory);

arrow::fs::AssertFileInfo(fs.get(), PreexistingObjectPath(), FileType::File);

for (auto const& info : hierarchy.contents) {

auto expected_type = FileType::NotFound;

if (info.path() == hierarchy.base_dir ||

!fs::internal::IsAncestorOf(hierarchy.base_dir, info.path())) {

expected_type = info.type();

}

arrow::fs::AssertFileInfo(fs.get(), info.path(), expected_type);

}

auto fs = GcsFileSystem::Make(TestGcsOptions());

EXPECT_RAISES_WITH_MESSAGE_THAT(NotImplemented, HasSubstr("too dangerous"),

fs->DeleteRootDirContents());

auto fs = GcsFileSystem::Make(TestGcsOptions());

ASSERT_OK(fs->DeleteFile(PreexistingObjectPath()));

arrow::fs::AssertFileInfo(fs.get(), PreexistingObjectPath(), FileType::NotFound);

auto fs = GcsFileSystem::Make(TestGcsOptions());

const auto path = PreexistingBucketPath() + "DeleteFileDirectoryFails/";

ASSERT_RAISES(IOError, fs->DeleteFile(path));

auto fs = GcsFileSystem::Make(TestGcsOptions());

const auto destination_path = PreexistingBucketPath() + "move-destination";

ASSERT_OK(fs->Move(PreexistingObjectPath(), destination_path));

arrow::fs::AssertFileInfo(fs.get(), destination_path, FileType::File);

arrow::fs::AssertFileInfo(fs.get(), PreexistingObjectPath(), FileType::NotFound);

auto fs = GcsFileSystem::Make(TestGcsOptions());

ASSERT_RAISES(IOError, fs->Move(PreexistingBucketPath() + "folder/",

PreexistingBucketPath() + "new-name"));

auto fs = GcsFileSystem::Make(TestGcsOptions());

ASSERT_OK(fs->CreateDir(PreexistingBucketPath() + "destination", false));

ASSERT_RAISES(IOError, fs->Move(PreexistingObjectPath(),

PreexistingBucketPath() + "destination"));

auto fs = GcsFileSystem::Make(TestGcsOptions());

const auto destination_path = PreexistingBucketPath() + "move-destination";

ASSERT_RAISES(Invalid, fs->Move("gs://" + PreexistingObjectPath(), destination_path));

ASSERT_RAISES(Invalid, fs->Move(PreexistingObjectPath(), "gs://" + destination_path));

auto fs = GcsFileSystem::Make(TestGcsOptions());

const auto destination_path = PreexistingBucketPath() + "copy-destination";

ASSERT_OK(fs->CopyFile(PreexistingObjectPath(), destination_path));

arrow::fs::AssertFileInfo(fs.get(), destination_path, FileType::File);

auto fs = GcsFileSystem::Make(TestGcsOptions());

const auto destination_path = PreexistingBucketPath() + "copy-destination";

ASSERT_RAISES(IOError, fs->CopyFile(NotFoundObjectPath(), destination_path));

auto fs = GcsFileSystem::Make(TestGcsOptions());

const auto destination_path = PreexistingBucketPath() + "copy-destination";

ASSERT_RAISES(Invalid,

fs->CopyFile("gs://" + PreexistingObjectPath(), destination_path));

ASSERT_RAISES(Invalid,

fs->CopyFile(PreexistingObjectPath(), "gs://" + destination_path));

auto fs = GcsFileSystem::Make(TestGcsOptions());

std::shared_ptr<io::InputStream> stream;

ASSERT_OK_AND_ASSIGN(stream, fs->OpenInputStream(PreexistingObjectPath()));

std::array<char, 1024> buffer{};

std::int64_t size;

ASSERT_OK_AND_ASSIGN(size, stream->Read(buffer.size(), buffer.data()));

EXPECT_EQ(std::string(buffer.data(), size), kLoremIpsum);

auto fs = GcsFileSystem::Make(TestGcsOptions());

std::shared_ptr<io::InputStream> stream;

ASSERT_OK_AND_ASSIGN(stream, fs->OpenInputStream(PreexistingObjectPath()));

std::string contents;

std::shared_ptr<Buffer> buffer;

do {

ASSERT_OK_AND_ASSIGN(buffer, stream->Read(16));

contents.append(buffer->ToString());

} while (buffer && buffer->size() != 0);

EXPECT_EQ(contents, kLoremIpsum);

auto fs = GcsFileSystem::Make(TestGcsOptions());

arrow::fs::FileInfo info;

ASSERT_OK_AND_ASSIGN(info, fs->GetFileInfo(PreexistingObjectPath()));

std::shared_ptr<io::InputStream> stream;

ASSERT_OK_AND_ASSIGN(stream, fs->OpenInputStream(info));