Skip to content

Commit 97e72ee

Browse files
wesmwesm
committed
ARROW-9251: [C++] Relocate integration testing JSON code implementation to src/arrow/testing
While this code is not being shipped in any packages, I think it would be better for it to live in the testing directory so that its purpose is clear I think there may be potentially some value in exposing `ArrayFromJSON` (in ipc/json_simple.h) in bindings at some point so I have left this code where it is, though it might be better to move it to arrow/json Closes apache#7557 from wesm/ARROW-9251 Authored-by: Wes McKinney <wesm@apache.org> Signed-off-by: Wes McKinney <wesm@apache.org>
1 parent 6848da1 commit 97e72ee

22 files changed

Lines changed: 724 additions & 707 deletions

‎cpp/src/arrow/CMakeLists.txt‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,6 @@ set(ARROW_SRCS
173173
io/memory.cc
174174
io/slow.cc
175175
io/transform.cc
176-
testing/util.cc
177176
util/basic_decimal.cc
178177
util/bit_block_counter.cc
179178
util/bit_run_reader.cc
@@ -279,9 +278,12 @@ endif()
279278
set(ARROW_TESTING_SRCS
280279
io/test_common.cc
281280
ipc/test_common.cc
281+
testing/json_integration.cc
282+
testing/json_internal.cc
282283
testing/gtest_util.cc
283284
testing/random.cc
284-
testing/generator.cc)
285+
testing/generator.cc
286+
testing/util.cc)
285287

286288
# Add dependencies for third-party allocators.
287289
# If possible we only want memory_pool.cc to wait for allocators to finish building,
@@ -409,8 +411,7 @@ if(ARROW_IPC)
409411
ipc/writer.cc)
410412

411413
if(ARROW_JSON)
412-
list(
413-
APPEND ARROW_SRCS ipc/json_integration.cc ipc/json_internal.cc ipc/json_simple.cc)
414+
list(APPEND ARROW_SRCS ipc/json_simple.cc)
414415
endif()
415416
endif()
416417

@@ -471,6 +472,17 @@ if(ARROW_BUILD_STATIC AND WIN32)
471472
target_compile_definitions(arrow_static PUBLIC ARROW_STATIC)
472473
endif()
473474

475+
if(ARROW_WITH_BACKTRACE)
476+
find_package(Backtrace)
477+
478+
foreach(LIB_TARGET ${ARROW_LIBRARIES})
479+
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_EXPORTING)
480+
if(Backtrace_FOUND AND ARROW_WITH_BACKTRACE)
481+
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_WITH_BACKTRACE)
482+
endif()
483+
endforeach()
484+
endif()
485+
474486
if(ARROW_BUILD_TESTS
475487
OR ARROW_BUILD_BENCHMARKS
476488
OR ARROW_BUILD_INTEGRATION
@@ -500,20 +512,11 @@ if(ARROW_BUILD_TESTS
500512
add_dependencies(arrow_testing ${ARROW_TESTING_LIBRARIES})
501513

502514
if(ARROW_BUILD_STATIC AND WIN32)
503-
target_compile_definitions(arrow_testing_static PUBLIC ARROW_STATIC)
515+
target_compile_definitions(arrow_testing_static PUBLIC ARROW_TESTING_STATIC)
504516
endif()
505517

506-
set(ARROW_LIBRARIES ${ARROW_LIBRARIES} ${ARROW_TESTING_LIBRARIES})
507-
endif()
508-
509-
if(ARROW_WITH_BACKTRACE)
510-
find_package(Backtrace)
511-
512-
foreach(LIB_TARGET ${ARROW_LIBRARIES})
513-
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_EXPORTING)
514-
if(Backtrace_FOUND AND ARROW_WITH_BACKTRACE)
515-
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_WITH_BACKTRACE)
516-
endif()
518+
foreach(LIB_TARGET ${ARROW_TESTING_LIBRARIES})
519+
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_TESTING_EXPORTING)
517520
endforeach()
518521
endif()
519522

‎cpp/src/arrow/csv/test_common.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,28 +22,28 @@
2222
#include <vector>
2323

2424
#include "arrow/csv/parser.h"
25-
#include "arrow/util/visibility.h"
25+
#include "arrow/testing/visibility.h"
2626

2727
namespace arrow {
2828
namespace csv {
2929

30-
ARROW_EXPORT
30+
ARROW_TESTING_EXPORT
3131
std::string MakeCSVData(std::vector<std::string> lines);
3232

3333
// Make a BlockParser from a vector of lines representing a CSV file
34-
ARROW_EXPORT
34+
ARROW_TESTING_EXPORT
3535
void MakeCSVParser(std::vector<std::string> lines, ParseOptions options, int32_t num_cols,
3636
std::shared_ptr<BlockParser>* out);
3737

38-
ARROW_EXPORT
38+
ARROW_TESTING_EXPORT
3939
void MakeCSVParser(std::vector<std::string> lines, ParseOptions options,
4040
std::shared_ptr<BlockParser>* out);
4141

42-
ARROW_EXPORT
42+
ARROW_TESTING_EXPORT
4343
void MakeCSVParser(std::vector<std::string> lines, std::shared_ptr<BlockParser>* out);
4444

4545
// Make a BlockParser from a vector of strings representing a single CSV column
46-
ARROW_EXPORT
46+
ARROW_TESTING_EXPORT
4747
void MakeColumnParser(std::vector<std::string> items, std::shared_ptr<BlockParser>* out);
4848

4949
} // namespace csv

‎cpp/src/arrow/filesystem/test_util.h‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <vector>
2424

2525
#include "arrow/filesystem/filesystem.h"
26+
#include "arrow/testing/visibility.h"
2627

2728
namespace arrow {
2829
namespace fs {
@@ -43,43 +44,43 @@ static inline FileInfo Dir(std::string path) {
4344
return info;
4445
}
4546

46-
ARROW_EXPORT
47+
ARROW_TESTING_EXPORT
4748
void CreateFile(FileSystem* fs, const std::string& path, const std::string& data);
4849

4950
// Sort a vector of FileInfo by lexicographic path order
50-
ARROW_EXPORT
51+
ARROW_TESTING_EXPORT
5152
void SortInfos(std::vector<FileInfo>* infos);
5253

53-
ARROW_EXPORT
54+
ARROW_TESTING_EXPORT
5455
void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type);
5556

56-
ARROW_EXPORT
57+
ARROW_TESTING_EXPORT
5758
void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type,
5859
TimePoint mtime);
5960

60-
ARROW_EXPORT
61+
ARROW_TESTING_EXPORT
6162
void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type,
6263
TimePoint mtime, int64_t size);
6364

64-
ARROW_EXPORT
65+
ARROW_TESTING_EXPORT
6566
void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type,
6667
int64_t size);
6768

68-
ARROW_EXPORT
69+
ARROW_TESTING_EXPORT
6970
void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type);
7071

71-
ARROW_EXPORT
72+
ARROW_TESTING_EXPORT
7273
void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type,
7374
TimePoint mtime);
7475

75-
ARROW_EXPORT
76+
ARROW_TESTING_EXPORT
7677
void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type,
7778
TimePoint mtime, int64_t size);
7879

79-
ARROW_EXPORT
80+
ARROW_TESTING_EXPORT
8081
void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type, int64_t size);
8182

82-
ARROW_EXPORT
83+
ARROW_TESTING_EXPORT
8384
void AssertFileContents(FileSystem* fs, const std::string& path,
8485
const std::string& expected_data);
8586

@@ -94,7 +95,7 @@ void AssertDurationBetween(Duration d, double min_secs, double max_secs) {
9495
// To use this class, subclass both from it and ::testing::Test,
9596
// implement GetEmptyFileSystem(), and use GENERIC_FS_TEST_FUNCTIONS()
9697
// to define the various tests.
97-
class ARROW_EXPORT GenericFileSystemTest {
98+
class ARROW_TESTING_EXPORT GenericFileSystemTest {
9899
public:
99100
virtual ~GenericFileSystemTest();
100101

‎cpp/src/arrow/flight/test_integration_client.cc‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,12 @@
3030
#include "arrow/io/file.h"
3131
#include "arrow/io/test_common.h"
3232
#include "arrow/ipc/dictionary.h"
33-
#include "arrow/ipc/json_integration.h"
3433
#include "arrow/ipc/writer.h"
3534
#include "arrow/record_batch.h"
3635
#include "arrow/table.h"
3736
#include "arrow/testing/extension_type.h"
3837
#include "arrow/testing/gtest_util.h"
38+
#include "arrow/testing/json_integration.h"
3939
#include "arrow/util/logging.h"
4040

4141
#include "arrow/flight/api.h"
@@ -51,7 +51,7 @@ namespace arrow {
5151
namespace flight {
5252

5353
/// \brief Helper to read all batches from a JsonReader
54-
Status ReadBatches(std::unique_ptr<ipc::internal::json::JsonReader>& reader,
54+
Status ReadBatches(std::unique_ptr<testing::IntegrationJsonReader>& reader,
5555
std::vector<std::shared_ptr<RecordBatch>>* chunks) {
5656
std::shared_ptr<RecordBatch> chunk;
5757
for (int i = 0; i < reader->num_record_batches(); i++) {
@@ -153,11 +153,11 @@ class IntegrationTestScenario : public flight::Scenario {
153153
FlightDescriptor descr{FlightDescriptor::PATH, "", {FLAGS_path}};
154154

155155
// 1. Put the data to the server.
156-
std::unique_ptr<ipc::internal::json::JsonReader> reader;
156+
std::unique_ptr<testing::IntegrationJsonReader> reader;
157157
std::cout << "Opening JSON file '" << FLAGS_path << "'" << std::endl;
158158
auto in_file = *io::ReadableFile::Open(FLAGS_path);
159159
ABORT_NOT_OK(
160-
ipc::internal::json::JsonReader::Open(default_memory_pool(), in_file, &reader));
160+
testing::IntegrationJsonReader::Open(default_memory_pool(), in_file, &reader));
161161

162162
std::shared_ptr<Schema> original_schema = reader->schema();
163163
std::vector<std::shared_ptr<RecordBatch>> original_data;

‎cpp/src/arrow/flight/test_integration_server.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@
2929
#include <gflags/gflags.h>
3030

3131
#include "arrow/io/test_common.h"
32-
#include "arrow/ipc/json_integration.h"
3332
#include "arrow/record_batch.h"
3433
#include "arrow/table.h"
34+
#include "arrow/testing/json_integration.h"
3535
#include "arrow/util/logging.h"
3636

3737
#include "arrow/flight/internal.h"

‎cpp/src/arrow/io/test_common.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,25 +21,25 @@
2121
#include <string>
2222
#include <vector>
2323

24+
#include "arrow/testing/visibility.h"
2425
#include "arrow/type_fwd.h"
25-
#include "arrow/util/visibility.h"
2626

2727
namespace arrow {
2828
namespace io {
2929

3030
class MemoryMappedFile;
3131

32-
ARROW_EXPORT
32+
ARROW_TESTING_EXPORT
3333
void AssertFileContents(const std::string& path, const std::string& contents);
3434

35-
ARROW_EXPORT bool FileExists(const std::string& path);
35+
ARROW_TESTING_EXPORT bool FileExists(const std::string& path);
3636

37-
ARROW_EXPORT bool FileIsClosed(int fd);
37+
ARROW_TESTING_EXPORT bool FileIsClosed(int fd);
3838

39-
ARROW_EXPORT
39+
ARROW_TESTING_EXPORT
4040
Status ZeroMemoryMap(MemoryMappedFile* file);
4141

42-
class ARROW_EXPORT MemoryMapFixture {
42+
class ARROW_TESTING_EXPORT MemoryMapFixture {
4343
public:
4444
void TearDown();
4545

‎cpp/src/arrow/ipc/CMakeLists.txt‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -47,22 +47,6 @@ endfunction()
4747
add_arrow_test(feather_test)
4848
add_arrow_ipc_test(read_write_test)
4949
add_arrow_ipc_test(json_simple_test)
50-
add_arrow_ipc_test(json_test)
51-
52-
# json_integration_test is two things at the same time:
53-
# - an executable that can be called to answer integration test requests
54-
# - a self-(unit)test for the C++ side of integration testing
55-
if(ARROW_BUILD_TESTS)
56-
add_arrow_test(json_integration_test EXTRA_LINK_LIBS ${GFLAGS_LIBRARIES})
57-
add_dependencies(arrow-integration arrow-json-integration-test)
58-
elseif(ARROW_BUILD_INTEGRATION)
59-
add_executable(arrow-json-integration-test json_integration_test.cc)
60-
target_link_libraries(arrow-json-integration-test ${ARROW_TEST_LINK_LIBS}
61-
${GFLAGS_LIBRARIES} GTest::gtest)
62-
63-
add_dependencies(arrow-json-integration-test arrow arrow_testing)
64-
add_dependencies(arrow-integration arrow-json-integration-test)
65-
endif()
6650

6751
# Headers: top level
6852
arrow_install_all_headers("arrow/ipc")

0 commit comments

Comments
 (0)